Update list of crawlers

diff --git a/lib/core.inc b/lib/core.inc
index 8026a71..6d6be04 100644 (file)
--- a/lib/core.inc
+++ b/lib/core.inc
@@ -218,6 +218,22 @@ function is_spider()
                         'Seobility',
                         'MetaJobBot',
                         'BellPagesCA/1.0',
+                        'SafeDNSBot',
+                        'Uptimebot/1.0',
+                        'DomainStatsBot/1.0',
+                        'MetaJobBot',
+                        'SEOkicks-Robot',
+                        'Cliqzbot',
+                        'bingbot/3',
+                        'SEOkicks-Robot',
+                        'MojeekBot/',
+                        'oBot/',
+                        'CCBot/',
+                        'TurnitinBot',
+                        'ips-agent',
+                        'ExtLinksBot/',
+                        'Barkrowler/',
+                        'adscanner/',
                         'Cityreview Robot');
 
   foreach ($known_spiders as $spider_id) {