Update list of crawlers
authorJoey Schulze <joey@infodrom.org>
Thu, 28 Jun 2018 12:29:56 +0000 (14:29 +0200)
committerJoey Schulze <joey@infodrom.org>
Thu, 28 Jun 2018 12:29:56 +0000 (14:29 +0200)
lib/core.inc

index 8026a71..6d6be04 100644 (file)
@@ -218,6 +218,22 @@ function is_spider()
                         'Seobility',
                         'MetaJobBot',
                         'BellPagesCA/1.0',
                         'Seobility',
                         'MetaJobBot',
                         'BellPagesCA/1.0',
+                        'SafeDNSBot',
+                        'Uptimebot/1.0',
+                        'DomainStatsBot/1.0',
+                        'MetaJobBot',
+                        'SEOkicks-Robot',
+                        'Cliqzbot',
+                        'bingbot/3',
+                        'SEOkicks-Robot',
+                        'MojeekBot/',
+                        'oBot/',
+                        'CCBot/',
+                        'TurnitinBot',
+                        'ips-agent',
+                        'ExtLinksBot/',
+                        'Barkrowler/',
+                        'adscanner/',
                         'Cityreview Robot');
 
   foreach ($known_spiders as $spider_id) {
                         'Cityreview Robot');
 
   foreach ($known_spiders as $spider_id) {