projects
/
infodrom.org
/
www.zeitungsliste.de
/ commitdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
| commitdiff |
tree
raw
|
patch
| inline |
side by side
(parent:
0fab2d8
)
Update list of crawlers
author
Joey Schulze
<joey@infodrom.org>
Thu, 28 Jun 2018 12:29:56 +0000
(14:29 +0200)
committer
Joey Schulze
<joey@infodrom.org>
Thu, 28 Jun 2018 12:29:56 +0000
(14:29 +0200)
lib/core.inc
patch
|
blob
|
history
diff --git
a/lib/core.inc
b/lib/core.inc
index
8026a71
..
6d6be04
100644
(file)
--- a/
lib/core.inc
+++ b/
lib/core.inc
@@
-218,6
+218,22
@@
function is_spider()
'Seobility',
'MetaJobBot',
'BellPagesCA/1.0',
+ 'SafeDNSBot',
+ 'Uptimebot/1.0',
+ 'DomainStatsBot/1.0',
+ 'MetaJobBot',
+ 'SEOkicks-Robot',
+ 'Cliqzbot',
+ 'bingbot/3',
+ 'SEOkicks-Robot',
+ 'MojeekBot/',
+ 'oBot/',
+ 'CCBot/',
+ 'TurnitinBot',
+ 'ips-agent',
+ 'ExtLinksBot/',
+ 'Barkrowler/',
+ 'adscanner/',
'Cityreview Robot');
foreach ($known_spiders as $spider_id) {