118 lines
1.8 KiB
Plaintext
118 lines
1.8 KiB
Plaintext
|
User-agent: Googlebot-Mobile
|
||
|
|
Disallow: /
|
||
|
|
|
||
|
|
User-agent: AdsBot-Google
|
||
|
|
Disallow: /
|
||
|
|
|
||
|
|
User-agent: Yahoo Pipes 2.0
|
||
|
|
Disallow: /
|
||
|
|
|
||
|
|
User-agent: AhrefsBot
|
||
|
|
Disallow: /
|
||
|
|
|
||
|
|
User-agent: asterias
|
||
|
|
Disallow: /
|
||
|
|
|
||
|
|
User-agent: TurnitinBot
|
||
|
|
Disallow: /
|
||
|
|
|
||
|
|
User-Agent: Oracle Secure Enterprise Search
|
||
|
|
Disallow: /
|
||
|
|
|
||
|
|
# User-agent: Baiduspider
|
||
|
|
# Disallow: /
|
||
|
|
|
||
|
|
# User-agent: Yandex
|
||
|
|
# Disallow: /
|
||
|
|
|
||
|
|
User-Agent: Mail.RU_Bot
|
||
|
|
Disallow: /
|
||
|
|
|
||
|
|
User-agent: YisouSpider
|
||
|
|
Disallow: /
|
||
|
|
|
||
|
|
User-agent: EasouSpider
|
||
|
|
Disallow: /
|
||
|
|
|
||
|
|
User-agent: Sosospider
|
||
|
|
Disallow: /
|
||
|
|
|
||
|
|
User-agent: Riddler
|
||
|
|
Disallow: /
|
||
|
|
|
||
|
|
User-agent: Daumoa
|
||
|
|
Disallow: /
|
||
|
|
|
||
|
|
User-agent: Exabot
|
||
|
|
Disallow: /
|
||
|
|
|
||
|
|
User-agent: NerdyBot
|
||
|
|
Disallow: /
|
||
|
|
|
||
|
|
User-agent: 008
|
||
|
|
Disallow: /
|
||
|
|
|
||
|
|
User-agent: ccbot
|
||
|
|
Disallow: /
|
||
|
|
|
||
|
|
User-agent: discobot
|
||
|
|
Disallow: /
|
||
|
|
|
||
|
|
User-agent: OmegaSeek
|
||
|
|
Disallow: /
|
||
|
|
|
||
|
|
User-agent: discoverybot
|
||
|
|
Disallow: /
|
||
|
|
|
||
|
|
User-agent: MJ12bot
|
||
|
|
Disallow: /
|
||
|
|
|
||
|
|
User-agent: wotbox
|
||
|
|
Disallow: /
|
||
|
|
|
||
|
|
User-agent: yacy
|
||
|
|
Disallow: /
|
||
|
|
|
||
|
|
User-agent: Twitterbot
|
||
|
|
Disallow:
|
||
|
|
|
||
|
|
User-agent: Blekkobot
|
||
|
|
Disallow: /
|
||
|
|
|
||
|
|
User-agent: Abonti
|
||
|
|
Disallow: /
|
||
|
|
|
||
|
|
user-agent: CrystalSemanticsBot
|
||
|
|
Disallow: /
|
||
|
|
|
||
|
|
User-agent: *
|
||
|
|
Disallow: /etext
|
||
|
|
Disallow: /dirs/etext
|
||
|
|
Disallow: /dirs/1
|
||
|
|
Disallow: /dirs/2
|
||
|
|
Disallow: /dirs/3
|
||
|
|
Disallow: /dirs/4
|
||
|
|
Disallow: /dirs/5
|
||
|
|
Disallow: /dirs/6
|
||
|
|
Disallow: /dirs/7
|
||
|
|
Disallow: /dirs/8
|
||
|
|
Disallow: /dirs/9
|
||
|
|
Disallow: /catalog/world/ # old search engine
|
||
|
|
Disallow: /ebooks/search # new search engine
|
||
|
|
Disallow: /ebooks/send/ # dropbox et al.
|
||
|
|
Disallow: /ratelimiter
|
||
|
|
Disallow: /zipcat
|
||
|
|
Disallow: /addresses/ # don't remove, ask webmaster
|
||
|
|
Disallow: /author/
|
||
|
|
Disallow: /internal/ # private
|
||
|
|
Disallow: /etext/magnet # against brain damaged robots
|
||
|
|
Disallow: /cache/ # on-the-fly generated files
|
||
|
|
Disallow: /cdproject/
|
||
|
|
Disallow: /w/ # wiki edit pages etc.
|
||
|
|
Disallow: /wiki/Special:Search
|
||
|
|
Disallow: /wiki/Special:Random
|
||
|
|
|
||
|
|
Crawl-delay: 5
|
||
|
|
|
||
|
|
Sitemap: http://www.gutenberg.org/ebooks/sitemaps/
|