User-Agent: *
Allow: /

User-Agent: MJ12bot
Disallow: /

# Yahoo Slurp
# See http://help.yahoo.com/l/us/yahoo/search/webcrawler/slurp-03.html
User-agent: Slurp
Crawl-delay: 100
Disallow:

User-agent: dotbot 
Disallow: /