# We allow search engines to crawl dogweb, but not other parts of the site
User-agent: Googlebot
User-agent: ia_archiver #Alexa
User-agent: BingPreview #Microsoft
User-agent: bingbot #Microsoft
User-agent: MSNBot #Microsoft
User-agent: Slurp #Yahoo
User-agent: Ask Jeeves/Teoma #Ask
Allow: /hundedatabase/
Disallow: /
Crawl-delay: 5

# Nothing else is allowed to crawl the site. Webscraping is NOT allowed
User-agent: *
Disallow: /