# robots.txt for http://dir-diver.com

User-agent: *
Disallow: /text
Disallow: /cgi-bin
Disallow: /files