#
# This file is used to allow crawlers to index our site. 
#
# List of all web robots: http://www.robotstxt.org
#
# Check robots.txt at:
# http://www.searchengineworld.com/cgi-bin/robotcheck.cgi
#
# Details about Googlebot available at: http://www.google.com/bot.html
# The Google and Yahoo search engine can see everything
User-agent: Slurp 
Disallow: /downloads/
Disallow: /images/
Disallow: /pdfs/
Disallow: /ftp/
Disallow: /admin/
Disallow: /js/

User-agent: Googlebot 
Disallow: /downloads/
Disallow: /images/
Disallow: /pdfs/
Disallow: /ftp/
Disallow: /admin/
Disallow: /js/

# All other robots will be restricted from accessing the Google-specific index pages
User-agent: * 
Disallow: /downloads/
Disallow: /images/
Disallow: /pdfs/
Disallow: /ftp/
Disallow: /admin/
Disallow: /js/