# Make changes for all web spiders
User-agent: *
Disallow: / 

# Tell "WebSpider" where it can't go
# User-agent: WebSpider
# Disallow: /marketing
# Disallow: /sales

# Allow all other robots to browse everywhere
# User-agent: *
# Disallow:

# Allow "SpiderOne" in the site
# User-agent: SpiderOne
# Disallow:

# Deny all other spiders
# User-agent: *
# Disallow: / 

# Ignore FrontPage files
# User-agent: *
# Disallow: /_borders
# Disallow: /_derived
# Disallow: /_fpclass
# Disallow: /_overlay
# Disallow: /_private
# Disallow: /_themes
# Disallow: /_vti_bin
# Disallow: /_vti_cnf
# Disallow: /_vti_log
# Disallow: /_vti_map
# Disallow: /_vti_pvt
# Disallow: /_vti_txt