# /robots.txt file for http://webcrawler.com/
# mail webmaster@webcrawler.com for constructive criticism

User-agent:webcrawler
Disallow:/cgi
Disallow:/bb_test
Disallow:/formdata
Disallow:/stats
Disallow:/SpryAssets


User-agent:*
Disallow:/cgi
Disallow:/bb_test
Disallow:/formdata
Disallow:/stats
Disallow:/SpryAssets