# /robots.txt file for http://webcrawler.com/ # mail webmaster@webcrawler.com for constructive criticism User-agent: webcrawler Disallow: /collister Disallow: /bignuts-c/* Disallow: /displaywright Disallow: /hcls Disallow: /boyd Disallow: /terra-orig Disallow: /SEI Disallow: /sulzch Disallow: /culinary Disallow: /netcdrom Disallow: /golf_real_estate Disallow: /ajemmite Disallow: /houstonproducersforum Disallow: /clayton Disallow: /skeeters Disallow: /terramark Disallow: /foreclosehouston User-agent: Googlebot Disallow: /collister Disallow: /bignuts-c/* Disallow: /displaywright Disallow: /hcls Disallow: /boyd Disallow: /terra-orig Disallow: /SEI Disallow: /sulzch Disallow: /culinary Disallow: /netcdrom Disallow: /golf_real_estate Disallow: /ajemmite Disallow: /houstonproducersforum Disallow: /clayton Disallow: /skeeters Disallow: /terramark Disallow: /foreclosehouston User-agent: Yahoo Disallow: /collister Disallow: /bignuts-c/* Disallow: /displaywright Disallow: /hcls Disallow: /boyd Disallow: /terra-orig Disallow: /SEI Disallow: /sulzch Disallow: /culinary Disallow: /netcdrom Disallow: /golf_real_estate Disallow: /ajemmite Disallow: /houstonproducersforum Disallow: /clayton Disallow: /skeeters Disallow: /terramark Disallow: /foreclosehouston User-agent: MSNbot Disallow: /collister Disallow: /bignuts-c/* Disallow: /displaywright Disallow: /hcls Disallow: /boyd Disallow: /terra-orig Disallow: /SEI Disallow: /sulzch Disallow: /culinary Disallow: /netcdrom Disallow: /golf_real_estate Disallow: /ajemmite Disallow: /houstonproducersforum Disallow: /clayton Disallow: /skeeters Disallow: /terramark Disallow: /foreclosehouston User-agent: Excite Disallow: /collister Disallow: /bignuts-c/* Disallow: /displaywright Disallow: /hcls Disallow: /boyd Disallow: /terra-orig Disallow: /SEI Disallow: /sulzch Disallow: /culinary Disallow: /netcdrom Disallow: /golf_real_estate Disallow: /ajemmite Disallow: /houstonproducersforum Disallow: /clayton Disallow: /skeeters Disallow: /terramark Disallow: /foreclosehouston User-agent: Lycos Disallow: /collister Disallow: /bignuts-c/* Disallow: /displaywright Disallow: /hcls Disallow: /boyd Disallow: /terra-orig Disallow: /SEI Disallow: /sulzch Disallow: /culinary Disallow: /netcdrom Disallow: /golf_real_estate Disallow: /ajemmite Disallow: /houstonproducersforum Disallow: /clayton Disallow: /skeeters Disallow: /terramark Disallow: /foreclosehouston User-agent: EntireWeb Disallow: /collister Disallow: /bignuts-c Disallow: /displaywright Disallow: /hcls Disallow: /boyd Disallow: /terra-orig Disallow: /SEI Disallow: /sulzch Disallow: /culinary Disallow: /netcdrom Disallow: /golf_real_estate Disallow: /ajemmite Disallow: /houstonproducersforum Disallow: /clayton Disallow: /skeeters Disallow: /terramark Disallow: /foreclosehouston User-agent: Exactseek Disallow: /collister Disallow: /bignuts-c Disallow: /displaywright Disallow: /hcls Disallow: /boyd Disallow: /terra-orig Disallow: /SEI Disallow: /sulzch Disallow: /culinary Disallow: /netcdrom Disallow: /golf_real_estate Disallow: /ajemmite Disallow: /houstonproducersforum Disallow: /clayton Disallow: /skeeters Disallow: /terramark Disallow: /foreclosehouston