# /robots.txt file for http://webcrawler.com/
# mail webmaster@webcrawler.com for constructive criticism


User-agent: *
Disallow: */ppuser/*
Disallow: */perpage/*
Disallow: */date/*
Disallow: */si/*
Disallow: */sl/*
Disallow: */cpage/*
Disallow: /member.php