# spiders instructions for topurls.be
User-agent: The Python Robot
Disallow: /

User-agent: *
Disallow: /cache/
Disallow: /gsmcache/
Disallow: /go/
Disallow: /gsm/