# Directions for robots. See this URL: # http://www.robotstxt.org/robotstxt.html # for a description of the file format. User-agent: HTTrack User-agent: puf User-agent: MSIECrawler Disallow: / # The Krugle web crawler (though based on Nutch) is OK. User-agent: Krugle Allow: / Disallow: /~guido/orlijn/ Disallow: /webstats/ # No one should be crawling us with Nutch. User-agent: Nutch Disallow: / # Hide old versions of the documentation and various large sets of files. User-agent: * Disallow: /~guido/orlijn/ Disallow: /webstats/