# see http://www.robotstxt.org/ # directed to all robots User-agent: * Disallow: /generated/ # allow htdig User-agent: htdig Disallow: