# /robots.txt # ht://Dig is allowed to go everywhere User-agent: htdig Disallow: # google.com's googlebot has free run User-agent: googlebot Disallow: # everyone else is asked to leave User-agent: * Disallow: /