# robots.txt file to exclude files from access by web robots User-agent: * Allow: /site/ Disallow: /site/image/ Disallow: /site/script/ User-agent: sitecheck.internetseer.com Disallow: /