User-Agent: * Allow: /sitemap_index.xml User-agent: * # disallow all files in these directories Disallow: /cgi-bin/ Disallow: /openx/ Disallow: /bro/ User-agent: Googlebot # allow all files ending with these extensions Allow: .js Allow: .css # disallow all files ending with these extensions Disallow: /*.gz$ Disallow: /*.cgi$ # allow google image bot to search all images User-agent: Googlebot-Image Allow: /* Allow: /wp-content/uploads/