# robots.txt for http://www.researchandmarkets.com/ # Sitemap: http://www.researchandmarkets.com/reports/sitemap_index.xml.gz # for googlebot, use google webmaster tools User-agent: msnbot Crawl-delay: 30 #Disallow: /*.PDF$ User-agent: * Disallow: /admin Disallow: /rmadm Disallow: /publishers Disallow: /agreement Disallow: /my_basket.asp Disallow: /basket_add.asp Disallow: /view_question.asp Disallow: /affiliate Disallow: /samples Disallow: /feats/download_sample.asp #samples folder ets. is designed to be search engine friendly # so at some point, search engines should be allowed to access it Shane 2009-09-14 #not sure why this is even here, Shane 2009-09-14 #User-agent: googlebot-pm #Allow: /scontent User-agent: Slurp Crawl-delay: 30