############################### # robots.txt for http://www.wolkyshop.com/ # # all directories which should not be indexed # User-agent: * Disallow: /fjc/ Disallow: /fjc_css/ Disallow: /fjc_includes/ Disallow: /fjc_templates/ Disallow: /site_css/ Disallow: /site_images/ Disallow: /site_includes/ Sitemap: http://www.wolkyshop.com/sitemap.xml # # All search robots (*) # else you should give a name User-agent: Slurp Crawl-delay: 20 User-Agent: msnbot Crawl-Delay: 20 User-agent: Yahoo! Slurp Crawl-delay: 20 User-agent: Jyxobot Disallow: / User-Agent: MJ12bot Disallow: / User-agent: CazoodleBot Disallow: / User-agent: Baiduspider Disallow: / User-agent: Fasterfox Disallow: / # ###############################