# Blocks robots from specific folders / directories for the forum User-agent: * Disallow: /cgi-bin/ Disallow: /files/ # Block MegaIndex.ru User-agent: MegaIndex.ru/2.0 Disallow: / User-agent: MegaIndex.ru Disallow: / User-agent: megaIndex.ru Disallow: / # Block YandexBot User-agent: Yandex Disallow: / # Block Baidu User-agent: Baiduspider User-agent: Baiduspider-video User-agent: Baiduspider-image Disallow: / # Block Youdao User-agent: YoudaoBot Disallow: / # Block Majestic User-agent: MJ12bot Disallow: / # Block PetalBot User-agent: PetalBot Disallow: / User-agent: AspiegelBot Disallow: / # Block AhrefsBot User-agent: AhrefsBot Disallow: / # Block MauiBot User-agent: MauiBot Disallow: / # Block SEO User-agent: SemrushBot Disallow: / # http://www.brandwatch.com/magpie-crawler/ User-agent: magpie-crawler Disallow: / #Bing User-agent: Bingbot Disallow: /files/jump/ Disallow: /files/jump2/ Disallow: /test/ Disallow: /index.php?action=.xml;type=rss Disallow: /index.php?type=rss;action=.xml Disallow: /index.php?wap2 Disallow: /index.php?action=recent Crawl-delay: 10