Block various marketing crawlers

egrapa · Oct 21, 2017 · 7125c86 · 7125c86
1 parent 52da58a
commit 7125c86
Showing 1 changed file with 7 additions and 0 deletions.
diff --git a/phpBB/robots.txt b/phpBB/robots.txt
@@ -22,3 +22,10 @@ Crawl-delay: 10
 # This has a massively high crawl-rate, and it keeps breaking the site.
 User-agent: Yandex
 Disallow: /
+
+# These are marketing crawlers and other useless junk.
+User-agent: AhrefsBot
+User-agent: DotBot
+User-agent: SemrushBot
+User-agent: MJ12bot
+Disallow: /