Add more user agents to robots.txt

ImageMarkup · Oct 22, 2024 · 5b5e437 · 5b5e437
1 parent dcfe809
commit 5b5e437
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/isic/core/templates/core/robots.txt b/isic/core/templates/core/robots.txt
@@ -1,2 +1,4 @@
 User-agent: Amazonbot
+User-agent: AhrefsBot
+User-agent: MJ12bot
 Disallow: /