Merge pull request #997 from ImageMarkup/update-robots-txt

ImageMarkup · Oct 22, 2024 · 0ada9b3 · 0ada9b3
2 parents 657e44c + 5b5e437
commit 0ada9b3
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/isic/core/templates/core/robots.txt b/isic/core/templates/core/robots.txt
@@ -1,2 +1,4 @@
 User-agent: Amazonbot
+User-agent: AhrefsBot
+User-agent: MJ12bot
 Disallow: /