add entries in robots.txt for new bots (#619)

GoteoFoundation · Sep 9, 2024 · e659089 · e659089
1 parent 5cca4ce
commit e659089
Showing 1 changed file with 16 additions and 0 deletions.
diff --git a/public/robots.txt b/public/robots.txt
@@ -1,4 +1,20 @@
+User-agent: GPTBot
+Disallow: /
+
+User-agent: Amazonbot
+Disallow: /
+
+User-agent: SemrushBot
+Disallow: /
+
+User-agent: AhrefsBot
+Disallow: /
+
+User-Agent: ImagesiftBot
+Crawl-delay: 60
+
 User-agent: *
 Disallow: /admin
 Disallow: /admin/
 Disallow: /dashboard/
+