Merge branch 'main' of github.com:Emilia-Capital/eco-friendly-robots-txt

README.md

@@ -25,34 +25,38 @@ If you're developing on this plugin, you will probably want to run tests and lin
 The default output of this plugin is as follows:
 
 ```txt
-# This site is very specific about who it allows crawling from. Our default is to not allow crawling:
+# This site is very specific about who it allows crawling from.
+# Our default is to not allow crawling:
 User-agent: *
 Disallow: /
 
 # Below are the crawlers that are allowed to crawl this site.
-# Below that list, you'll find paths that are blocked, even for them, and then paths within those blocked paths that are allowed.
+# Below that list, you'll find paths that are blocked, even for them,
+# and then paths within those blocked paths that are allowed.
+User-agent: Applebot
+User-agent: ia_archiver
+User-agent: Baiduspider
+User-agent: Bingbot
+User-agent: DuckDuckBot
 User-agent: Googlebot
 User-agent: AdsBot-Google
 User-agent: MediaPartners-Google
-User-agent: Applebot
 User-agent: Yandex
-User-agent: Baiduspider
-User-agent: Bingbot
 User-agent: Slurp
-User-agent: DuckDuckBot
-User-agent: ia_archiver
 User-agent: FacebookExternalHit
-User-agent: Twitterbot
 User-agent: LinkedInBot
-Disallow: /wp-json/
-Disallow: /?rest_route=
-Disallow: /wp-admin/
-Disallow: /wp-content/cache/
-Disallow: /wp-content/plugins/
-Disallow: /xmlrpc.php
-Disallow: /wp-includes/
-Allow: /wp-includes/css/
-Allow: /wp-includes/js/
+User-agent: WhatsApp
+User-agent: Twitterbot
+Allow: /
+Disallow: /wp-includes/css/
+Disallow: /wp-includes/js/
+Allow: /wp-json/
+Allow: /?rest_route=
+Allow: /wp-admin/
+Allow: /wp-content/cache/
+Allow: /wp-content/plugins/
+Allow: /xmlrpc.php
+Allow: /wp-includes/
 
 # XML Sitemap:
 Sitemap: https://example.com/sitemap_index.xml