From 9e964741c8eccc4fbdc635d4d2fddbabef8437cf Mon Sep 17 00:00:00 2001 From: Richard Waltman Date: Wed, 18 Sep 2024 09:48:17 -0500 Subject: [PATCH] Add ability for site to be scraped by bots --- .prettierrc | 6 ++++++ website/docusaurus.config.js | 3 ++- website/static/robots.txt | 2 ++ 3 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 .prettierrc create mode 100644 website/static/robots.txt diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 000000000..aa5496332 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,6 @@ +{ + "trailingComma": "es5", + "tabWidth": 2, + "semi": false, + "singleQuote": false +} diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 27f30ba0f..febc670c8 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -10,6 +10,7 @@ module.exports = { customFields: { gaGtag: true, }, + noIndex: false, onBrokenLinks: "log", onBrokenMarkdownLinks: "log", trailingSlash: true, @@ -101,4 +102,4 @@ module.exports = { algoliaOptions: {}, }, }, -}; +} diff --git a/website/static/robots.txt b/website/static/robots.txt new file mode 100644 index 000000000..6f27bb66a --- /dev/null +++ b/website/static/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: \ No newline at end of file