feat(linter): add new link checker - lychee (oxsecurity#2673)

* feat(linter): add lychee

* test(lychee): add test

* fix(test): update wrong path

* chore(cspell): add linter's words to excludes

* test(lychee): update test dir config

* chore(changelog): add lychee

* chore(ignore): add .lycheeignore to the root

* chore(cspell): add lycheeignore

* chore(lychee): add lychee.toml config file

* chore(lychee): update base config

* chore(lychee): update base config

* chore(lychee): update base config

* chore(cspell): add lycheecache

* chore(links): update outdated

* chore(links): update outdated

* chore(schema): add lychee

* chore(links): update outdated + config

* chore(links): update oudated + config

* chore(links): update outdated

.automation/build.py

@@ -106,7 +106,7 @@
 
 IDE_LIST = {
     "atom": {"label": "Atom", "url": "https://atom.io/"},
-    "brackets": {"label": "Brackets", "url": "http://brackets.io/"},
+    "brackets": {"label": "Brackets", "url": "https://brackets.io/"},
     "eclipse": {"label": "Eclipse", "url": "https://www.eclipse.org/"},
     "emacs": {"label": "Emacs", "url": "https://www.gnu.org/software/emacs/"},
     "idea": {

.automation/generated/linter-links-previews.json

@@ -249,6 +249,11 @@
     "image": "https://avatars3.githubusercontent.com/u/17838737?s=400&v=4",
     "title": "luarocks/luacheck"
   },
+  "lychee": {
+    "description": "Fast, async, stream-based link checker written in Rust. Finds broken URLs and mail addresses inside Markdown, HTML, reStructuredText, websites and more!",
+    "image": "https://avatars.githubusercontent.com/u/74027027?s=200&v=4",
+    "title": "lycheeverse/lychee"
+  },
   "markdown-link-check": {
     "description": "checks that all of the hyperlinks in a markdown text to determine if they are alive or dead - tcort/markdown-link-check",
     "image": "https://avatars2.githubusercontent.com/u/216720?s=400&v=4",
@@ -559,4 +564,4 @@
     "image": "https://avatars3.githubusercontent.com/u/5244945?s=400&v=4",
     "title": "adrienverge/yamllint"
   }
-}
+}

.automation/generated/linter-versions.json

@@ -48,6 +48,7 @@
     "kubeval": "0.16.1",
     "lintr": "0.0.0",
     "luacheck": "1.1.1",
+    "lychee": "0.13.0",
     "markdown-link-check": "3.11.2",
     "markdown-table-formatter": "1.4.0",
     "markdownlint": "0.34.0",

.automation/test/spell_lychee/markdown_bad_1.md

@@ -0,0 +1,28 @@
+## Bad Markdown
+
+This is just a standard bad markdown.
+
+###### Second level header
+
+This header does **NOT** follow the **step** down from `level 1`.
+
+- Here it *is*
+  - Some more indention
+      - why so much?
+
+```
+ls -la
+```
+
+| this   | is a wrong | table              |
+|--------|------------           |--------------------|
+| hahaha |                 naaaaaaah  | wrong formatting !            |
+| hahaha | naaaaaaah  | wrong formatting ! |
+
+# Walk away
+
+We're all done **here**.
+- [Link Action]https://github.com
+- [Link Action 2](#wesh)
+- [Link Action 3](http://www.glouglouglglsdgdfgfdgsfgdfgdf.com)
+- [Link Action 4](http://www.glouglouglglsdgdfgfdgsfgdfgdf.com2)

.automation/test/spell_lychee/markdown_good_1.md

@@ -0,0 +1,25 @@
+# Good Markdown
+
+This is just a standard good markdown.
+
+## Second level header
+
+This header follows the step down from `level 1`.
+
+- Here it *is*
+  - Some more **indention**
+    - why so much?
+
+```bash
+ls -la
+```
+
+| this | is a good | table    |
+|------|-----------|----------|
+| haha | yesssssss | good !   |
+| haha | yesssssss | gooood ! |
+
+### Walk away
+
+We're all done **here**.
+- [Link Action](https://github.com)

.automation/test/spell_lychee/reports/ERROR-SPELL_LYCHEE.txt

@@ -0,0 +1,19 @@
+Results of lychee linter (version 0.0.0)
+See documentation on https://lychee.cli.rs
+-----------------------------------------------
+
+❌ [ERROR] for workspace /github/workspace
+Linter raw log:
+📝 Summary
+---------------------
+🔍 Total............3
+✅ Successful.......1
+⏳ Timeouts.........0
+🔀 Redirected.......0
+👻 Excluded.........0
+❓ Unknown..........0
+🚫 Errors...........2
+
+Errors in /github/workspace/.automation/test/spell_lychee/markdown_bad_1.md
+✗ [ERR] http://www.glouglouglglsdgdfgfdgsfgdfgdf.com2/ | Failed: Network error: dns error: no record found for Query { name: Name("www.glouglouglglsdgdfgfdgsfgdfgdf.com2."), query_type: AAAA, query_class: IN }
+✗ [ERR] http://www.glouglouglglsdgdfgfdgsfgdfgdf.com/ | Failed: Network error: dns error: no record found for Query { name: Name("www.glouglouglglsdgdfgfdgsfgdfgdf.com."), query_type: AAAA, query_class: IN }

.automation/test/spell_lychee/reports/SUCCESS-SPELL_LYCHEE.txt

@@ -0,0 +1,15 @@
+Results of lychee linter (version 0.0.0)
+See documentation on https://lychee.cli.rs
+-----------------------------------------------
+
+✅ [SUCCESS] for workspace /github/workspace
+Linter raw log:
+📝 Summary
+---------------------
+🔍 Total............1
+✅ Successful.......1
+⏳ Timeouts.........0
+🔀 Redirected.......0
+👻 Excluded.........0
+❓ Unknown..........0
+🚫 Errors...........0

.cspell.json

@@ -57,6 +57,7 @@
         "eadme",
         "enableassertions",
         "enablesystemassertions",
+        "epub",
         "esprima",
         "etailed",
         "eventemitter3",
@@ -82,6 +83,9 @@
         "leavevmode",
         "lodddddddxk0XXXX0c",
         "looool",
+        "lycheecache",
+        "lycheeignore",
+        "lycheeverse",
         "metavuln",
         "minimatch",
         "minimist",
@@ -1434,4 +1438,4 @@
         "zaach",
         "zricethezav"
     ]
-}
+}

.github/linters/lychee.toml

@@ -0,0 +1,62 @@
+# Docs: https://lychee.cli.rs/#/usage/config
+# Example config: https://github.com/lycheeverse/lychee/blob/master/lychee.example.toml
+
+#############################  Cache  ###############################
+
+# Enable link caching. This can be helpful to avoid checking the same links on multiple runs.
+cache = true
+
+# Discard all cached requests older than this duration.
+max_cache_age = "1d"
+
+#############################  Runtime  #############################
+
+# Maximum number of allowed redirects.
+max_redirects = 6
+
+# Maximum number of allowed retries before a link is declared dead.
+max_retries = 2
+
+# Maximum number of concurrent link checks.
+# max_concurrency = 2
+
+#############################  Requests  ############################
+
+# User agent to send with each request.
+user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.43"
+
+# Minimum wait time in seconds between retries of failed requests.
+retry_wait_time = 2
+
+# Comma-separated list of accepted status codes for valid links.
+accept = [200, 206, 429]
+
+# Only test links with the given schemes (e.g. https).
+# Omit to check links with any scheme.
+scheme = [ "https", "http" ]
+
+# Custom request headers
+headers = ['Accept-Encoding: deflate, compress, gzip, br, zstd']
+
+#############################  Exclusions  ##########################
+
+# Ignore case of paths when matching glob patterns.
+glob_ignore_case = true
+
+# Exclude all private IPs from checking.
+exclude_all_private = true
+
+# Exclude private IP address ranges from checking.
+exclude_private = true
+
+# Exclude link-local IP address range from checking.
+exclude_link_local = true
+
+# Exclude loopback IP address range and localhost from checking.
+exclude_loopback = true
+
+# Exclude all mail addresses from checking.
+exclude_mail = true
+
+# Exclude these filesystem paths from getting checked.
+exclude_path = ["node_modules", "site", "docs", ".venv", "CHANGELOG.md", "megalinter/descriptors/schemas", ".automation/generated", ".automation/test"]

.github/workflows/deploy-BETA-linters.yml

@@ -168,6 +168,7 @@ jobs:
             "spell_cspell",
             "spell_proselint",
             "spell_vale",
+            "spell_lychee",
             "sql_sql_lint",
             "sql_sqlfluff",
             "sql_tsqllint",