diff --git a/dictionaries/bash/cspell-ext.json b/dictionaries/bash/cspell-ext.json index a96eb4a8b62..b20b6675101 100644 --- a/dictionaries/bash/cspell-ext.json +++ b/dictionaries/bash/cspell-ext.json @@ -2,15 +2,7 @@ "id": "bash", "name": "bash", "description": "Bash dictionary.", - "readonly": true, - "dictionaryDefinitions": [ - { - "name": "bash", - "path": "./dict/bash-words.txt", - "description": "Bash dictionary." - } - ], - "dictionaries": [], + "import": ["@cspell/dict-shell"], "languageSettings": [ { "languageId": "shellscript", diff --git a/dictionaries/bash/cspell-tools.config.yaml b/dictionaries/bash/cspell-tools.config.yaml deleted file mode 100644 index d2c79cc70fd..00000000000 --- a/dictionaries/bash/cspell-tools.config.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-tools/cspell-tools.config.schema.json - -targets: - - name: bash-words - sources: - - ./src/bash-words.txt - - listFile: ../shell/common-source-files.txt - - ../shell/src/shell-bash-words.txt - format: plaintext - targetDirectory: './dict' - generateNonStrict: false diff --git a/dictionaries/bash/dict/README.md b/dictionaries/bash/dict/README.md deleted file mode 100644 index ce67123010f..00000000000 --- a/dictionaries/bash/dict/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Dict Directory - -NOTE: This directory contains generated content. Please edit [src](../src/README.md) files. diff --git a/dictionaries/bash/dict/bash-words.txt b/dictionaries/bash/dict/bash-words.txt deleted file mode 100644 index dce7aa7a329..00000000000 --- a/dictionaries/bash/dict/bash-words.txt +++ /dev/null @@ -1,217 +0,0 @@ - -# cspell-tools: keep-case no-split - ---noprofile ---norc ---rcfile -BASHOPTS -BASH_ARGV0 -BASH_COMPAT -BASH_SOURCE -BASH_VERSINFO -BASH_XTRACEFD -CHILD_MAX -COLUMNS -COMPREPLY -COMP_WORDBREAKS -EPOCHREALTIME -EUID -EXECIGNORE -FUNCNAME -FUNCNEST -GLOBIGNORE -HISTCONTROL -HISTFILE -HISTFILESIZE -HISTIGNORE -HISTSIZE -HISTTIMEFORMAT -HOME -HOSTFILE -HOSTNAME -HOSTTYPE -IFS -LANG -LC_ALL -LC_COLLATE -LC_CTYPE -LC_MESSAGES -LC_NUMERIC -LC_TIME -LINES -MACHTYPE -MAIL -MAILCHECK -MAILPATH -MANPATH -OLDPWD -OPTARG -OPTERR -OPTIND -OSTYPE -PATH -PIPESTATUS -POSIXLY_CORRECT -POSIX_PEDANTIC -PPID -PROMPT_COMMAND -PS1 -PS2 -PS3 -PS4 -READLINE_LINE -READLINE_MARK -READLINE_POINT -SCRIPTDIR -SECONDS -SHELL -SHELLOPTS -SRANDOM -SSH2_CLIENT -SSH_CLIENT -TERM -TERMCAP -TERMINFO -TEXTDOMAIN -TEXTDOMAINDIR -TIMEFORMAT -TMOUT -TZ -UID -alias -allexport -alnum -autoremove -bashscript -bg -bind -braceexpand -break -builtin -caller -case -cd -chsh -cntrl -command -compgen -complete -compopt -continue -coproc -dbus -dearmor -declare -dirs -disown -do -done -dpkg -echo -elif -else -emacs -enable -enarmor -errexit -errtrace -esac -euxo -eval -exec -exit -export -fallocate -false -fc -fg -fi -for -fstype -function -functrace -getopts -globstar -hash -hashall -help -histexpand -history -if -ignoreeof -in -inlib -jobs -keymap -keymaps -keyrings -keyword -kill -let -local -logout -mapfile -mkdir -mktemp -monitor -mvim -neovim -newgidmap -newuidmap -noclobber -noexec -noglob -nolog -notify -nounset -nvim -onecmd -pgrep -physical -pipefail -popd -posix -printf -privileged -proxyd -punct -pushd -pwd -read -readarray -readline -readlink -readonly -realpath -return -select -set -shellcheck -shellscript -shfmt -shift -shopt -source -suspend -test -then -time -times -tput -trap -true -type -typeset -uidmap -ulimit -umask -unalias -unset -until -verbose -vi -vim -wait -while -xdigit -xtrace diff --git a/dictionaries/bash/package.json b/dictionaries/bash/package.json index 08609a588ba..fdc06c306ff 100644 --- a/dictionaries/bash/package.json +++ b/dictionaries/bash/package.json @@ -12,8 +12,8 @@ "./cspell-ext.json": "./cspell-ext.json" }, "scripts": { - "build": "cspell-tools-cli build", - "test": "shx cat src/bash-words.txt | cspell -c ./cspell-ext.json \"--locale=*\" \"--languageId=shellscript\" stdin", + "build": "echo ok", + "test": "shx cat samples/bash-words.txt | cspell -c ./cspell-ext.json \"--locale=*\" \"--languageId=shellscript\" stdin", "prepublishOnly": "echo pre-publish", "prepare:dictionary": "pnpm run build" }, @@ -38,5 +38,8 @@ "files": [ "dict/*", "cspell-ext.json" - ] + ], + "dependencies": { + "@cspell/dict-shell": "workspace:*" + } } diff --git a/dictionaries/bash/src/bash-words.txt b/dictionaries/bash/samples/bash-words.txt similarity index 100% rename from dictionaries/bash/src/bash-words.txt rename to dictionaries/bash/samples/bash-words.txt diff --git a/dictionaries/bash/src/README.md b/dictionaries/bash/src/README.md deleted file mode 100644 index 5ac40a8e845..00000000000 --- a/dictionaries/bash/src/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Source Directory - -All source files used to generate the dictionary should be stored in this directory. diff --git a/dictionaries/shell/cspell-tools.config.yaml b/dictionaries/shell/cspell-tools.config.yaml index 83dd584c891..8a79470d2d5 100644 --- a/dictionaries/shell/cspell-tools.config.yaml +++ b/dictionaries/shell/cspell-tools.config.yaml @@ -5,7 +5,6 @@ targets: sources: - listFile: ./common-source-files.txt - ./src/shell-bash-words.txt - - ../bash/src/bash-words.txt format: plaintext targetDirectory: './dict' generateNonStrict: false diff --git a/dictionaries/shell/dict/shell-all-words.txt b/dictionaries/shell/dict/shell-all-words.txt index 0f151c60673..bafa7bf4c84 100644 --- a/dictionaries/shell/dict/shell-all-words.txt +++ b/dictionaries/shell/dict/shell-all-words.txt @@ -2,11 +2,15 @@ # cspell-tools: keep-case no-split --no-globalrcs +--noprofile +--norc +--rcfile ARCHFLAGS BASHOPTS BASH_ARGV0 BASH_COMPAT BASH_SOURCE +BASH_VERSINFO BASH_XTRACEFD CHILD_MAX COLUMNS @@ -46,6 +50,7 @@ MAILPATH MANPAGER MANPATH OLDPWD +OPTARG OPTERR OPTIND OSTYPE @@ -66,6 +71,7 @@ SCRIPTDIR SECONDS SHELL SHELLOPTS +SRANDOM SSH2_CLIENT SSH_CLIENT TERM @@ -74,6 +80,7 @@ TERMINFO TEXTDOMAIN TEXTDOMAINDIR TIMEFMT +TIMEFORMAT TMOUT TTYIDLE TZ @@ -127,15 +134,18 @@ epochtime errexit errtrace esac +euxo eval exec exit export +fallocate false fc fg fi for +fstype funcfiletrace function functrace @@ -151,6 +161,8 @@ ignoreeof in inlib jobs +keymap +keymaps keyrings keyword kill @@ -176,6 +188,7 @@ nounset nvim ohmyzsh onecmd +pgrep physical pipefail popd @@ -191,6 +204,7 @@ pushd pwd read readarray +readline readlink readonly realpath diff --git a/dictionaries/shell/src/shell-bash-words.txt b/dictionaries/shell/src/shell-bash-words.txt index 65184d7009a..462bb4cc620 100644 --- a/dictionaries/shell/src/shell-bash-words.txt +++ b/dictionaries/shell/src/shell-bash-words.txt @@ -1,4 +1,8 @@ # Bash related terms +--noprofile +--norc +--rcfile +BASH_VERSINFO bashscript braceexpand builtin @@ -6,28 +10,40 @@ caller compgen complete compopt +coproc declare dirs disown enable +EPOCHREALTIME errtrace +euxo +fallocate for +fstype function functrace globstar hashall histexpand +keymap +keymaps let local mapfile onecmd +OPTARG +pgrep pipefail popd pushd readarray +readline readonly select shopt source +SRANDOM suspend +TIMEFORMAT typeset diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e746573df51..4279010b2ad 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -95,7 +95,11 @@ importers: dictionaries/aws: {} - dictionaries/bash: {} + dictionaries/bash: + dependencies: + '@cspell/dict-shell': + specifier: workspace:* + version: link:../shell dictionaries/bg_BG: {} @@ -5377,7 +5381,7 @@ snapshots: '@types/vinyl@2.0.12': dependencies: '@types/expect': 1.20.4 - '@types/node': 22.10.5 + '@types/node': 20.17.12 '@vercel/nft@0.27.10(encoding@0.1.13)': dependencies: diff --git a/scripts/merge.mjs b/scripts/merge.mjs new file mode 100755 index 00000000000..64ac12d12b4 --- /dev/null +++ b/scripts/merge.mjs @@ -0,0 +1,68 @@ +#!/usr/bin/env node + +/** + * Merge lists of words into a single list. + */ + +import { program } from 'commander'; +import { promises as fs } from 'fs'; +import { sortSourceContent } from './lib/sortContent.mjs'; + +let info = console.log; + +async function readAndMergeFiles(files) { + let content = ''; + + for (const file of files) { + content += `## ${file}\n`; + content += await fs.readFile(file, 'utf8'); + } + + return content; +} + +function removeComments(content) { + return content.replace(/#.*$/gm, ''); +} + +async function processFiles(files, excluded) { + const mergedContent = await readAndMergeFiles(files); + const excludedWords = removeComments(await readAndMergeFiles(excluded)) + .split('\n') + .map((line) => line.trim()) + .filter((line) => !!line); + const exclude = new Set(excludedWords); + + if (!mergedContent.trim()) return; + + const lines = mergedContent + .split('\n') + .map((line) => line.trim()) + .filter((line) => !exclude.has(removeComments(line).trim())); + const content = lines.join('\n'); + + const sorted = sortSourceContent(content).replace(/^(.*\n)\1+/gm, '$1'); + + const filename = files[0]; + + if (sorted === content) { + info(`${filename} - ok`); + return; + } + + await fs.writeFile(filename, sorted, 'utf8'); + info(`${filename} - updated`); +} + +program + .name('merge') + .description('Merge source files into a single list.') + .argument('', 'files to read') + .option('-x, --exclude ', 'Files containing words to exclude.', []) + .action(async (files, options) => { + // console.log('%o', options); + const ignore = options.exclude || []; + await processFiles(files, ignore); + }); + +program.parseAsync(); diff --git a/scripts/package.json b/scripts/package.json index 248f6ec44fb..dc973ab315c 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -9,6 +9,7 @@ "gen-dictionary-info": "gen-dictionary-info.mjs", "update-package-json": "update-package-json.mjs", "update-contributors": "update-contributors.mjs", + "merge": "merge.mjs", "pipe-cmds": "pipe-cmds.mjs" }, "scripts": {