diff --git a/README.md b/README.md index de1e0c5..9eec797 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,16 @@ The interactive CLI can guide you through the following steps: npx esperf ``` +### Flags + +#### parallelism + +Amount of threads to use. + +``` +npx esperf --parallelism 4 +``` + ## License MIT diff --git a/src/main.ts b/src/main.ts index 056def5..40e3e93 100644 --- a/src/main.ts +++ b/src/main.ts @@ -8,6 +8,23 @@ import {scanFiles} from './stages/scan-files.js'; import {fixFiles} from './stages/fix-files.js'; import {traverseFiles} from './stages/traverse-files.js'; import {scanDependencies} from './stages/scan-dependencies.js'; +import {availableParallelism} from 'node:os'; + +const maxThreads = availableParallelism(); + +const params: { + parallelism: number; +} = { + parallelism: Math.round(maxThreads * 0.5) +}; + +for (let i = 0; i < process.argv.length; ++i) { + if (process.argv[i] === '--parallelism ') { + params.parallelism = Math.round( + Math.min(Math.max(Number(process.argv[i]) + 1, 1), maxThreads) + ); + } +} const availableManifests: Record = { native: modReplacements.nativeReplacements, @@ -196,43 +213,48 @@ async function runModuleReplacements(): Promise { scanSpinner.message('Scanning files'); - const files = await traverseFiles(options.filesDir); + try { + const files = await traverseFiles(options.filesDir); - const scanFilesResult = await scanFiles( - files, - manifestReplacements, - scanSpinner - ); + const scanFilesResult = await scanFiles( + files, + manifestReplacements, + params.parallelism, + scanSpinner + ); - if (scanFilesResult.length > 0) { - dependenciesFound = true; - } + if (scanFilesResult.length > 0) { + dependenciesFound = true; + } - if (dependenciesFound) { - scanSpinner.stop('Replaceable modules found.', 2); - } else { - scanSpinner.stop('No replaceable modules found.'); - } + if (dependenciesFound) { + scanSpinner.stop('Replaceable modules found.', 2); + } else { + scanSpinner.stop('No replaceable modules found.'); + } - if ( - options.autoUninstall && - (dependenciesToRemove.length > 0 || devDependenciesToRemove.length > 0) - ) { - const npmSpinner = cl.spinner(); + if ( + options.autoUninstall && + (dependenciesToRemove.length > 0 || devDependenciesToRemove.length > 0) + ) { + const npmSpinner = cl.spinner(); - npmSpinner.start('Removing npm dependencies'); + npmSpinner.start('Removing npm dependencies'); - if (dependenciesToRemove.length > 0) { - await x('npm', ['rm', '-S', ...dependenciesToRemove]); - } - if (devDependenciesToRemove.length > 0) { - await x('npm', ['rm', '-D', ...devDependenciesToRemove]); - } + if (dependenciesToRemove.length > 0) { + await x('npm', ['rm', '-S', ...dependenciesToRemove]); + } + if (devDependenciesToRemove.length > 0) { + await x('npm', ['rm', '-D', ...devDependenciesToRemove]); + } - npmSpinner.stop('npm dependencies removed'); - } + npmSpinner.stop('npm dependencies removed'); + } - if (options.fix) { - await fixFiles(scanFilesResult); + if (options.fix) { + await fixFiles(scanFilesResult); + } + } catch (error) { + scanSpinner.stop(error as string, 1); } } diff --git a/src/stages/scan-files.ts b/src/stages/scan-files.ts index bd619b8..38eeb25 100644 --- a/src/stages/scan-files.ts +++ b/src/stages/scan-files.ts @@ -1,112 +1,55 @@ import * as modReplacements from 'module-replacements'; -import {ts as sg} from '@ast-grep/napi'; -import {readFile} from 'node:fs/promises'; -import dedent from 'dedent'; -import pc from 'picocolors'; import * as cl from '@clack/prompts'; import {type FileReplacement} from '../shared-types.js'; -import {suggestReplacement} from '../suggest-replacement.js'; +import {availableParallelism} from 'node:os'; +import {Worker} from 'node:worker_threads'; +import path from 'path'; +import {fileURLToPath} from 'url'; -async function scanFile( - filePath: string, - contents: string, - lines: string[], - replacements: modReplacements.ModuleReplacement[] -): Promise { - const ast = sg.parse(contents); - const root = ast.root(); - const matches: modReplacements.ModuleReplacement[] = []; +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const available = availableParallelism(); - for (const replacement of replacements) { - const imports = root.findAll({ - rule: { - any: [ - { - pattern: { - context: `import $NAME from '${replacement.moduleName}'`, - strictness: 'relaxed' - } - }, - { - pattern: { - context: `require('${replacement.moduleName}')`, - strictness: 'relaxed' - } - } - ] - } - }); - - if (imports.length > 0) { - matches.push(replacement); - } - - for (const node of imports) { - const range = node.range(); - let snippet: string = ''; - - const prevLine = lines[range.start.line - 1]; - const line = lines[range.start.line]; - const nextLine = lines[range.start.line + 1]; - - if (prevLine) { - snippet += `${range.start.line} | ${prevLine}\n`; - } - - snippet += `${range.start.line + 1} | ${pc.red(line)}\n`; - - if (nextLine) { - snippet += `${range.start.line + 2} | ${nextLine}\n`; - } - - suggestReplacement(replacement, { - type: 'file', - path: filePath, - line: range.start.line, - column: range.start.column, - snippet - }); - } - } - - if (matches.length === 0) { - return null; - } - - return { - path: filePath, - contents, - replacements: matches - }; -} - -export async function scanFiles( +export function scanFiles( files: string[], replacements: modReplacements.ModuleReplacement[], + threads: number, spinner: ReturnType ): Promise { - const results: FileReplacement[] = []; - - for (const file of files) { - try { - const contents = await readFile(file, 'utf8'); - const lines = contents.split('\n'); - - spinner.message(`Scanning ${file}`); - - const scanResult = await scanFile(file, contents, lines, replacements); - - if (scanResult) { - results.push(scanResult); - } - } catch (err) { - cl.log.error(dedent` - Could not read file ${file}: - - ${String(err)} - `); + return new Promise((resolve, reject) => { + let i = 0; + let tasks = 0; + const filesLength = files.length; + const results: FileReplacement[] = []; + + for (const file of files.splice(0, threads)) { + const worker = new Worker(`${__dirname}/workers/scan-file.js`); + // todo, what todo with the errors? + worker.on('error', (error) => reject(error.message)); + worker.on('message', (message) => { + if (message?.type === 'result') { + if (message.value) results.push(message.value); + i += 1; + if (i === filesLength) { + resolve(results); + } + } else { + reject(message.value); + } + if (files.length > 0) { + if (available >= tasks) { + const file = files.shift(); + spinner.message(`Scanning file: ${file}`); + worker.postMessage({file, replacements}); + } else { + tasks -= 1; + } + } else { + worker.terminate(); + } + }); + spinner.message(`Scanning file: ${file}`); + worker.postMessage({file, replacements}); } - } - - return results; + }); } diff --git a/src/stages/workers/scan-file.ts b/src/stages/workers/scan-file.ts new file mode 100644 index 0000000..b9670dd --- /dev/null +++ b/src/stages/workers/scan-file.ts @@ -0,0 +1,115 @@ +import * as modReplacements from 'module-replacements'; +import {ts as sg} from '@ast-grep/napi'; +import dedent from 'dedent'; +import pc from 'picocolors'; +import {type FileReplacement} from './../../shared-types.js'; +import {suggestReplacement} from './../../suggest-replacement.js'; +import {parentPort} from 'node:worker_threads'; +import {open} from 'node:fs/promises'; + +async function scanFile( + filePath: string, + contents: string, + lines: string[], + replacements: modReplacements.ModuleReplacement[] +): Promise { + const ast = sg.parse(contents); + const root = ast.root(); + const matches: modReplacements.ModuleReplacement[] = []; + + for (const replacement of replacements) { + const imports = root.findAll({ + rule: { + any: [ + { + pattern: { + context: `import $NAME from '${replacement.moduleName}'`, + strictness: 'relaxed' + } + }, + { + pattern: { + context: `require('${replacement.moduleName}')`, + strictness: 'relaxed' + } + } + ] + } + }); + + if (imports.length > 0) { + matches.push(replacement); + } + + for (const node of imports) { + const range = node.range(); + let snippet: string = ''; + + const prevLine = lines[range.start.line - 1]; + const line = lines[range.start.line]; + const nextLine = lines[range.start.line + 1]; + + if (prevLine) { + snippet += `${range.start.line} | ${prevLine}\n`; + } + + snippet += `${range.start.line + 1} | ${pc.red(line)}\n`; + + if (nextLine) { + snippet += `${range.start.line + 2} | ${nextLine}\n`; + } + + suggestReplacement(replacement, { + type: 'file', + path: filePath, + line: range.start.line, + column: range.start.column, + snippet + }); + } + } + + if (matches.length === 0) { + return null; + } + + return { + path: filePath, + contents, + replacements: matches + }; +} + +async function scanTask( + file: string, + replacements: modReplacements.ModuleReplacement[] +) { + try { + const fd = await open(file); + const contents = await fd.readFile({encoding: 'utf-8'}); + const lines = contents.split('\n'); + const scanResult = await scanFile(file, contents, lines, replacements); + await fd.close(); + + parentPort?.postMessage({type: 'result', value: scanResult}); + } catch (err) { + parentPort?.postMessage({ + type: 'error', + value: dedent` + Could not read file ${file}: + + ${String(err)} + ` + }); + } +} + +parentPort?.on( + 'message', + (message: { + file: string; + replacements: modReplacements.ModuleReplacement[]; + }) => { + scanTask(message.file, message.replacements); + } +);