es-tooling · VandeurenGlenn · Aug 10, 2024 · Aug 11, 2024 · Aug 11, 2024 · Aug 11, 2024
diff --git a/README.md b/README.md
@@ -28,6 +28,16 @@ The interactive CLI can guide you through the following steps:
 npx esperf
 ```
 
+### Flags
+
+#### parallelism
+
+Amount of threads to use.
+
+```
+npx esperf --parallelism 4
+```
+
 ## License
 
 MIT
diff --git a/src/main.ts b/src/main.ts
@@ -8,6 +8,23 @@ import {scanFiles} from './stages/scan-files.js';
 import {fixFiles} from './stages/fix-files.js';
 import {traverseFiles} from './stages/traverse-files.js';
 import {scanDependencies} from './stages/scan-dependencies.js';
+import {availableParallelism} from 'node:os';
+
+const maxThreads = availableParallelism();
+
+const params: {
+  parallelism: number;
+} = {
+  parallelism: Math.round(maxThreads * 0.5)
+};
+
+for (let i = 0; i < process.argv.length; ++i) {
+  if (process.argv[i] === '--parallelism ') {
+    params.parallelism = Math.round(
+      Math.min(Math.max(Number(process.argv[i]) + 1, 1), maxThreads)
+    );
+  }
+}
 
 const availableManifests: Record<string, modReplacements.ManifestModule> = {
   native: modReplacements.nativeReplacements,
@@ -196,43 +213,48 @@ async function runModuleReplacements(): Promise<void> {
 
   scanSpinner.message('Scanning files');
 
-  const files = await traverseFiles(options.filesDir);
+  try {
+    const files = await traverseFiles(options.filesDir);
 
-  const scanFilesResult = await scanFiles(
-    files,
-    manifestReplacements,
-    scanSpinner
-  );
+    const scanFilesResult = await scanFiles(
+      files,
+      manifestReplacements,
+      params.parallelism,
+      scanSpinner
+    );
 
-  if (scanFilesResult.length > 0) {
-    dependenciesFound = true;
-  }
+    if (scanFilesResult.length > 0) {
+      dependenciesFound = true;
+    }
 
-  if (dependenciesFound) {
-    scanSpinner.stop('Replaceable modules found.', 2);
-  } else {
-    scanSpinner.stop('No replaceable modules found.');
-  }
+    if (dependenciesFound) {
+      scanSpinner.stop('Replaceable modules found.', 2);
+    } else {
+      scanSpinner.stop('No replaceable modules found.');
+    }
 
-  if (
-    options.autoUninstall &&
-    (dependenciesToRemove.length > 0 || devDependenciesToRemove.length > 0)
-  ) {
-    const npmSpinner = cl.spinner();
+    if (
+      options.autoUninstall &&
+      (dependenciesToRemove.length > 0 || devDependenciesToRemove.length > 0)
+    ) {
+      const npmSpinner = cl.spinner();
 
-    npmSpinner.start('Removing npm dependencies');
+      npmSpinner.start('Removing npm dependencies');
 
-    if (dependenciesToRemove.length > 0) {
-      await x('npm', ['rm', '-S', ...dependenciesToRemove]);
-    }
-    if (devDependenciesToRemove.length > 0) {
-      await x('npm', ['rm', '-D', ...devDependenciesToRemove]);
-    }
+      if (dependenciesToRemove.length > 0) {
+        await x('npm', ['rm', '-S', ...dependenciesToRemove]);
+      }
+      if (devDependenciesToRemove.length > 0) {
+        await x('npm', ['rm', '-D', ...devDependenciesToRemove]);
+      }
 
-    npmSpinner.stop('npm dependencies removed');
-  }
+      npmSpinner.stop('npm dependencies removed');
+    }
 
-  if (options.fix) {
-    await fixFiles(scanFilesResult);
+    if (options.fix) {
+      await fixFiles(scanFilesResult);
+    }
+  } catch (error) {
+    scanSpinner.stop(error as string, 1);
   }
 }
diff --git a/src/stages/scan-files.ts b/src/stages/scan-files.ts
@@ -1,112 +1,55 @@
 import * as modReplacements from 'module-replacements';
-import {ts as sg} from '@ast-grep/napi';
-import {readFile} from 'node:fs/promises';
-import dedent from 'dedent';
-import pc from 'picocolors';
 import * as cl from '@clack/prompts';
 import {type FileReplacement} from '../shared-types.js';
-import {suggestReplacement} from '../suggest-replacement.js';
+import {availableParallelism} from 'node:os';
+import {Worker} from 'node:worker_threads';
+import path from 'path';
+import {fileURLToPath} from 'url';
 
-async function scanFile(
-  filePath: string,
-  contents: string,
-  lines: string[],
-  replacements: modReplacements.ModuleReplacement[]
-): Promise<FileReplacement | null> {
-  const ast = sg.parse(contents);
-  const root = ast.root();
-  const matches: modReplacements.ModuleReplacement[] = [];
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const available = availableParallelism();
 
-  for (const replacement of replacements) {
-    const imports = root.findAll({
-      rule: {
-        any: [
-          {
-            pattern: {
-              context: `import $NAME from '${replacement.moduleName}'`,
-              strictness: 'relaxed'
-            }
-          },
-          {
-            pattern: {
-              context: `require('${replacement.moduleName}')`,
-              strictness: 'relaxed'
-            }
-          }
-        ]
-      }
-    });
-
-    if (imports.length > 0) {
-      matches.push(replacement);
-    }
-
-    for (const node of imports) {
-      const range = node.range();
-      let snippet: string = '';
-
-      const prevLine = lines[range.start.line - 1];
-      const line = lines[range.start.line];
-      const nextLine = lines[range.start.line + 1];
-
-      if (prevLine) {
-        snippet += `${range.start.line} | ${prevLine}\n`;
-      }
-
-      snippet += `${range.start.line + 1} | ${pc.red(line)}\n`;
-
-      if (nextLine) {
-        snippet += `${range.start.line + 2} | ${nextLine}\n`;
-      }
-
-      suggestReplacement(replacement, {
-        type: 'file',
-        path: filePath,
-        line: range.start.line,
-        column: range.start.column,
-        snippet
-      });
-    }
-  }
-
-  if (matches.length === 0) {
-    return null;
-  }
-
-  return {
-    path: filePath,
-    contents,
-    replacements: matches
-  };
-}
-
-export async function scanFiles(
+export function scanFiles(
   files: string[],
   replacements: modReplacements.ModuleReplacement[],
+  threads: number,
   spinner: ReturnType<typeof cl.spinner>
 ): Promise<FileReplacement[]> {
-  const results: FileReplacement[] = [];
-
-  for (const file of files) {
-    try {
-      const contents = await readFile(file, 'utf8');
-      const lines = contents.split('\n');
-
-      spinner.message(`Scanning ${file}`);
-
-      const scanResult = await scanFile(file, contents, lines, replacements);
-
-      if (scanResult) {
-        results.push(scanResult);
-      }
-    } catch (err) {
-      cl.log.error(dedent`
-        Could not read file ${file}:
-
-        ${String(err)}
-      `);
+  return new Promise((resolve, reject) => {
+    let i = 0;
+    let tasks = 0;
+    const filesLength = files.length;
+    const results: FileReplacement[] = [];
+
+    for (const file of files.splice(0, threads)) {
+      const worker = new Worker(`${__dirname}/workers/scan-file.js`);
+      // todo, what todo with the errors?
+      worker.on('error', (error) => reject(error.message));
+      worker.on('message', (message) => {
+        if (message?.type === 'result') {
+          if (message.value) results.push(message.value);
+          i += 1;
+          if (i === filesLength) {
+            resolve(results);
+          }
+        } else {
+          reject(message.value);
+        }
+        if (files.length > 0) {
+          if (available >= tasks) {
+            const file = files.shift();
+            spinner.message(`Scanning file: ${file}`);
+            worker.postMessage({file, replacements});
+          } else {
+            tasks -= 1;
+          }
+        } else {
+          worker.terminate();
+        }
+      });
+      spinner.message(`Scanning file: ${file}`);
+      worker.postMessage({file, replacements});
     }
-  }
-
-  return results;
+  });
 }
diff --git a/src/stages/workers/scan-file.ts b/src/stages/workers/scan-file.ts
@@ -0,0 +1,115 @@
+import * as modReplacements from 'module-replacements';
+import {ts as sg} from '@ast-grep/napi';
+import dedent from 'dedent';
+import pc from 'picocolors';
+import {type FileReplacement} from './../../shared-types.js';
+import {suggestReplacement} from './../../suggest-replacement.js';
+import {parentPort} from 'node:worker_threads';
+import {open} from 'node:fs/promises';
+
+async function scanFile(
+  filePath: string,
+  contents: string,
+  lines: string[],
+  replacements: modReplacements.ModuleReplacement[]
+): Promise<FileReplacement | null> {
+  const ast = sg.parse(contents);
+  const root = ast.root();
+  const matches: modReplacements.ModuleReplacement[] = [];
+
+  for (const replacement of replacements) {
+    const imports = root.findAll({
+      rule: {
+        any: [
+          {
+            pattern: {
+              context: `import $NAME from '${replacement.moduleName}'`,
+              strictness: 'relaxed'
+            }
+          },
+          {
+            pattern: {
+              context: `require('${replacement.moduleName}')`,
+              strictness: 'relaxed'
+            }
+          }
+        ]
+      }
+    });
+
+    if (imports.length > 0) {
+      matches.push(replacement);
+    }
+
+    for (const node of imports) {
+      const range = node.range();
+      let snippet: string = '';
+
+      const prevLine = lines[range.start.line - 1];
+      const line = lines[range.start.line];
+      const nextLine = lines[range.start.line + 1];
+
+      if (prevLine) {
+        snippet += `${range.start.line} | ${prevLine}\n`;
+      }
+
+      snippet += `${range.start.line + 1} | ${pc.red(line)}\n`;
+
+      if (nextLine) {
+        snippet += `${range.start.line + 2} | ${nextLine}\n`;
+      }
+
+      suggestReplacement(replacement, {
+        type: 'file',
+        path: filePath,
+        line: range.start.line,
+        column: range.start.column,
+        snippet
+      });
+    }
+  }
+
+  if (matches.length === 0) {
+    return null;
+  }
+
+  return {
+    path: filePath,
+    contents,
+    replacements: matches
+  };
+}
+
+async function scanTask(
+  file: string,
+  replacements: modReplacements.ModuleReplacement[]
+) {
+  try {
+    const fd = await open(file);
+    const contents = await fd.readFile({encoding: 'utf-8'});
+    const lines = contents.split('\n');
+    const scanResult = await scanFile(file, contents, lines, replacements);
+    await fd.close();
+
+    parentPort?.postMessage({type: 'result', value: scanResult});
+  } catch (err) {
+    parentPort?.postMessage({
+      type: 'error',
+      value: dedent`
+      Could not read file ${file}:
+
+      ${String(err)}
+    `
+    });
+  }
+}
+
+parentPort?.on(
+  'message',
+  (message: {
+    file: string;
+    replacements: modReplacements.ModuleReplacement[];
+  }) => {
+    scanTask(message.file, message.replacements);
+  }
+);