Add worker

src/stages/scan-files.ts

@@ -1,112 +1,80 @@
 import * as modReplacements from 'module-replacements';
-import {ts as sg} from '@ast-grep/napi';
-import {readFile} from 'node:fs/promises';
-import dedent from 'dedent';
-import pc from 'picocolors';
 import * as cl from '@clack/prompts';
 import {type FileReplacement} from '../shared-types.js';
-import {suggestReplacement} from '../suggest-replacement.js';
+import {cpuUsage} from 'node:process';
+import {cpus, availableParallelism} from 'node:os';
+import {Worker} from 'node:worker_threads';
+import {fork} from 'node:child_process';
+import Events from 'node:events';
+const __dirname = import.meta.dirname;
 
-async function scanFile(
-  filePath: string,
-  contents: string,
-  lines: string[],
-  replacements: modReplacements.ModuleReplacement[]
-): Promise<FileReplacement | null> {
-  const ast = sg.parse(contents);
-  const root = ast.root();
-  const matches: modReplacements.ModuleReplacement[] = [];
-
-  for (const replacement of replacements) {
-    const imports = root.findAll({
-      rule: {
-        any: [
-          {
-            pattern: {
-              context: `import $NAME from '${replacement.moduleName}'`,
-              strictness: 'relaxed'
-            }
-          },
-          {
-            pattern: {
-              context: `require('${replacement.moduleName}')`,
-              strictness: 'relaxed'
-            }
-          }
-        ]
-      }
-    });
-
-    if (imports.length > 0) {
-      matches.push(replacement);
-    }
-
-    for (const node of imports) {
-      const range = node.range();
-      let snippet: string = '';
-
-      const prevLine = lines[range.start.line - 1];
-      const line = lines[range.start.line];
-      const nextLine = lines[range.start.line + 1];
-
-      if (prevLine) {
-        snippet += `${range.start.line} | ${prevLine}\n`;
-      }
-
-      snippet += `${range.start.line + 1} | ${pc.red(line)}\n`;
+const events = new Events();
 
-      if (nextLine) {
-        snippet += `${range.start.line + 2} | ${nextLine}\n`;
-      }
+let tasks = 0;
 
-      suggestReplacement(replacement, {
-        type: 'file',
-        path: filePath,
-        line: range.start.line,
-        column: range.start.column,
-        snippet
-      });
+export async function _scanFile(
+  file: string,
+  replacements: modReplacements.ModuleReplacement[]
+): Promise<FileReplacement> {
+  // return new Promise((resolve, reject) => {
+  const worker = fork(`${__dirname}/workers/scan-file.js`);
+  worker.on('error', (error) => reject(error));
+  worker.once('message', (message) => {
+    if (message?.type === 'result') {
+      // resolve(message.value);
+      events.emit('file-scan-worker-result', message.value);
+    } else {
+      console.error(message.value);
+
+      // reject(message.value);
     }
-  }
-
-  if (matches.length === 0) {
-    return null;
-  }
-
-  return {
-    path: filePath,
-    contents,
-    replacements: matches
-  };
+    events.emit('file-scan-worker-done');
+    worker.kill();
+  });
+  worker.send({file, replacements});
+  // });
 }
 
-export async function scanFiles(
+export function scanFiles(
   files: string[],
   replacements: modReplacements.ModuleReplacement[],
-  spinner: ReturnType<typeof cl.spinner>
+  spinner: ReturnType<typeof cl.spinner>,
+  results: FileReplacement[]
 ): Promise<FileReplacement[]> {
-  const results: FileReplacement[] = [];
-
-  for (const file of files) {
-    try {
-      const contents = await readFile(file, 'utf8');
-      const lines = contents.split('\n');
-
-      spinner.message(`Scanning ${file}`);
-
-      const scanResult = await scanFile(file, contents, lines, replacements);
+  return new Promise((resolve, reject) => {
+    if (!results) results = [];
+    let i = 0;
+    const filesLength = files.length;
+
+    const runJob = () => {
+      const available = availableParallelism();
+
+      let maxThreads;
+      if (!tasks) maxThreads = available;
+      else maxThreads = available - tasks;
+
+      tasks = tasks + maxThreads;
+      const targets = files.splice(
+        0,
+        files.length < maxThreads ? files.length : maxThreads
+      );
+
+      spinner.message(`Scanning files: ${targets.join(', ')}`);
+      targets.forEach((file) => _scanFile(file, replacements));
+    };
+    events.on('file-scan-worker-done', () => {
+      if (files.length > 0) runJob();
+      tasks -= 1;
+    });
 
-      if (scanResult) {
-        results.push(scanResult);
+    events.on('file-scan-worker-result', (result) => {
+      results.push(result);
+      i += 1;
+      if (i === filesLength) {
+        resolve(results);
       }
-    } catch (err) {
-      cl.log.error(dedent`
-        Could not read file ${file}:
-
-        ${String(err)}
-      `);
-    }
-  }
+    });
 
-  return results;
+    runJob();
+  });
 }

src/stages/workers/scan-file.ts

@@ -0,0 +1,123 @@
+import * as modReplacements from 'module-replacements';
+import {ts as sg} from '@ast-grep/napi';
+import {readFile} from 'node:fs/promises';
+import dedent from 'dedent';
+import pc from 'picocolors';
+import {type FileReplacement} from './../../shared-types.js';
+import {suggestReplacement} from './../../suggest-replacement.js';
+
+async function scanFile(
+  filePath: string,
+  contents: string,
+  lines: string[],
+  replacements: modReplacements.ModuleReplacement[]
+): Promise<FileReplacement | null> {
+  const ast = sg.parse(contents);
+  const root = ast.root();
+  const matches: modReplacements.ModuleReplacement[] = [];
+
+  for (const replacement of replacements) {
+    const imports = root.findAll({
+      rule: {
+        any: [
+          {
+            pattern: {
+              context: `import $NAME from '${replacement.moduleName}'`,
+              strictness: 'relaxed'
+            }
+          },
+          {
+            pattern: {
+              context: `require('${replacement.moduleName}')`,
+              strictness: 'relaxed'
+            }
+          }
+        ]
+      }
+    });
+
+    if (imports.length > 0) {
+      matches.push(replacement);
+    }
+
+    for (const node of imports) {
+      const range = node.range();
+      let snippet: string = '';
+
+      const prevLine = lines[range.start.line - 1];
+      const line = lines[range.start.line];
+      const nextLine = lines[range.start.line + 1];
+
+      if (prevLine) {
+        snippet += `${range.start.line} | ${prevLine}\n`;
+      }
+
+      snippet += `${range.start.line + 1} | ${pc.red(line)}\n`;
+
+      if (nextLine) {
+        snippet += `${range.start.line + 2} | ${nextLine}\n`;
+      }
+
+      suggestReplacement(replacement, {
+        type: 'file',
+        path: filePath,
+        line: range.start.line,
+        column: range.start.column,
+        snippet
+      });
+    }
+  }
+
+  if (matches.length === 0) {
+    return null;
+  }
+
+  return {
+    path: filePath,
+    contents,
+    replacements: matches
+  };
+}
+
+async function scanTask(
+  file: string,
+  replacements: modReplacements.ModuleReplacement[]
+) {
+  try {
+    const contents = await readFile(file, 'utf8');
+    const lines = contents.split('\n');
+
+    const scanResult = await scanFile(file, contents, lines, replacements);
+
+    // if (scanResult) {
+    // @ts-ignore
+    process.send({type: 'result', value: scanResult});
+    // } else {
+    // process.exit(0);
+    // }
+  } catch (err) {
+    process.send({
+      type: 'error',
+      value: dedent`
+      Could not read file ${file}:
+
+      ${String(err)}
+    `
+    });
+    // cl.log.error(dedent`
+    //   Could not read file ${file}:
+
+    //   ${String(err)}
+    // `);
+  }
+}
+
+process.on(
+  'message',
+  (message: {
+    file: string;
+    replacements: modReplacements.ModuleReplacement[];
+  }) => {
+    scanTask(message.file, message.replacements);
+  }
+);