Refactor: IndexedFiles class

src/igir.ts

@@ -21,12 +21,12 @@ import DATParentInferrer from './modules/datParentInferrer.js';
 import DATScanner from './modules/datScanner.js';
 import Dir2DatCreator from './modules/dir2DatCreator.js';
 import DirectoryCleaner from './modules/directoryCleaner.js';
-import FileIndexer from './modules/fileIndexer.js';
 import FixdatCreator from './modules/fixdatCreator.js';
 import MovedROMDeleter from './modules/movedRomDeleter.js';
 import PatchScanner from './modules/patchScanner.js';
 import ReportGenerator from './modules/reportGenerator.js';
 import ROMHeaderProcessor from './modules/romHeaderProcessor.js';
+import ROMIndexer from './modules/romIndexer.js';
 import ROMScanner from './modules/romScanner.js';
 import StatusGenerator from './modules/statusGenerator.js';
 import ArrayPoly from './polyfill/arrayPoly.js';
@@ -35,6 +35,7 @@ import DAT from './types/dats/dat.js';
 import Parent from './types/dats/parent.js';
 import DATStatus from './types/datStatus.js';
 import File from './types/files/file.js';
+import IndexedFiles from './types/indexedFiles.js';
 import Options from './types/options.js';
 import OutputFactory from './types/outputFactory.js';
 import Patch from './types/patches/patch.js';
@@ -73,9 +74,7 @@ export default class Igir {
     // Scan and process input files
     let dats = await this.processDATScanner();
     const indexedRoms = await this.processROMScanner();
-    const roms = [...indexedRoms.values()]
-      .flat()
-      .reduce(ArrayPoly.reduceUnique(), []);
+    const roms = indexedRoms.getFiles();
     const patches = await this.processPatchScanner();
 
     // Set up progress bar and input for DAT processing
@@ -218,7 +217,7 @@ export default class Igir {
     return dats;
   }
 
-  private async processROMScanner(): Promise<Map<string, File[]>> {
+  private async processROMScanner(): Promise<IndexedFiles> {
     const romScannerProgressBarName = 'Scanning for ROMs';
     const romProgressBar = await this.logger.addProgressBar(romScannerProgressBarName);
 
@@ -229,7 +228,7 @@ export default class Igir {
       .process(rawRomFiles);
 
     await romProgressBar.setName('Indexing ROMs');
-    const indexedRomFiles = await new FileIndexer(this.options, romProgressBar)
+    const indexedRomFiles = await new ROMIndexer(this.options, romProgressBar)
       .index(romFilesWithHeaders);
 
     await romProgressBar.setName(romScannerProgressBarName); // reset
@@ -254,7 +253,7 @@ export default class Igir {
   private async generateCandidates(
     progressBar: ProgressBar,
     dat: DAT,
-    indexedRoms: Map<string, File[]>,
+    indexedRoms: IndexedFiles,
     patches: Patch[],
   ): Promise<Map<Parent, ReleaseCandidate[]>> {
     const candidates = await new CandidateGenerator(this.options, progressBar)

src/modules/candidateGenerator.ts

@@ -12,6 +12,7 @@ import Archive from '../types/files/archives/archive.js';
 import ArchiveEntry from '../types/files/archives/archiveEntry.js';
 import Zip from '../types/files/archives/zip.js';
 import File from '../types/files/file.js';
+import IndexedFiles from '../types/indexedFiles.js';
 import Options from '../types/options.js';
 import OutputFactory from '../types/outputFactory.js';
 import ReleaseCandidate from '../types/releaseCandidate.js';
@@ -37,9 +38,9 @@ export default class CandidateGenerator extends Module {
    */
   async generate(
     dat: DAT,
-    hashCodeToInputFiles: Map<string, File[]>,
+    indexedFiles: IndexedFiles,
   ): Promise<Map<Parent, ReleaseCandidate[]>> {
-    if (hashCodeToInputFiles.size === 0) {
+    if (indexedFiles.getFiles().length === 0) {
       this.progressBar.logTrace(`${dat.getNameShort()}: no input ROMs to make candidates from`);
       return new Map();
     }
@@ -71,7 +72,7 @@ export default class CandidateGenerator extends Module {
             dat,
             game,
             release,
-            hashCodeToInputFiles,
+            indexedFiles,
           );
           if (releaseCandidate) {
             releaseCandidates.push(releaseCandidate);
@@ -104,9 +105,9 @@ export default class CandidateGenerator extends Module {
     dat: DAT,
     game: Game,
     release: Release | undefined,
-    hashCodeToInputFiles: Map<string, File[]>,
+    indexedFiles: IndexedFiles,
   ): Promise<ReleaseCandidate | undefined> {
-    const romsToInputFiles = this.getInputFilesForGame(game, hashCodeToInputFiles);
+    const romsToInputFiles = this.getInputFilesForGame(dat, game, indexedFiles);
 
     // For each Game's ROM, find the matching File
     const romFiles = await Promise.all(
@@ -130,7 +131,7 @@ export default class CandidateGenerator extends Module {
 
         // If the input file is headered...
         if (inputFile.getFileHeader()
-          // ..and we want a headered ROM
+          // ...and we want a headered ROM
           && (inputFile.getCrc32() === rom.getCrc32()
             || inputFile.getMd5() === rom.getMd5()
             || inputFile.getSha1() === rom.getSha1())
@@ -141,6 +142,7 @@ export default class CandidateGenerator extends Module {
           )
         ) {
           // ...then forget the input file's header, so that we don't later remove it
+          this.progressBar.logTrace(`${dat.getNameShort()}: ${game.getName()}: not removing header, ignoring that one was found for: ${inputFile.toString()}`);
           inputFile = inputFile.withoutFileHeader();
         }
 
@@ -154,6 +156,7 @@ export default class CandidateGenerator extends Module {
           && this.options.shouldLink()
         ) {
           // ...then we can't use this file
+          this.progressBar.logTrace(`${dat.getNameShort()}: ${game.getName()}: can't use headered ROM as target for link: ${inputFile.toString()}`);
           return [rom, undefined];
         }
 
@@ -214,12 +217,13 @@ export default class CandidateGenerator extends Module {
   }
 
   private getInputFilesForGame(
+    dat: DAT,
     game: Game,
-    hashCodeToInputFiles: Map<string, File[]>,
+    indexedFiles: IndexedFiles,
   ): Map<ROM, File> {
     const romsAndInputFiles = game.getRoms().map((rom) => ([
       rom,
-      (hashCodeToInputFiles.get(rom.hashCode()) ?? []),
+      indexedFiles.findFiles(rom) ?? [],
     ])) satisfies [ROM, File[]][];
 
     // Detect if there is one input archive that contains every ROM, and prefer to use its entries.
@@ -274,6 +278,7 @@ export default class CandidateGenerator extends Module {
     // An Archive was found, use that as the only possible input file
     // For each of this Game's ROMs, find the matching ArchiveEntry from this Archive
     return new Map(romsAndInputFiles.map(([rom, inputFiles]) => {
+      this.progressBar.logTrace(`${dat.getNameShort()}: ${game.getName()}: preferring input archive that contains every ROM: ${archiveWithEveryRom.getFilePath()}`);
       const archiveEntry = inputFiles.find((
         inputFile,
       ) => inputFile.getFilePath() === archiveWithEveryRom.getFilePath()) as File;

src/modules/movedRomDeleter.ts

@@ -78,7 +78,7 @@ export default class MovedROMDeleter extends Module {
         //  the unique set of ArchiveEntry hash codes to know if every ArchiveEntry was "consumed"
         //  during writing.
         const movedEntryHashCodes = new Set(
-          movedEntries.flatMap((file) => file.hashCodes()),
+          movedEntries.flatMap((file) => file.hashCode()),
         );
 
         const inputEntries = groupedInputRoms.get(filePath) ?? [];
@@ -94,7 +94,7 @@ export default class MovedROMDeleter extends Module {
           }
 
           // Otherwise, the entry needs to have been explicitly moved
-          return !entry.hashCodes().some((hashCode) => movedEntryHashCodes.has(hashCode));
+          return !movedEntryHashCodes.has(entry.hashCode());
         });
         if (unmovedEntries.length > 0) {
           this.progressBar.logWarn(`${filePath}: not deleting moved file, ${unmovedEntries.length.toLocaleString()} archive entr${unmovedEntries.length !== 1 ? 'ies were' : 'y was'} unmatched:\n${unmovedEntries.sort().map((entry) => `  ${entry}`).join('\n')}`);

src/modules/romIndexer.ts

@@ -0,0 +1,103 @@
+import path from 'node:path';
+
+import ProgressBar, { ProgressBarSymbol } from '../console/progressBar.js';
+import FsPoly from '../polyfill/fsPoly.js';
+import ArchiveEntry from '../types/files/archives/archiveEntry.js';
+import Rar from '../types/files/archives/rar.js';
+import SevenZip from '../types/files/archives/sevenZip.js';
+import Tar from '../types/files/archives/tar.js';
+import Zip from '../types/files/archives/zip.js';
+import File from '../types/files/file.js';
+import IndexedFiles, { AllChecksums, ChecksumsToFiles } from '../types/indexedFiles.js';
+import Options from '../types/options.js';
+import Module from './module.js';
+
+/**
+ * This class indexes {@link File}s by their {@link File.hashCode}, and sorts duplicate files by a
+ * set of preferences.
+ */
+export default class ROMIndexer extends Module {
+  protected readonly options: Options;
+
+  constructor(options: Options, progressBar: ProgressBar) {
+    super(progressBar, ROMIndexer.name);
+    this.options = options;
+  }
+
+  /**
+   * Index files.
+   */
+  async index(files: File[]): Promise<IndexedFiles> {
+    this.progressBar.logTrace(`indexing ${files.length.toLocaleString()} file${files.length !== 1 ? 's' : ''}`);
+    await this.progressBar.setSymbol(ProgressBarSymbol.INDEXING);
+    // await this.progressBar.reset(files.length);
+
+    // Index the files
+    const result = IndexedFiles.fromFiles(files);
+    // Then apply some sorting preferences
+    Object.keys(result).forEach((checksum) => this.sortMap(result[checksum as keyof AllChecksums]));
+
+    this.progressBar.logTrace(`found ${result.getSize()} unique file${result.getSize() !== 1 ? 's' : ''}`);
+
+    this.progressBar.logTrace('done indexing files');
+    return result;
+  }
+
+  private sortMap(checksumsToFiles: ChecksumsToFiles): void {
+    const outputDir = path.resolve(this.options.getOutputDirRoot());
+    const outputDirDisk = FsPoly.disksSync().find((mount) => outputDir.startsWith(mount));
+
+    [...checksumsToFiles.values()]
+      .forEach((files) => files
+        .sort((fileOne, fileTwo) => {
+          // Prefer un-archived files
+          const fileOneArchived = ROMIndexer.archiveEntryPriority(fileOne);
+          const fileTwoArchived = ROMIndexer.archiveEntryPriority(fileTwo);
+          if (fileOneArchived !== fileTwoArchived) {
+            return fileOneArchived - fileTwoArchived;
+          }
+
+          // Then, prefer files that are NOT already in the output directory
+          // This is in case the output file is invalid and we're trying to overwrite it with
+          // something else. Otherwise, we'll just attempt to overwrite the invalid output file with
+          // itself, still resulting in an invalid output file.
+          const fileOneInOutput = path.resolve(fileOne.getFilePath()).startsWith(outputDir) ? 1 : 0;
+          const fileTwoInOutput = path.resolve(fileTwo.getFilePath()).startsWith(outputDir) ? 1 : 0;
+          if (fileOneInOutput !== fileTwoInOutput) {
+            return fileOneInOutput - fileTwoInOutput;
+          }
+
+          // Then, prefer files that are on the same disk for fs efficiency see {@link FsPoly#mv}
+          if (outputDirDisk) {
+            const fileOneInOutputDisk = path.resolve(fileOne.getFilePath())
+              .startsWith(outputDirDisk) ? 0 : 1;
+            const fileTwoInOutputDisk = path.resolve(fileTwo.getFilePath())
+              .startsWith(outputDirDisk) ? 0 : 1;
+            if (fileOneInOutputDisk !== fileTwoInOutputDisk) {
+              return fileOneInOutputDisk - fileTwoInOutputDisk;
+            }
+          }
+
+          // Otherwise, be deterministic
+          return fileOne.getFilePath().localeCompare(fileTwo.getFilePath());
+        }));
+  }
+
+  /**
+   * This ordering should match {@link FileFactory#archiveFrom}
+   */
+  private static archiveEntryPriority(file: File): number {
+    if (!(file instanceof ArchiveEntry)) {
+      return 0;
+    } if (file.getArchive() instanceof Zip) {
+      return 1;
+    } if (file.getArchive() instanceof Tar) {
+      return 2;
+    } if (file.getArchive() instanceof Rar) {
+      return 3;
+    } if (file.getArchive() instanceof SevenZip) {
+      return 4;
+    }
+    return 99;
+  }
+}