From d8ef6436b945d477e6de3058c0dd854bd69acb76 Mon Sep 17 00:00:00 2001 From: Christian Emmer <10749361+emmercm@users.noreply.github.com> Date: Sun, 22 Oct 2023 18:53:17 -0700 Subject: [PATCH] Fix: replace robloach-datfile with a custom CMPro DAT parser (#784) --- package-lock.json | 59 -------- package.json | 1 - src/modules/datScanner.ts | 62 ++++++--- src/types/dats/cmpro/cmProParser.ts | 206 ++++++++++++++++++++++++++++ 4 files changed, 250 insertions(+), 78 deletions(-) create mode 100644 src/types/dats/cmpro/cmProParser.ts diff --git a/package-lock.json b/package-lock.json index c5fffea8c..b50e50cb3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30,7 +30,6 @@ "node-disk-info": "1.3.0", "node-unrar-js": "2.0.0", "reflect-metadata": "0.1.13", - "robloach-datfile": "2.4.0", "semver": "7.5.4", "simple-statistics": "7.8.3", "strip-ansi": "7.1.0", @@ -3184,14 +3183,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/assign-symbols": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-1.0.0.tgz", - "integrity": "sha512-Q+JC7Whu8HhmTdBph/Tq59IoRtoy6KAm5zzPv00WdujX82lbAL8K7WVjne7vdCsAmbF4AYaDOPyO3k0kl8qIrw==", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/async": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", @@ -5047,18 +5038,6 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, - "node_modules/extend-shallow": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz", - "integrity": "sha512-BwY5b5Ql4+qZoefgMj2NUmx+tehVTH/Kf4k1ZEtOHNFcm2wSxMRo992l6X3TIgni2eZVTZ85xMOjF31fwZAj6Q==", - "dependencies": { - "assign-symbols": "^1.0.0", - "is-extendable": "^1.0.1" - }, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -5982,17 +5961,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-extendable": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", - "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", - "dependencies": { - "is-plain-object": "^2.0.4" - }, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -6073,17 +6041,6 @@ "node": ">=8" } }, - "node_modules/is-plain-object": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", - "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", - "dependencies": { - "isobject": "^3.0.1" - }, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/is-regex": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz", @@ -6196,14 +6153,6 @@ "node": ">=16" } }, - "node_modules/isobject": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", - "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/istanbul-lib-coverage": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.0.tgz", @@ -9057,14 +9006,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/robloach-datfile": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/robloach-datfile/-/robloach-datfile-2.4.0.tgz", - "integrity": "sha512-owIJNLBTe6siSv0vEJLZFZRYm41xb5OdWaS1rwUmkfovrsFhsTAYveFQoZhy4bis2TCc++zG0TfOwYCErLeJlQ==", - "dependencies": { - "extend-shallow": "^3.0.2" - } - }, "node_modules/run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", diff --git a/package.json b/package.json index acca3b971..2813523e1 100644 --- a/package.json +++ b/package.json @@ -83,7 +83,6 @@ "node-disk-info": "1.3.0", "node-unrar-js": "2.0.0", "reflect-metadata": "0.1.13", - "robloach-datfile": "2.4.0", "semver": "7.5.4", "simple-statistics": "7.8.3", "strip-ansi": "7.1.0", diff --git a/src/modules/datScanner.ts b/src/modules/datScanner.ts index 7d236e70e..228f72f5d 100644 --- a/src/modules/datScanner.ts +++ b/src/modules/datScanner.ts @@ -3,7 +3,6 @@ import path from 'node:path'; import { parse } from '@fast-csv/parse'; import async, { AsyncResultCallback } from 'async'; -import robloachDatfile from 'robloach-datfile'; import xml2js from 'xml2js'; import ProgressBar, { ProgressBarSymbol } from '../console/progressBar.js'; @@ -11,6 +10,7 @@ import Constants from '../constants.js'; import ArrayPoly from '../polyfill/arrayPoly.js'; import bufferPoly from '../polyfill/bufferPoly.js'; import fsPoly from '../polyfill/fsPoly.js'; +import CMProParser, { DATProps, GameProps, ROMProps } from '../types/dats/cmpro/cmProParser.js'; import DAT from '../types/dats/dat.js'; import DATObject from '../types/dats/datObject.js'; import Game from '../types/dats/game.js'; @@ -232,7 +232,7 @@ export default class DATScanner extends Scanner { return xmlDat; } - const cmproDatParsed = await this.parseCmproDat(datFile, fileContents); + const cmproDatParsed = this.parseCmproDat(datFile, fileContents); if (cmproDatParsed) { return cmproDatParsed; } @@ -286,10 +286,9 @@ export default class DATScanner extends Scanner { return undefined; } - private async parseCmproDat(datFile: File, fileContents: string): Promise { + private parseCmproDat(datFile: File, fileContents: string): DAT | undefined { /** - * Sanity check that this might be a CMPro file, otherwise {@link robloachDatfile} has a chance - * to throw fatal errors. + * Sanity check that this might be a CMPro file. */ if (fileContents.match(/^(clrmamepro|game|resource) \(\r?\n(\t.+\r?\n)+\)$/m) === null) { return undefined; @@ -297,27 +296,46 @@ export default class DATScanner extends Scanner { this.progressBar.logTrace(`${datFile.toString()}: attempting to parse CMPro DAT`); - let cmproDat; + let cmproDat: DATProps; try { - cmproDat = await robloachDatfile.parse(fileContents); + cmproDat = new CMProParser(fileContents).parse(); } catch (error) { this.progressBar.logDebug(`${datFile.toString()}: failed to parse CMPro DAT: ${error}`); return undefined; } - if (cmproDat.length === 0) { - this.progressBar.logWarn(`${datFile.toString()}: failed to parse CMPro DAT, no header or games found`); - return undefined; - } this.progressBar.logTrace(`${datFile.toString()}: parsed CMPro DAT, deserializing to DAT`); - const header = new Header(cmproDat[0]); + const header = new Header({ + name: cmproDat.clrmamepro?.name, + description: cmproDat.clrmamepro?.description, + version: cmproDat.clrmamepro?.version, + date: cmproDat.clrmamepro?.date, + author: cmproDat.clrmamepro?.author, + url: cmproDat.clrmamepro?.url, + comment: cmproDat.clrmamepro?.comment, + }); + + let cmproDatGames: GameProps[] = []; + if (cmproDat.game) { + if (Array.isArray(cmproDat.game)) { + cmproDatGames = cmproDat.game; + } else { + cmproDatGames = [cmproDat.game]; + } + } + + const games = cmproDatGames.flatMap((game) => { + let gameRoms: ROMProps[] = []; + if (game.rom) { + if (Array.isArray(game.rom)) { + gameRoms = game.rom; + } else { + gameRoms = [game.rom]; + } + } - const cmproGames = cmproDat.slice(1); - const games = cmproGames.flatMap((obj) => { - const game = obj as DatfileGame; - // TODO(cemmer): https://github.com/RobLoach/datfile/issues/2 - const roms = (game.entries ?? []) + const roms = gameRoms .filter((rom) => rom.name) // we need ROM filenames .map((entry) => new ROM({ name: entry.name ?? '', @@ -328,7 +346,15 @@ export default class DATScanner extends Scanner { })); return new Game({ - ...game, + name: game.name, + category: undefined, + description: game.description, + bios: undefined, + device: undefined, + cloneOf: game.cloneof, + romOf: game.romof, + sampleOf: undefined, + release: undefined, rom: roms, }); }); diff --git a/src/types/dats/cmpro/cmProParser.ts b/src/types/dats/cmpro/cmProParser.ts new file mode 100644 index 000000000..9cbb5838c --- /dev/null +++ b/src/types/dats/cmpro/cmProParser.ts @@ -0,0 +1,206 @@ +export interface DATProps extends CMProObject { + clrmamepro?: ClrMameProProps, + game?: GameProps | GameProps[], + resource?: GameProps | Resource[], +} + +export interface ClrMameProProps extends CMProObject { + name?: string, + description?: string, + category?: string, + version?: string, + forcemerging?: 'none' | 'split' | 'full', + forcezipping?: 'yes' | 'no', + sampleOf?: string, + // NON-STANDARD PROPERTIES + date?: string, + author?: string, + homepage?: string, + url?: string, + comment?: string, +} + +export interface GameProps extends CMProObject { + name: string, + description: string, + year?: string, + manufacturer?: string, + cloneof?: string, + romof?: string, + sampleof?: string, + rom?: ROMProps | ROMProps[], + disk?: DiskProps | DiskProps[], + sample?: SampleProps | SampleProps[], + // NON-STANDARD PROPERTIES + serial?: string, + publisher?: string, + releaseyear?: string, + releasemonth?: string, + developer?: string, + users?: string, + esrbrating?: string, +} + +export interface ROMProps extends CMProObject { + name: string, + merge?: string, + size?: string, + crc?: string, + flags?: string, + md5?: string, + sha1?: string, + // NON-STANDARD PROPERTIES + serial?: string, +} + +export interface DiskProps extends ROMProps {} + +export interface SampleProps extends CMProObject { + name: string, +} + +export interface Resource extends GameProps {} + +type CMProValue = CMProObject | string | undefined; + +type CMProObject = { [key: string]: CMProValue | CMProValue[] }; + +/** + * A parser for CMPRo schema DATs. + * @see http://www.logiqx.com/DatFAQs/CMPro.php + */ +export default class CMProParser { + private static readonly WHITESPACE_CHARS = new Set([' ', '\t', '\n', '\r', '\v']); + + private readonly contents: string; + + private pos = 0; + + constructor(contents: string) { + this.contents = contents; + } + + /** + * Parse the CMPro DAT's file contents. + */ + public parse(): DATProps { + this.pos = 0; + + const result: CMProObject = {}; + while (this.pos < this.contents.length) { + const tag = this.parseTag(); + const value = this.parseValue(); + + const existing = result[tag]; + if (existing !== undefined) { + if (Array.isArray(existing)) { + result[tag] = [...existing, value]; + } else { + result[tag] = [existing, value]; + } + } else { + result[tag] = value; + } + + this.skipWhitespace(); + } + return result; + } + + private skipWhitespace(): void { + while (CMProParser.WHITESPACE_CHARS.has(this.contents.charAt(this.pos))) { + this.pos += 1; + } + } + + private parseObject(): CMProObject { + if (this.contents.charAt(this.pos) === '(') { + this.pos += 1; + } + this.skipWhitespace(); + + const result: CMProObject = {}; + while (this.contents.charAt(this.pos) !== ')') { + const tag = this.parseTag(); + const value = this.parseValue(); + + const existing = result[tag]; + if (existing !== undefined) { + if (Array.isArray(existing)) { + result[tag] = [...existing, value]; + } else { + result[tag] = [existing, value]; + } + } else { + result[tag] = value; + } + + this.skipWhitespace(); + } + this.pos += 1; + return result; + } + + private parseTag(): string { + this.skipWhitespace(); + + const initialPos = this.pos; + while (!CMProParser.WHITESPACE_CHARS.has(this.contents.charAt(this.pos))) { + this.pos += 1; + } + + return this.contents.slice(initialPos, this.pos); + } + + private parseValue(): CMProValue { + this.skipWhitespace(); + + // Parse object + if (this.contents.charAt(this.pos) === '(') { + this.pos += 1; + return this.parseObject(); + } + + // Parse quoted string + if (this.contents.charAt(this.pos) === '"') { + return this.parseQuotedString(); + } + + // Parse unquoted string + return this.parseUnquotedString(); + } + + private parseQuotedString(): string { + if (this.contents.charAt(this.pos) !== '"') { + throw new Error('invalid quoted string'); + } + this.pos += 1; + + const initialPos = this.pos; + while (this.pos < this.contents.length) { + // String termination, return the value + if (this.contents.charAt(this.pos) === '"') { + const value = this.contents.slice(initialPos, this.pos); + this.pos += 1; + return value; + } + + // Quoted character, skip it + if (this.contents.charAt(this.pos) === '\\') { + this.pos += 2; + } else { + this.pos += 1; + } + } + + throw new Error('invalid quoted string'); + } + + private parseUnquotedString(): string { + const initialPos = this.pos; + while (!CMProParser.WHITESPACE_CHARS.has(this.contents.charAt(this.pos))) { + this.pos += 1; + } + return this.contents.slice(initialPos, this.pos); + } +}