Skip to content

Commit

Permalink
Merge pull request #166 from kaitai-io/switch-to-js-yaml
Browse files Browse the repository at this point in the history
Use js-yaml (instead of yaml.js) for YAML parsing
  • Loading branch information
generalmimon authored Feb 10, 2024
2 parents 80c94b1 + d1a7ddd commit 5de330d
Show file tree
Hide file tree
Showing 12 changed files with 264 additions and 22 deletions.
32 changes: 32 additions & 0 deletions LICENSE-3RD-PARTY.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1230,6 +1230,38 @@ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
================================================================================

================================================================================
js-yaml

License name: MIT
License URL: https://github.com/nodeca/js-yaml/blob/master/LICENSE
License applies to files under the folder lib/_npm/js-yaml/

Source: https://github.com/nodeca/js-yaml
================================================================================
(The MIT License)

Copyright (C) 2011-2015 by Vitaly Puzrin

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
================================================================================

================================================================================
jsTree

Expand Down
4 changes: 2 additions & 2 deletions Playground.html
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@
<body>
<script>
loader.paths = {
"yamljs": "lib/_npm/yamljs/yaml",
"js-yaml": "lib/_npm/js-yaml/js-yaml.min",
"KaitaiStream": "lib/_npm/kaitai-struct/KaitaiStream",
"kaitai-struct-compiler": "lib/_npm/kaitai-struct-compiler/kaitai-struct-compiler"
};

require(["Playground"]);
</script>
</body>
Expand Down
5 changes: 5 additions & 0 deletions docs/wiki/3rd-party-libraries.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,11 @@ Source: https://github.com/jquery/jquery

License: MIT (https://raw.githubusercontent.com/jquery/jquery/master/LICENSE.txt)

## js-yaml
Source: https://github.com/nodeca/js-yaml

License: MIT (https://github.com/nodeca/js-yaml/blob/master/LICENSE)

## jsTree
Website: https://www.jstree.com/

Expand Down
2 changes: 1 addition & 1 deletion index.html
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
<script src="lib/_npm/requirejs/require.js"></script>
<script src="lib/_npm/jquery/jquery.min.js"></script>
<script src="lib/_npm/bootstrap/js/bootstrap.min.js"></script>
<script src="lib/_npm/yamljs/yaml.js"></script>
<script src="lib/_npm/ace/ace.js"></script>

<script src="js/autorefresh.js"></script>
Expand All @@ -37,6 +36,7 @@
["bowser", "jstree", "localforage", "goldenlayout", "vue", "kaitai-struct-compiler", "dateformat"].forEach(
name => paths[name] = `../../lib/_npm/${name}/${name}`);
paths["big-integer"] = "../../lib/_npm/BigInteger/BigInteger";
paths["js-yaml"] = "../../lib/_npm/js-yaml/js-yaml.min";

requirejs.config({ baseUrl: "js/v1/", paths: paths });
require(["app.unsupportedBrowser"]);
Expand Down
165 changes: 165 additions & 0 deletions lib/ts-types/js-yaml.d.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
// Type definitions for js-yaml v4.0.0
// Project: https://github.com/nodeca/js-yaml

// Adapted from https://github.com/DefinitelyTyped/DefinitelyTyped/blob/65afafadd93759348909de1cdad5df5789c94326/types/js-yaml/index.d.ts
// 1. fixed compatibility with TypeScript 2 (which we're still
// using at the time of writing) by replacing `unknown` with `any`
// 2. wrap everything in `declare namespace jsyaml { ... }` and export it
// as "js-yaml" module so that it works in our setup

declare namespace jsyaml {
export function load(str: string, opts?: LoadOptions): any;

export class Type {
constructor(tag: string, opts?: TypeConstructorOptions);

kind: "sequence" | "scalar" | "mapping" | null;

resolve(data: any): boolean;

construct(data: any, type?: string): any;

instanceOf: object | null;
predicate: ((data: object) => boolean) | null;
represent: ((data: object) => any) | { [x: string]: (data: object) => any } | null;
representName: ((data: object) => any) | null;
defaultStyle: string | null;
multi: boolean;
styleAliases: { [x: string]: any };
}

export class Schema {
constructor(definition: SchemaDefinition | Type[] | Type);

extend(types: SchemaDefinition | Type[] | Type): Schema;
}

export function loadAll(str: string, iterator?: null, opts?: LoadOptions): any[];
export function loadAll(str: string, iterator: (doc: any) => void, opts?: LoadOptions): void;

export function dump(obj: any, opts?: DumpOptions): string;

export interface LoadOptions {
/** string to be used as a file path in error/warning messages. */
filename?: string | undefined;

/** function to call on warning messages. */
onWarning?(this: null, e: YAMLException): void;

/** specifies a schema to use. */
schema?: Schema | undefined;
/** compatibility with JSON.parse behaviour. */
json?: boolean | undefined;

/** listener for parse events */
listener?(this: State, eventType: EventType, state: State): void;
}

export type EventType = "open" | "close";

export interface State {
input: string;
filename: string | null;
schema: Schema;
onWarning: (this: null, e: YAMLException) => void;
json: boolean;
length: number;
position: number;
line: number;
lineStart: number;
lineIndent: number;
version: null | number;
checkLineBreaks: boolean;
kind: string;
result: any;
implicitTypes: Type[];
}

export interface DumpOptions {
/** indentation width to use (in spaces). */
indent?: number | undefined;
/** when true, will not add an indentation level to array elements */
noArrayIndent?: boolean | undefined;
/** do not throw on invalid types (like function in the safe schema) and skip pairs and single values with such types. */
skipInvalid?: boolean | undefined;
/** specifies level of nesting, when to switch from block to flow style for collections. -1 means block style everwhere */
flowLevel?: number | undefined;
/** Each tag may have own set of styles. - "tag" => "style" map. */
styles?: { [x: string]: any } | undefined;
/** specifies a schema to use. */
schema?: Schema | undefined;
/** if true, sort keys when dumping YAML. If a function, use the function to sort the keys. (default: false) */
sortKeys?: boolean | ((a: any, b: any) => number) | undefined;
/** set max line width. (default: 80) */
lineWidth?: number | undefined;
/** if true, don't convert duplicate objects into references (default: false) */
noRefs?: boolean | undefined;
/** if true don't try to be compatible with older yaml versions. Currently: don't quote "yes", "no" and so on, as required for YAML 1.1 (default: false) */
noCompatMode?: boolean | undefined;
/**
* if true flow sequences will be condensed, omitting the space between `key: value` or `a, b`. Eg. `'[a,b]'` or `{a:{b:c}}`.
* Can be useful when using yaml for pretty URL query params as spaces are %-encoded. (default: false).
*/
condenseFlow?: boolean | undefined;
/** strings will be quoted using this quoting style. If you specify single quotes, double quotes will still be used for non-printable characters. (default: `'`) */
quotingType?: "'" | "\"" | undefined;
/** if true, all non-key strings will be quoted even if they normally don't need to. (default: false) */
forceQuotes?: boolean | undefined;
/** callback `function (key, value)` called recursively on each key/value in source object (see `replacer` docs for `JSON.stringify`). */
replacer?: ((key: string, value: any) => any) | undefined;
}

export interface TypeConstructorOptions {
kind?: "sequence" | "scalar" | "mapping" | undefined;
resolve?: ((data: any) => boolean) | undefined;
construct?: ((data: any, type?: string) => any) | undefined;
instanceOf?: object | undefined;
predicate?: ((data: object) => boolean) | undefined;
represent?: ((data: object) => any) | { [x: string]: (data: object) => any } | undefined;
representName?: ((data: object) => any) | undefined;
defaultStyle?: string | undefined;
multi?: boolean | undefined;
styleAliases?: { [x: string]: any } | undefined;
}

export interface SchemaDefinition {
implicit?: Type[] | undefined;
explicit?: Type[] | undefined;
}

/** only strings, arrays and plain objects: http://www.yaml.org/spec/1.2/spec.html#id2802346 */
export let FAILSAFE_SCHEMA: Schema;
/** only strings, arrays and plain objects: http://www.yaml.org/spec/1.2/spec.html#id2802346 */
export let JSON_SCHEMA: Schema;
/** same as JSON_SCHEMA: http://www.yaml.org/spec/1.2/spec.html#id2804923 */
export let CORE_SCHEMA: Schema;
/** all supported YAML types */
export let DEFAULT_SCHEMA: Schema;

export interface Mark {
buffer: string;
column: number;
line: number;
name: string;
position: number;
snippet: string;
}

export class YAMLException extends Error {
constructor(reason?: string, mark?: Mark);

toString(compact?: boolean): string;

name: string;

reason: string;

message: string;

mark: Mark;
}
}

declare module "js-yaml" {
export = jsyaml;
}
29 changes: 23 additions & 6 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
"font-awesome": "^4.7.0",
"golden-layout": "^1.5.9",
"jquery": "^3.5.0",
"js-yaml": "^4.1.0",
"jstree": "^3.3.4",
"kaitai-struct": "next",
"kaitai-struct-compiler": "next",
Expand Down
6 changes: 3 additions & 3 deletions src/Playground.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import KaitaiStructCompiler = require("kaitai-struct-compiler");
import KaitaiStream = require("KaitaiStream");
import { YAML } from "yamljs";
import * as jsyaml from "js-yaml";
import { TemplateCompiler, ITemplateSchema } from "./worker/TemplateCompiler";
import { ExpressionParser } from "./worker/ExpressionLanguage/ExpressionParser";

Expand Down Expand Up @@ -31,8 +31,8 @@ async function run() {

const ksyContent = await (await fetch("template_compiler/test.ksy")).text();
const templateContent = await (await fetch("template_compiler/test.kcy.yaml")).text();
const ksy = <KsySchema.IKsyFile>YAML.parse(ksyContent, null, null, true);
const kcy = <ITemplateSchema>YAML.parse(templateContent);
const ksy = <KsySchema.IKsyFile>jsyaml.load(ksyContent, { schema: jsyaml.CORE_SCHEMA });
const kcy = <ITemplateSchema>jsyaml.load(templateContent, { schema: jsyaml.CORE_SCHEMA });

const compiledTemplate = TemplateCompiler.compileTemplateSchema(kcy);
console.log("compiledTemplate", compiledTemplate);
Expand Down
19 changes: 14 additions & 5 deletions src/v1/KaitaiServices.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { fss, IFsItem } from "./app.files";
import { performanceHelper } from "./utils/PerformanceHelper";
import KaitaiStructCompiler = require("kaitai-struct-compiler");
import * as jsyaml from "js-yaml";

class SchemaUtils {
static ksyNameToJsName(ksyName: string, isProp: boolean) {
Expand Down Expand Up @@ -62,7 +63,7 @@ class JsImporter implements IYamlImporter {
const sourceAppendix = mode === 'abs' ? 'kaitai.io' : 'local storage';
let ksyContent;
try {
ksyContent = await fss[importedFsType].get(`${loadFn}.ksy`);
ksyContent = await fss[importedFsType].get(fn);
} catch (e) {
const error = new Error(`failed to import spec ${fn} from ${sourceAppendix}${e.message ? ': ' + e.message : ''}`);

Expand All @@ -78,11 +79,11 @@ class JsImporter implements IYamlImporter {
};
throw error;
}
const ksyModel = <KsySchema.IKsyFile>YAML.parse(<string>ksyContent);
const ksyModel = <KsySchema.IKsyFile>parseYaml(<string>ksyContent, fn);
Object.assign(this.ksyTypes, SchemaUtils.collectKsyTypes(ksyModel));

// we have to modify the schema (add typesByJsName for example) before sending into the compiler, so we need a copy
const compilerSchema = <KsySchema.IKsyFile>YAML.parse(<string>ksyContent);
const compilerSchema = <KsySchema.IKsyFile>parseYaml(<string>ksyContent, fn);
return compilerSchema;
}
}
Expand All @@ -100,11 +101,11 @@ export class CompilerService {
var perfYamlParse = performanceHelper.measureAction("YAML parsing");

try {
this.ksySchema = <KsySchema.IKsyFile>YAML.parse(srcYaml);
this.ksySchema = <KsySchema.IKsyFile>parseYaml(srcYaml, srcYamlFsItem.fn);
this.ksyTypes = SchemaUtils.collectKsyTypes(this.ksySchema);

// we have to modify the schema (add typesByJsName for example) before sending into the compiler, so we need a copy
var compilerSchema = <KsySchema.IKsyFile>YAML.parse(srcYaml);
var compilerSchema = <KsySchema.IKsyFile>parseYaml(srcYaml, srcYamlFsItem.fn);
} catch (parseErr) {
return Promise.reject(new CompilationError("yaml", parseErr));
}
Expand Down Expand Up @@ -132,3 +133,11 @@ export class CompilerService {
}
}
}

function parseYaml(yamlContents: string, filename: string) {
const options = {
schema: jsyaml.CORE_SCHEMA,
filename: filename,
};
return jsyaml.load(yamlContents, options);
}
Loading

0 comments on commit 5de330d

Please sign in to comment.