Setup test harness for JSON Schemas

- Create tests/ directory in repo root, containing a Yarn package for
  writing tests in TypeScript via ts-jest

- Use @hyperjump/json-schema for its draft 2020-12 support and schema
  validation logic

- Test that all schemas are valid

- Test that all example values provided by schema definitions are valid
  according to their definition

- Initialize mapping of which schema definitions extend which others

- Test that all example values provided by schema definitions that
  extend other schemas are valid according to their base schema

- Hook up tests to CI

.github/workflows/ci.yml

@@ -1,9 +1,28 @@
-name: "Docs check"
+name: Continuous integration checks
 on:
 - pull_request
 
 jobs:
-  build:
+  run-schema-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+          cache: yarn
+          cache-dependency-path: ./tests/yarn.lock
+
+      - name: Install dependencies
+        run: yarn install --frozen-lockfile
+        working-directory: ./tests
+
+      - name: Run schema tests
+        run: yarn test
+        working-directory: ./tests
+
+  build-web:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

tests/.gitignore

@@ -0,0 +1 @@
+node_modules

tests/jest.config.cjs

@@ -0,0 +1,20 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+  preset: "ts-jest",
+  testEnvironment: "node",
+  extensionsToTreatAsEsm: [".ts"],
+  moduleFileExtensions: ["ts", "js"],
+  moduleNameMapper: {
+    '^(\\.{1,2}/.*)\\.js$': '$1',
+  },
+  transform: {
+    // '^.+\\.[tj]sx?$' to process js/ts with `ts-jest`
+    // '^.+\\.m?[tj]sx?$' to process js/ts/mjs/mts with `ts-jest`
+    '^.+\\.tsx?$': [
+      'ts-jest',
+      {
+        useESM: true,
+      },
+    ],
+  },
+};

tests/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "@ethdebug/format-tests",
+  "version": "0.1.0-0",
+  "description": "Test harness for validating ethdebug/format schemas",
+  "main": "index.js",
+  "type": "module",
+  "license": "MIT",
+  "private": true,
+  "scripts": {
+    "test": "node --experimental-vm-modules $(yarn bin jest)"
+  },
+  "devDependencies": {
+    "@hyperjump/json-schema": "^1.6.7",
+    "@jest/globals": "^29.7.0",
+    "jest": "^29.7.0",
+    "json-schema-typed": "^8.0.1",
+    "ts-jest": "^29.1.1",
+    "typescript": "^5.3.3",
+    "yaml": "^2.3.4"
+  }
+}

tests/schemas/examples.test.ts

@@ -0,0 +1,69 @@
+import { expect, describe, it } from "@jest/globals";
+import { validate } from "@hyperjump/json-schema/draft-2020-12";
+
+import schemas, {
+  type JSONSchema,
+  schemaExtensions
+} from "../src/schemas.js";
+import printErrors from "../src/printErrors.js";
+
+// loads schemas into global hyperjump json schema validator
+import "../src/loadSchemas.js";
+
+describe("Examples", () => {
+  for (const [id, schema] of Object.entries(schemas)) {
+    const exampledDefinitionNames = definitionsWithExamples(schema);
+
+    if (exampledDefinitionNames.length > 0) {
+      describe(id, () => {
+        for (const name of exampledDefinitionNames) {
+          const definitionSchemaId = `${id}#/$defs/${name}`;
+
+          describe(`#/$defs/${name}`, () => {
+            const {
+              examples = []
+            } = (schema!.$defs![name] as JSONSchema);
+
+            for (const [index, example] of examples.entries()) {
+              describe(`example #${index}`, () => {
+                it(`is a valid ${name}`, async () => {
+                  const output = await validate(definitionSchemaId, example);
+                  expect(output.valid).toBe(true);
+                })
+
+                const {
+                  extends: parentSchemaIds = new Set([])
+                } = (schemaExtensions[id] || {})[name];
+
+                // NOTE this is currently not recursive (it probably should be)
+                for (const parentSchemaId of parentSchemaIds) {
+                  it(`is also a valid ${parentSchemaId}`, async () => {
+                    const output = await validate(parentSchemaId, example);
+                    expect(output.valid).toBe(true);
+                  });
+                }
+              });
+            }
+          });
+        }
+      });
+    }
+  }
+});
+
+function definitionsWithExamples(schema: JSONSchema): string[] {
+  if (!("$defs" in schema) || !schema.$defs) {
+    return [];
+  }
+
+  return Object.entries(schema.$defs)
+    .flatMap(([name, definition]) => (
+      typeof definition !== "boolean" &&
+      "examples" in definition &&
+      definition.examples &&
+      definition.examples.length > 0
+    )
+      ? [name]
+      : []
+    )
+}

tests/schemas/validity.test.ts

@@ -0,0 +1,30 @@
+import { describe, it } from "@jest/globals";
+import {
+  validate,
+  InvalidSchemaError,
+} from "@hyperjump/json-schema/draft-2020-12";
+
+import schemas from "../src/schemas.js";
+import printErrors from "../src/printErrors.js";
+
+// loads schemas into global hyperjump json schema validator
+import "../src/loadSchemas.js";
+
+describe("Valid schemas", () => {
+  for (const [id, schema] of Object.entries(schemas)) {
+    it(`should include ${id}`, async () => {
+      try {
+        await validate(id);
+      } catch (error) {
+        if (!(error instanceof InvalidSchemaError)) {
+          throw error;
+        }
+
+        throw new Error(`Invalid schema. Errors:\n${
+          printErrors(error.output)
+        }`);
+      }
+
+    });
+  }
+});

tests/src/loadSchemas.ts

@@ -0,0 +1,23 @@
+import * as fs from "fs";
+import * as path from "path";
+
+import {
+  addSchema,
+  validate,
+  setMetaSchemaOutputFormat,
+  InvalidSchemaError,
+} from "@hyperjump/json-schema/draft-2020-12";
+import * as YAML from "yaml";
+import type { JSONSchema } from "json-schema-typed/draft-2020-12"
+
+import schemas from "./schemas.js";
+
+const main = () => {
+  setMetaSchemaOutputFormat("BASIC");
+
+  for (const schema of Object.values(schemas)) {
+    addSchema(schema as any);
+  }
+}
+
+main();

tests/src/printErrors.ts

@@ -0,0 +1,17 @@
+import type { OutputUnit } from "@hyperjump/json-schema/draft-2020-12";
+
+const printErrors = (output: OutputUnit): string => output.errors!
+  .map((error) => {
+    if (!error.valid && !error.keyword.endsWith("#validate")) {
+      return `${
+        error.instanceLocation
+      } fails schema constraint ${
+        error.absoluteKeywordLocation
+      }`;
+    }
+  })
+  .filter((message): message is string => !!message)
+  .map(message => `  - ${message}`)
+  .join("\n");
+
+export default printErrors;

tests/src/schemas.ts

@@ -0,0 +1,59 @@
+import * as fs from "fs";
+import * as path from "path";
+
+import * as YAML from "yaml";
+import type { JSONSchema as JSONSchemaTyped } from "json-schema-typed/draft-2020-12"
+
+export type JSONSchema = Exclude<JSONSchemaTyped, boolean>;
+
+const schemasRoot = path.resolve("../schemas");
+
+const readSchemas = (): {
+  [id: string]: JSONSchema
+} => {
+  const schemaPaths = [
+    "type/base.schema.yaml"
+  ];
+
+  const schemas = schemaPaths
+    .map((schemaPath) => {
+      const contents = (
+        fs.readFileSync(path.join(schemasRoot, schemaPath))
+      ).toString();
+
+      const schema = YAML.parse(contents);
+
+      const { $id: id } = schema;
+
+      return {
+        [id]: schema
+      };
+    })
+    .reduce((a, b) => ({ ...a, ...b }), {});
+
+  return schemas;
+}
+
+export const schemaExtensions: {
+  [schemaId: string]: {
+    [definitionName: string]: {
+      extends: Set<string /* fully qualified base schema ID */>;
+    }
+  }
+} = {
+  "schema:ethdebug/format/type/base": {
+    "ElementaryType": {
+      extends: new Set([
+        "schema:ethdebug/format/type/base"
+      ])
+    },
+    "ComplexType": {
+      extends: new Set([
+        "schema:ethdebug/format/type/base"
+      ])
+    },
+  }
+}
+
+const schemas = readSchemas();
+export default schemas;

tests/tsconfig.json

@@ -0,0 +1,109 @@
+{
+  "compilerOptions": {
+    /* Visit https://aka.ms/tsconfig to read more about this file */
+
+    /* Projects */
+    // "incremental": true,                              /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
+    // "composite": true,                                /* Enable constraints that allow a TypeScript project to be used with project references. */
+    // "tsBuildInfoFile": "./.tsbuildinfo",              /* Specify the path to .tsbuildinfo incremental compilation file. */
+    // "disableSourceOfProjectReferenceRedirect": true,  /* Disable preferring source files instead of declaration files when referencing composite projects. */
+    // "disableSolutionSearching": true,                 /* Opt a project out of multi-project reference checking when editing. */
+    // "disableReferencedProjectLoad": true,             /* Reduce the number of projects loaded automatically by TypeScript. */
+
+    /* Language and Environment */
+    "target": "es2020",                                  /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
+    // "lib": [],                                        /* Specify a set of bundled library declaration files that describe the target runtime environment. */
+    // "jsx": "preserve",                                /* Specify what JSX code is generated. */
+    // "experimentalDecorators": true,                   /* Enable experimental support for legacy experimental decorators. */
+    // "emitDecoratorMetadata": true,                    /* Emit design-type metadata for decorated declarations in source files. */
+    // "jsxFactory": "",                                 /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */
+    // "jsxFragmentFactory": "",                         /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
+    // "jsxImportSource": "",                            /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */
+    // "reactNamespace": "",                             /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */
+    // "noLib": true,                                    /* Disable including any library files, including the default lib.d.ts. */
+    // "useDefineForClassFields": true,                  /* Emit ECMAScript-standard-compliant class fields. */
+    // "moduleDetection": "auto",                        /* Control what method is used to detect module-format JS files. */
+
+    /* Modules */
+    "module": "esnext",                                /* Specify what module code is generated. */
+    // "rootDir": "./",                                  /* Specify the root folder within your source files. */
+    "moduleResolution": "nodenext",                     /* Specify how TypeScript looks up a file from a given module specifier. */
+    // "baseUrl": "./",                                  /* Specify the base directory to resolve non-relative module names. */
+    // "paths": {},                                      /* Specify a set of entries that re-map imports to additional lookup locations. */
+    // "rootDirs": [],                                   /* Allow multiple folders to be treated as one when resolving modules. */
+    // "typeRoots": [],                                  /* Specify multiple folders that act like './node_modules/@types'. */
+    // "types": [],                                      /* Specify type package names to be included without being referenced in a source file. */
+    // "allowUmdGlobalAccess": true,                     /* Allow accessing UMD globals from modules. */
+    // "moduleSuffixes": [],                             /* List of file name suffixes to search when resolving a module. */
+    // "allowImportingTsExtensions": true,               /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */
+    // "resolvePackageJsonExports": true,                /* Use the package.json 'exports' field when resolving package imports. */
+    // "resolvePackageJsonImports": true,                /* Use the package.json 'imports' field when resolving imports. */
+    // "customConditions": [],                           /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */
+    // "resolveJsonModule": true,                        /* Enable importing .json files. */
+    // "allowArbitraryExtensions": true,                 /* Enable importing files with any extension, provided a declaration file is present. */
+    // "noResolve": true,                                /* Disallow 'import's, 'require's or '<reference>'s from expanding the number of files TypeScript should add to a project. */
+
+    /* JavaScript Support */
+    // "allowJs": true,                                  /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */
+    // "checkJs": true,                                  /* Enable error reporting in type-checked JavaScript files. */
+    // "maxNodeModuleJsDepth": 1,                        /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */
+
+    /* Emit */
+    // "declaration": true,                              /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
+    // "declarationMap": true,                           /* Create sourcemaps for d.ts files. */
+    // "emitDeclarationOnly": true,                      /* Only output d.ts files and not JavaScript files. */
+    // "sourceMap": true,                                /* Create source map files for emitted JavaScript files. */
+    // "inlineSourceMap": true,                          /* Include sourcemap files inside the emitted JavaScript. */
+    // "outFile": "./",                                  /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
+    // "outDir": "./",                                   /* Specify an output folder for all emitted files. */
+    // "removeComments": true,                           /* Disable emitting comments. */
+    // "noEmit": true,                                   /* Disable emitting files from a compilation. */
+    // "importHelpers": true,                            /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
+    // "importsNotUsedAsValues": "remove",               /* Specify emit/checking behavior for imports that are only used for types. */
+    // "downlevelIteration": true,                       /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
+    // "sourceRoot": "",                                 /* Specify the root path for debuggers to find the reference source code. */
+    // "mapRoot": "",                                    /* Specify the location where debugger should locate map files instead of generated locations. */
+    // "inlineSources": true,                            /* Include source code in the sourcemaps inside the emitted JavaScript. */
+    // "emitBOM": true,                                  /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
+    // "newLine": "crlf",                                /* Set the newline character for emitting files. */
+    // "stripInternal": true,                            /* Disable emitting declarations that have '@internal' in their JSDoc comments. */
+    // "noEmitHelpers": true,                            /* Disable generating custom helper functions like '__extends' in compiled output. */
+    // "noEmitOnError": true,                            /* Disable emitting files if any type checking errors are reported. */
+    // "preserveConstEnums": true,                       /* Disable erasing 'const enum' declarations in generated code. */
+    // "declarationDir": "./",                           /* Specify the output directory for generated declaration files. */
+    // "preserveValueImports": true,                     /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */
+
+    /* Interop Constraints */
+    // "isolatedModules": true,                          /* Ensure that each file can be safely transpiled without relying on other imports. */
+    // "verbatimModuleSyntax": true,                     /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */
+    // "allowSyntheticDefaultImports": true,             /* Allow 'import x from y' when a module doesn't have a default export. */
+    "esModuleInterop": true,                             /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */
+    // "preserveSymlinks": true,                         /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
+    "forceConsistentCasingInFileNames": true,            /* Ensure that casing is correct in imports. */
+
+    /* Type Checking */
+    "strict": true,                                      /* Enable all strict type-checking options. */
+    // "noImplicitAny": true,                            /* Enable error reporting for expressions and declarations with an implied 'any' type. */
+    // "strictNullChecks": true,                         /* When type checking, take into account 'null' and 'undefined'. */
+    // "strictFunctionTypes": true,                      /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
+    // "strictBindCallApply": true,                      /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */
+    // "strictPropertyInitialization": true,             /* Check for class properties that are declared but not set in the constructor. */
+    // "noImplicitThis": true,                           /* Enable error reporting when 'this' is given the type 'any'. */
+    // "useUnknownInCatchVariables": true,               /* Default catch clause variables as 'unknown' instead of 'any'. */
+    // "alwaysStrict": true,                             /* Ensure 'use strict' is always emitted. */
+    // "noUnusedLocals": true,                           /* Enable error reporting when local variables aren't read. */
+    // "noUnusedParameters": true,                       /* Raise an error when a function parameter isn't read. */
+    // "exactOptionalPropertyTypes": true,               /* Interpret optional property types as written, rather than adding 'undefined'. */
+    // "noImplicitReturns": true,                        /* Enable error reporting for codepaths that do not explicitly return in a function. */
+    // "noFallthroughCasesInSwitch": true,               /* Enable error reporting for fallthrough cases in switch statements. */
+    // "noUncheckedIndexedAccess": true,                 /* Add 'undefined' to a type when accessed using an index. */
+    // "noImplicitOverride": true,                       /* Ensure overriding members in derived classes are marked with an override modifier. */
+    // "noPropertyAccessFromIndexSignature": true,       /* Enforces using indexed accessors for keys declared using an indexed type. */
+    // "allowUnusedLabels": true,                        /* Disable error reporting for unused labels. */
+    // "allowUnreachableCode": true,                     /* Disable error reporting for unreachable code. */
+
+    /* Completeness */
+    // "skipDefaultLibCheck": true,                      /* Skip type checking .d.ts files that are included with TypeScript. */
+    "skipLibCheck": true                                 /* Skip type checking all .d.ts files. */
+  }
+}