epic: kmc-kmw KeymanWeb compiler in Typescript 🗜

common/include/kmx_file.h

@@ -7,6 +7,24 @@
 
 #include <km_types.h>
 
+/*
+  When we read .kmx files, they have no alignment guarantees, so we need to tell
+  the compiler to generate unaligned-safe code for accesses to COMP_ structure
+  members. Note we are assuming that COMP_KEYBOARD is aligned because it is
+  always the start of the file, so will be at the start of any buffer which will
+  automatically be aligned correctly.
+*/
+#ifdef __EMSCRIPTEN__
+typedef KMX_DWORD __attribute__((aligned(1))) KMX_DWORD_unaligned;
+typedef KMX_BOOL  __attribute__((aligned(1))) KMX_BOOL_unaligned;
+typedef KMX_WORD  __attribute__((aligned(1))) KMX_WORD_unaligned;
+#else
+// TODO: consider other platforms
+#define KMX_DWORD_unaligned KMX_DWORD
+#define KMX_BOOL_unaligned  KMX_BOOL
+#define KMX_WORD_unaligned  KMX_WORD
+#endif
+
 #ifdef KMN_KBP
 // TODO: move this to a common namespace keyman::common::kmx_file or similar in the future
 namespace km {
@@ -153,7 +171,13 @@ namespace kmx {
 
 #define TSS__KEYMAN_150_MAX  43
 
-#define TSS__MAX        43
+/* Keyman 17.0 system stores */
+
+#define TSS_DISPLAYMAP       44
+
+#define TSS__KEYMAN_170_MAX  44
+
+#define TSS__MAX        44
 
 /* wm_keyman_control_internal message control codes */
 
@@ -297,27 +321,27 @@ namespace kmx {
 #define K_NOTMODIFIERFLAG 0xFF00   // I4548
 
 struct COMP_STORE {
-  KMX_DWORD dwSystemID;
-  KMX_DWORD dpName;
-  KMX_DWORD dpString;
+  KMX_DWORD_unaligned dwSystemID;
+  KMX_DWORD_unaligned dpName;
+  KMX_DWORD_unaligned dpString;
   };
 
 struct COMP_KEY {
-  KMX_WORD Key;
-  KMX_WORD _reserved;
-  KMX_DWORD Line;
-  KMX_DWORD ShiftFlags;
-  KMX_DWORD dpOutput;
-  KMX_DWORD dpContext;
+  KMX_WORD_unaligned Key;
+  KMX_WORD_unaligned _reserved;
+  KMX_DWORD_unaligned Line;
+  KMX_DWORD_unaligned ShiftFlags;
+  KMX_DWORD_unaligned dpOutput;
+  KMX_DWORD_unaligned dpContext;
   };
 
 struct COMP_GROUP {
-  KMX_DWORD dpName;
-  KMX_DWORD dpKeyArray;   // [LPKEY] address of first item in key array
-  KMX_DWORD dpMatch;
-  KMX_DWORD dpNoMatch;
-  KMX_DWORD cxKeyArray;   // in array entries
-  KMX_BOOL  fUsingKeys;   // group(xx) [using keys] <-- specified or not
+  KMX_DWORD_unaligned dpName;
+  KMX_DWORD_unaligned dpKeyArray;   // [LPKEY] address of first item in key array
+  KMX_DWORD_unaligned dpMatch;
+  KMX_DWORD_unaligned dpNoMatch;
+  KMX_DWORD_unaligned cxKeyArray;   // in array entries
+  KMX_BOOL_unaligned  fUsingKeys;   // group(xx) [using keys] <-- specified or not
   };
 
 struct COMP_KEYBOARD {

common/schemas/displaymap/README.md

@@ -0,0 +1,77 @@
+# displaymap.schema.json
+
+This mapping file provides data for remapping the touch layout and visual
+keyboard key caps. The primary purpose of this file is to provide a pathway for
+consistent display of diacritics and other unattached marks which may be
+displayed on the keyboard by use of a special font with formatted glyphs in the
+Private Use Area, which will have consistent display across all platforms and
+not rely on platform-specific or font-specific rendering behaviors.
+
+This file can be generated by `kmc analyze osk-char-use` command, or hand
+crafted. The compiler uses only the `str` and `pua` values in the file, although
+there may be additional data in the file provided for reference purposes. The
+file should have the following structure, in this example, mapping the Unicode
+values `U+17BB U+17C7` (ុះ) to the Private Use Area code `U+F19F`:
+
+```json
+{
+  "map": [
+    {
+      "pua": "F19F",
+      "str": "ុះ",
+      "unicode": "17BB 17C7",
+      "usages": [
+        "khmer_angkor.kvks",
+        "khmer_angkor.keyman-touch-layout"
+      ]
+    },
+    ...
+  ]
+}
+```
+
+The file can be passed without modification to the [ttkbdfont.py script][2]
+(unsupported) to generate a Kbd font. The font may need some manual editing as
+insertion of dotted circle (`U+25CC`) as a base may not always be possible
+automatically, and combined marks may not render as a cluster in some scenarios.
+The open source tool [FontForge][3] is suitable for making these kinds of minor
+adjustments to the generated font.
+
+## Standard conventions for use of displayMaps
+
+In the Keyman keyboards repository, the PUA range used should start at `U+F100`.
+
+`&displayMap` JSON files should be named `Kbd<script>.json` where `<script>` is
+a [4 letter script subtag][4], for example `KbdKhmr.json`, and the corresponding
+font is named `KbdKhmr.ttf` (with face name `KbdKhmr`). These fonts are intended
+only for use in visual keyboards and touch layouts and generally will not be
+suitable for any other use. An optional suffix to a font may be included, if
+appropriate, for example, `KbdArab-Nastaliq.ttf` (face name `KbdArab-Nastaliq`).
+
+## Fields in the `map` objects
+
+* `pua`: must be a single hex value, by convention in the PUA range starting at
+  `U+F100`. Do not include the `U+` prefix.
+* `str`: contains the codepoints that will be mapped to the PUA code;
+  if `U+25CC` (◌, dotted circle) is specified in the key cap, it will be
+  stripped before matching in the `&displayMap`, so `U+25CC` should not be
+  included in `str`.
+* `unicode`: must be the unicode codepoints contained in the `str` field, as
+  hexadecimal values, without initial `U+` prefix. Used only for reference when
+  reading the file.
+* `usages`: data provided by `kmc analyze osk-char use` on which files this
+  specific string is found in. This may be an array of strings, or optionally
+  may be an array of objects with `filename` and `count` fields.
+
+See [`&displaymap` system store][1] online documentation for more information on
+how this format is used.
+
+## 2023-06-17 1.0
+
+* Initial version 1.0, describes osk usage data file format generated by kmc
+  analyze osk-char-use. Schema is kept reasonably loose for future expansion.
+
+  [1]: https://help.keyman.com/developer/language/reference/displaymap
+  [2]: https://github.com/silnrsi/palaso-python/blob/master/scripts/font/ttkbdfont.py
+  [3]: https://fontforge.org
+  [4]: https://help.keyman.com/developer/current-version/reference/bcp-47#toc-the-script-subtag

common/schemas/displaymap/displaymap.schema.json

@@ -0,0 +1,36 @@
+{
+  "$ref": "#/definitions/displayMap",
+
+  "definitions": {
+    "displayMap": {
+      "type": "object",
+      "properties": {
+        "map": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/map"
+          }
+        }
+      }
+    },
+    "map": {
+      "type": "object",
+      "properties": {
+        "pua": { "type": "string" },
+        "str": { "type": "string" },
+        "unicode": { "type": "string" },
+        "usages": { "anyOf": [
+          { "type": "array", "items": { "$ref": "#/definitions/usage" } },
+          { "type": "array", "items": { "type": "string" } }
+        ] }
+      }
+    },
+    "usage": {
+      "type": "object",
+      "properties": {
+        "filename": { "type": "string" },
+        "count": { "type": "number" }
+      }
+    }
+  }
+}

common/schemas/kpj-9.0/README.md

@@ -0,0 +1,11 @@
+# kpj-9.0.schema.json
+
+**Note:** `KeymanDeveloperProject.Options.Version` is currently implicitly
+always '1.0'. It will be required for version 2.0 and later of the format.
+
+This is a schema for for supporting legacy versions of .kpj, from Keyman
+Developer 9.0 and earlier, is now available. Note that this schema is not fully
+validating on the legacy fields, but just sufficient to pass a valid
+file.
+
+You should be using kpj.schema.json for most purposes

common/schemas/kpj-9.0/kpj-9.0.schema.json

@@ -0,0 +1,134 @@
+{
+  "title": "kpj-9.0.xsd",
+  "$schema": "http://json-schema.org/schema#",
+  "type": "object",
+  "properties": {
+    "KeymanDeveloperProject": {
+      "properties": {
+        "Options": {
+          "$ref": "#/definitions/Options"
+        },
+        "Files": {
+          "$ref": "#/definitions/Files"
+        },
+        "templatepath": { "type":"string", "$comment": "Keyman Developer 9, no longer used in modern .kpj files" },
+        "stringspath": { "type":"string", "$comment": "Keyman Developer 9, no longer used in modern .kpj files" },
+        "state": { "type": "string", "$comment": "Keyman Developer 9, no longer used in modern .kpj files" },
+        "Modules": { "oneOf": [{"type": "object"}, {"type": "string"}], "$comment": "Keyman Developer 9, no longer used in modern .kpj files" },
+        "MRU": { "type": "object", "$comment": "Keyman Developer 9, no longer used in modern .kpj files" }
+      },
+      "required": [
+      ],
+      "additionalProperties": false,
+      "type": "object"
+    }
+  },
+  "required": [
+    "KeymanDeveloperProject"
+  ],
+  "additionalProperties": false,
+  "definitions": {
+    "Options": {
+      "type": "object",
+      "properties": {
+        "BuildPath": {
+          "type": "string"
+        },
+        "SourcePath": {
+          "type": "string"
+        },
+        "CompilerWarningsAsErrors": {
+          "type": "string",
+          "pattern": "^(True|False)$"
+        },
+        "WarnDeprecatedCode": {
+          "type": "string",
+          "pattern": "^(True|False)$"
+        },
+        "CheckFilenameConventions": {
+          "type": "string",
+          "pattern": "^(True|False)$"
+        },
+        "ProjectType": {
+          "type": "string",
+          "pattern": "^(keyboard|lexicalmodel)$"
+        },
+        "Version": {
+          "type": "string",
+          "pattern": "^(1\\.0|2\\.0)$"
+        }
+      },
+      "required": [
+      ],
+      "additionalProperties": false
+    },
+    "Files": {
+      "type": "object",
+      "properties": {
+        "File": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/File"
+          }
+        }
+      },
+      "additionalProperties": false,
+      "required": [
+        "File"
+      ]
+    },
+    "File": {
+      "type": "object",
+      "properties": {
+        "ID": {
+          "type": "string"
+        },
+        "Filename": {
+          "type": "string"
+        },
+        "Filepath": {
+          "type": "string"
+        },
+        "FileVersion": {
+          "type": "string"
+        },
+        "FileType": {
+          "type": "string"
+        },
+        "Details": {
+          "$ref": "#/definitions/FileDetails"
+        },
+        "ParentFileID": {
+          "type": "string"
+        },
+        "FullPath": { "type":"string", "$comment": "Keyman Developer 9, no longer used in modern .kpj files" },
+        "IDEState": { "type":"object", "$comment": "Keyman Developer 9, no longer used in modern .kpj files" }
+      },
+      "required": [
+        "Filename"
+      ],
+      "additionalProperties": false
+    },
+    "FileDetails": {
+      "type": "object",
+      "properties": {
+        "Name": {
+          "type": "string"
+        },
+        "Copyright": {
+          "type": "string"
+        },
+        "Message": {
+          "type": "string"
+        },
+        "Version": {
+          "type": "string"
+        },
+        "Debug": { "type":"string", "$comment": "Keyman Developer 9, no longer used in modern .kpj files" }
+      },
+      "required": [
+      ],
+      "additionalProperties": false
+    }
+  }
+}

common/schemas/kpj/README.md

@@ -3,6 +3,9 @@
 **Note:** `KeymanDeveloperProject.Options.Version` is currently implicitly
 always '1.0'. It will be required for version 2.0 and later of the format.
 
+**Note:** An additional schema file, kpj-9.0.schema.json, for supporting legacy
+  versions of .kpj, from Keyman Developer 9.0 and earlier, is now available.
+
 ## 2023-02-27 2.0
 * Version 2.0 makes 'Files' optional (internally, Files/File will be ignored,
   deleted on load and populated from folder structure). Adds Options/SourcePath,

common/web/input-processor/tests/cases/languageProcessor.js

@@ -11,6 +11,7 @@ import { Mock } from '@keymanapp/keyboard-processor';
 import { LexicalModelCompiler } from '@keymanapp/kmc-model';
 import { fileURLToPath } from 'url';
 import path from 'path';
+import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers';
 
 // Required initialization setup.
 global.keyman = {}; // So that keyboard-based checks against the global `keyman` succeed.
@@ -22,9 +23,11 @@ String.kmwEnableSupplementaryPlane(false);
 // Test the KeyboardProcessor interface.
 describe('LanguageProcessor', function() {
   let worker;
+  const callbacks = new TestCompilerCallbacks();
 
   beforeEach(function() {
     worker = LMWorker.constructInstance();
+    callbacks.clear();
   });
 
   afterEach(function() {
@@ -54,7 +57,7 @@ describe('LanguageProcessor', function() {
   });
 
   describe('.predict', function() {
-    let compiler = new LexicalModelCompiler();
+    let compiler = new LexicalModelCompiler(callbacks);
     const MODEL_ID = 'example.qaa.trivial';
 
     // ES-module mode leaves out `__dirname`, so we rebuild it using other components.

common/web/keyman-version/package.json

@@ -20,7 +20,7 @@
   "license": "MIT",
   "type": "module",
   "devDependencies": {
-    "@types/node": "^18.7.13",
+    "@types/node": "^20.4.1",
     "typescript": "^4.9.5"
   }
 }

common/web/types/.gitignore

@@ -0,0 +1 @@
+src/schemas/