Merge pull request #7109 from keymanapp/feat/developer/ldml-compiler-kmx-builder

feat(developer): LDML compiler kmx-builder 🙀
2026-09-16 04:37:41 +00:00 · 2022-08-25 14:12:56 -05:00 · 2022-08-25 14:12:56 -05:00 · f49fd013a9
commit f49fd013a9
parent 85daf7d795 dc3e4c0d78
5 changed files with 257 additions and 43 deletions
--- a/developer/src/kmldmlc/package.json
+++ b/developer/src/kmldmlc/package.json
@ -20,6 +20,7 @@
  },
  "dependencies": {
    "commander": "^3.0.0",
+    "crc-32": "^1.2.2",
    "restructure": "^3.0.0",
    "typescript": "^4.5.4",
    "xml2js": "^0.4.19"
--- a/developer/src/kmldmlc/src/keyman/kmx/kmx-builder.ts
+++ b/developer/src/kmldmlc/src/keyman/kmx/kmx-builder.ts
@ -1,4 +1,6 @@
-import KMXFile from './kmx';
+import * as r from 'restructure';
+import * as crc32 from 'crc-32';
+import KMXFile, { GROUP, KEY, STORE } from './kmx';

 // These type-checking structures are here to ensure that
 // we match the structures from kmx.ts in the generator
@ -35,56 +37,244 @@ interface BUILDER_COMP_KEYBOARD_KMXPLUSINFO {
  dwKMXPlusSize: number;
 };

+interface BUILDER_COMP_STORE {
+  dwSystemID: number;
+  dpName: number;
+  dpString: number;
+};
+
+interface BUILDER_COMP_KEY {
+  Key: number;
+  _padding: number;
+  Line: number;
+  ShiftFlags: number;
+  dpOutput: number;
+  dpContext: number;
+};
+
+interface BUILDER_COMP_GROUP {
+  dpName: number;
+  dpKeyArray: number;
+  dpMatch: number;
+  dpNoMatch: number;
+  cxKeyArray: number;
+  fUsingKeys: number;
+};
+
+
 export default class KMXBuilder {
  file: KMXFile;
-  constructor(file: KMXFile) {
+  base_keyboard: number = 0;
+  base_kmxplus: number = 0;
+  comp_header: BUILDER_COMP_KEYBOARD;
+  comp_kmxplus: BUILDER_COMP_KEYBOARD_KMXPLUSINFO;
+  comp_stores: {base: number, store: STORE, obj: BUILDER_COMP_STORE}[] = [];
+  comp_groups: {base: number, group: GROUP, obj: BUILDER_COMP_GROUP, keys: {base: number, key: KEY, obj: BUILDER_COMP_KEY}[]}[] = [];
+  writeDebug: boolean = false;
+
+  constructor(file: KMXFile, writeDebug: boolean) {
    this.file = file;
+    this.writeDebug = writeDebug;
  }

-  compiledHeader(): BUILDER_COMP_KEYBOARD {
-    return {
+  calculateStringOffsetAndSize(string: string, base: number, requireString: boolean = false) {
+    if(string.length == 0 && !requireString) {
+      // Zero length strings take up no space in the file, and
+      // are treated as a 'null string'
+      return [0, base];
+    }
+    return [base, base + string.length * 2 + 2]; // include trailing zero
+  }
+
+  prepareFileBuffers() {
+    this.base_keyboard = 0;
+    this.base_kmxplus = 0;
+
+    //  Header
+
+    this.comp_header = {
      dwIdentifier: KMXFile.FILEID_COMPILED,
      dwFileVersion: KMXFile.VERSION_160,
      dwCheckSum: 0,
      KeyboardID: 0,
      IsRegistered: 1,
      version: 0,
-      cxStoreArray: this.file.keyboard.stores.length,
-      cxGroupArray: this.file.keyboard.groups.length,
+      cxStoreArray: 0,
+      cxGroupArray: 0,
      dpStoreArray: 0,
      dpGroupArray: 0,
      StartGroup_ANSI: 0xFFFFFFFF,
      StartGroup_Unicode: 0xFFFFFFFF,
-      dwFlags: this.file.keyboard.isKMXPlus ? KMXFile.KF_KMXPLUS : 0,
+      dwFlags: 0,
      dwHotKey: 0,
      dpBitmapOffset: 0,
      dwBitmapSize: 0
    };
+
+    let size = KMXFile.COMP_KEYBOARD_SIZE;
+
+    if(this.file.keyboard.isKMXPlus) {
+      this.base_kmxplus = size;
+
+      this.comp_kmxplus = {
+        dpKMXPlus: 0,
+        dwKMXPlusSize: 0
+      };
+
+      size += KMXFile.COMP_KEYBOARD_KMXPLUSINFO_SIZE;
+    }
+
+    // Stores
+
+    this.comp_header.cxStoreArray = this.file.keyboard.stores.length;
+    this.comp_header.dpStoreArray = this.comp_header.cxStoreArray ? size : 0;
+    let storeBase = size;
+    size += this.file.keyboard.stores.length * KMXFile.COMP_STORE_SIZE;
+    for(let store of this.file.keyboard.stores) {
+      let comp_store: BUILDER_COMP_STORE = {
+        dwSystemID: store.dwSystemID,
+        dpName: 0,
+        dpString: 0
+      };
+      this.comp_stores.push({base: storeBase, store: store, obj: comp_store});
+      if(this.writeDebug /*TODO: || store.isOption*/) {
+        [comp_store.dpName, size] = this.calculateStringOffsetAndSize(store.dpName, size);
+      }
+
+      [comp_store.dpString, size] = this.calculateStringOffsetAndSize(store.dpString, size);
+      storeBase += KMXFile.COMP_STORE_SIZE;
+    }
+
+    // Groups
+
+    this.comp_header.cxGroupArray = this.file.keyboard.groups.length;
+    this.comp_header.dpGroupArray = this.comp_header.cxGroupArray ? size : 0;
+    let groupBase = size;
+    size += this.file.keyboard.groups.length * KMXFile.COMP_GROUP_SIZE;
+    for(let group of this.file.keyboard.groups) {
+      let comp_group: BUILDER_COMP_GROUP = {
+        dpName: 0,
+        dpKeyArray: 0,
+        dpMatch: 0,
+        dpNoMatch: 0,
+        cxKeyArray: group.keys.length,
+        fUsingKeys: group.fUsingKeys ? 1 : 0
+      };
+
+      let comp_keys: {base: number, key: KEY, obj: BUILDER_COMP_KEY}[] = [];
+
+      this.comp_groups.push({base: groupBase, group: group, obj: comp_group, keys: comp_keys});
+
+      if(this.writeDebug) {
+        [comp_group.dpName, size] = this.calculateStringOffsetAndSize(group.dpName, size);
+      }
+      [comp_group.dpMatch, size] = this.calculateStringOffsetAndSize(group.dpMatch, size);
+      [comp_group.dpNoMatch, size] = this.calculateStringOffsetAndSize(group.dpNoMatch, size);
+
+      // Keys within a group
+
+      comp_group.dpKeyArray = group.keys.length ? size : 0;
+
+      let keyBase = size;
+      size += group.keys.length * KMXFile.COMP_KEY_SIZE;
+      for(let key of group.keys) {
+        let comp_key: BUILDER_COMP_KEY = {
+          Key: key.Key,
+          _padding: 0,
+          Line: key.Line,
+          ShiftFlags: key.ShiftFlags,
+          dpOutput: 0,
+          dpContext: 0
+        };
+        comp_keys.push({base: keyBase, key: key, obj: comp_key});
+        [comp_key.dpOutput, size] = this.calculateStringOffsetAndSize(key.dpOutput, size, true);
+        [comp_key.dpContext, size] = this.calculateStringOffsetAndSize(key.dpContext, size, true);
+        keyBase += KMXFile.COMP_KEY_SIZE;
+      }
+
+      groupBase += KMXFile.COMP_GROUP_SIZE;
+    }
+
+    size += this.calculateBitmapSize();
+    size += this.calculateKMXPlusSize();
+
+    return size;
  }

-  compiledKMXPlusHeader(): BUILDER_COMP_KEYBOARD_KMXPLUSINFO {
-    return {
-      dpKMXPlus: 0,
-      dwKMXPlusSize: 0
-    };
+  calculateBitmapSize() {
+    // TODO
+    return 0;
  }

-  concat(a: Uint8Array, b: Uint8Array): Uint8Array {
-    let c = new Uint8Array(a.length + b.length);
-    c.set(a);
-    c.set(b, a.length);
-    return c;
+  calculateKMXPlusSize() {
+    // TODO
+    return 0;
+  }
+
+  setString(file: Uint8Array, pos: number, str: string, requireString: boolean = false): void {
+    if(requireString && !str.length) {
+      // Just write zero terminator, as r.String for a zero-length string
+      // seems to fail.
+      let sbuf = r.uint16le;
+      file.set(sbuf.toBuffer(0), pos);
+    }
+    else if(pos && str.length) {
+      let sbuf = new r.String(null, 'utf16le'); // null-terminated string
+      file.set(sbuf.toBuffer(str), pos);
+    }
+  }
+
+  calculateCRC32(file: Uint8Array): number {
+    return ~crc32.buf(file, 0);
  }

  compile(): Uint8Array {
-    let file: Uint8Array = this.file.COMP_KEYBOARD.toBuffer(this.compiledHeader());
+    const fileSize = this.prepareFileBuffers();
+    let file: Uint8Array = new Uint8Array(fileSize);
+
+    // Write headers
+
+    const header = this.file.COMP_KEYBOARD.toBuffer(this.comp_header);
+    file.set(header, this.base_keyboard);
+
    if(this.file.keyboard.isKMXPlus) {
-      let kmxplus: Uint8Array = this.file.COMP_KEYBOARD_KMXPLUSINFO.toBuffer(this.compiledKMXPlusHeader());
-      file = this.concat(file, kmxplus);
+      const kmxplus: Uint8Array = this.file.COMP_KEYBOARD_KMXPLUSINFO.toBuffer(this.comp_kmxplus);
+      file.set(kmxplus, this.base_kmxplus);
    }
-    // TODO once we have the final buffer, we need to do a CRC32 of the entire contents
-    // TODO suggest using a single buffer that grows exponentially to save masses of allocs
-    // header.set
+
+    // Write store array and data
+
+    for(let store of this.comp_stores) {
+      file.set(this.file.COMP_STORE.toBuffer(store.obj), store.base);
+      if(this.writeDebug) {
+        this.setString(file, store.obj.dpName, store.store.dpName);
+      }
+      this.setString(file, store.obj.dpString, store.store.dpString);
+    }
+
+    // Write group array and data
+
+    for(let group of this.comp_groups) {
+      file.set(this.file.COMP_GROUP.toBuffer(group.obj), group.base);
+      if(this.writeDebug) {
+        this.setString(file, group.obj.dpName, group.group.dpName);
+      }
+      this.setString(file, group.obj.dpMatch, group.group.dpMatch);
+      this.setString(file, group.obj.dpNoMatch, group.group.dpNoMatch);
+
+      for(let key of group.keys) {
+        file.set(this.file.COMP_KEY.toBuffer(key.obj), key.base);
+        // for back-compat reasons, these are never NULL strings
+        this.setString(file, key.obj.dpContext, key.key.dpContext, true);
+        this.setString(file, key.obj.dpOutput, key.key.dpOutput, true);
+      }
+    }
+
+    // Finally, calculate and write the checksum
+
+    this.comp_header.dwCheckSum = this.calculateCRC32(file);
+    file.set(this.file.COMP_KEYBOARD.toBuffer(this.comp_header), this.base_keyboard);
+
    return file;
  }
 }
--- a/developer/src/kmldmlc/src/keyman/kmx/kmx.ts
+++ b/developer/src/kmldmlc/src/keyman/kmx/kmx.ts
@ -263,10 +263,11 @@ export default class KMXFile {
  public static readonly K_MODIFIERFLAG    = 0x007F;
  public static readonly K_NOTMODIFIERFLAG = 0xFF00;   // I4548

-  public static readonly KEYBOARDFILEHEADER_SIZE = 64;
-  public static readonly KEYBOARDFILESTORE_SIZE  = 12;
-  public static readonly KEYBOARDFILEGROUP_SIZE  = 24;
-  public static readonly KEYBOARDFILEKEY_SIZE    = 20;
+  public static readonly COMP_KEYBOARD_SIZE = 64;
+  public static readonly COMP_KEYBOARD_KMXPLUSINFO_SIZE = 8;
+  public static readonly COMP_STORE_SIZE  = 12;
+  public static readonly COMP_GROUP_SIZE  = 24;
+  public static readonly COMP_KEY_SIZE    = 20;

  /* In-memory representation of the keyboard */

@ -281,13 +282,13 @@ export default class KMXFile {
    // Binary-correct structures matching kmx_file.h

    this.COMP_STORE = new r.Struct({
-      dwSystemID: r.uint32,
-      dpName: r.uint32,
-      dpString: r.uint32
+      dwSystemID: r.uint32le,
+      dpName: r.uint32le,
+      dpString: r.uint32le
    });

-    if(this.COMP_STORE.size() != KMXFile.KEYBOARDFILESTORE_SIZE) {
-      throw "COMP_STORE size is "+this.COMP_STORE.size()+" but should be "+KMXFile.KEYBOARDFILESTORE_SIZE+" bytes";
+    if(this.COMP_STORE.size() != KMXFile.COMP_STORE_SIZE) {
+      throw "COMP_STORE size is "+this.COMP_STORE.size()+" but should be "+KMXFile.COMP_STORE_SIZE+" bytes";
    }

    this.COMP_KEY = new r.Struct({
@ -299,8 +300,8 @@ export default class KMXFile {
      dpContext: r.uint32le
    });

-    if(this.COMP_KEY.size() != KMXFile.KEYBOARDFILEKEY_SIZE) {
-      throw "COMP_KEY size is "+this.COMP_KEY.size()+" but should be "+KMXFile.KEYBOARDFILEKEY_SIZE+" bytes";
+    if(this.COMP_KEY.size() != KMXFile.COMP_KEY_SIZE) {
+      throw "COMP_KEY size is "+this.COMP_KEY.size()+" but should be "+KMXFile.COMP_KEY_SIZE+" bytes";
    }

    this.COMP_GROUP = new r.Struct({
@ -312,8 +313,8 @@ export default class KMXFile {
      fUsingKeys: r.uint32le   // group(xx) [using keys] <-- specified or not
    });

-    if(this.COMP_GROUP.size() != KMXFile.KEYBOARDFILEGROUP_SIZE) {
-      throw "COMP_GROUP size is "+this.COMP_GROUP.size()+" but should be "+KMXFile.KEYBOARDFILEGROUP_SIZE+" bytes";
+    if(this.COMP_GROUP.size() != KMXFile.COMP_GROUP_SIZE) {
+      throw "COMP_GROUP size is "+this.COMP_GROUP.size()+" but should be "+KMXFile.COMP_GROUP_SIZE+" bytes";
    }

    this.COMP_KEYBOARD_KMXPLUSINFO = new r.Struct({
@ -321,6 +322,10 @@ export default class KMXFile {
      dwKMXPlusSize: r.uint32le  // 0044 size in bytes of entire KMXPlus data
    });

+    if(this.COMP_KEYBOARD_KMXPLUSINFO.size() != KMXFile.COMP_KEYBOARD_KMXPLUSINFO_SIZE) {
+      throw "COMP_KEYBOARD_KMXPLUSINFO size is "+this.COMP_KEYBOARD_KMXPLUSINFO.size()+" but should be "+KMXFile.COMP_KEYBOARD_KMXPLUSINFO_SIZE+" bytes";
+    }
+
    this.COMP_KEYBOARD = new r.Struct({
      dwIdentifier: r.uint32le,   // 0000 Keyman compiled keyboard id

@ -348,8 +353,8 @@ export default class KMXFile {
      dwBitmapSize: r.uint32le   // 003C size in bytes of the bitmaps
    });

-    if(this.COMP_KEYBOARD.size() != KMXFile.KEYBOARDFILEHEADER_SIZE) {
-      throw "COMP_KEYBOARD size is "+this.COMP_KEYBOARD.size()+" but should be "+KMXFile.KEYBOARDFILEHEADER_SIZE+" bytes";
+    if(this.COMP_KEYBOARD.size() != KMXFile.COMP_KEYBOARD_SIZE) {
+      throw "COMP_KEYBOARD size is "+this.COMP_KEYBOARD.size()+" but should be "+KMXFile.COMP_KEYBOARD_SIZE+" bytes";
    }
  }
 }
--- a/developer/src/kmldmlc/src/keyman/ldmlkeyboard/compiler.ts
+++ b/developer/src/kmldmlc/src/keyman/ldmlkeyboard/compiler.ts
@ -27,7 +27,7 @@ export default class Compiler {
    // TODO: transform LDMLXMLSourceFile to the kmx data

    // Use the builder to generate the binary output file
-    let builder = new KMXBuilder(kmx);
+    let builder = new KMXBuilder(kmx, true);
    return builder.compile();
  }
 }
--- a/package-lock.json
+++ b/package-lock.json
@ -448,6 +448,7 @@
      "license": "MIT",
      "dependencies": {
        "commander": "^3.0.0",
+        "crc-32": "^1.2.2",
        "restructure": "^3.0.0",
        "typescript": "^4.5.4",
        "xml2js": "^0.4.19"
@ -676,8 +677,8 @@
        "typescript": "^4.5.4"
      },
      "optionalDependencies": {
-        "hetrodo-node-hide-console-window-napi": "git+ssh://git@github.com/keymanapp/hetrodo-node-hide-console-window-napi.git#858b23036a9963b40ad6ff3c5bacd421e5839b92",
-        "node-windows-trayicon": "git+ssh://git@github.com/keymanapp/node-windows-trayicon.git#1e46786082213f3edcddd5953e33f5abdc7ea05f"
+        "hetrodo-node-hide-console-window-napi": "keymanapp/hetrodo-node-hide-console-window-napi#keyman-15.0",
+        "node-windows-trayicon": "keymanapp/node-windows-trayicon#keyman-15.0"
      }
    },
    "developer/src/server/node_modules/@cspotcode/source-map-support": {
@ -2680,6 +2681,17 @@
        "node": ">= 0.10"
      }
    },
+    "node_modules/crc-32": {
+      "version": "1.2.2",
+      "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz",
+      "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==",
+      "bin": {
+        "crc32": "bin/crc32.njs"
+      },
+      "engines": {
+        "node": ">=0.8"
+      }
+    },
    "node_modules/create-require": {
      "version": "1.1.1",
      "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz",
@ -7465,11 +7477,11 @@
        "chalk": "^4.1.2",
        "copyfiles": "^2.4.1",
        "express": "^4.17.2",
-        "hetrodo-node-hide-console-window-napi": "git+ssh://git@github.com/keymanapp/hetrodo-node-hide-console-window-napi.git#858b23036a9963b40ad6ff3c5bacd421e5839b92",
+        "hetrodo-node-hide-console-window-napi": "keymanapp/hetrodo-node-hide-console-window-napi#keyman-15.0",
        "mocha": "^9.1.4",
        "multer": "^1.4.4",
        "ngrok": "^4.2.2",
-        "node-windows-trayicon": "git+ssh://git@github.com/keymanapp/node-windows-trayicon.git#1e46786082213f3edcddd5953e33f5abdc7ea05f",
+        "node-windows-trayicon": "keymanapp/node-windows-trayicon#keyman-15.0",
        "open": "^8.4.0",
        "ts-node": "^10.4.0",
        "tsc-watch": "^4.5.0",
@ -7723,6 +7735,7 @@
        "chai": "^4.3.4",
        "chalk": "^2.4.2",
        "commander": "^3.0.0",
+        "crc-32": "*",
        "mocha": "^8.4.0",
        "restructure": "^3.0.0",
        "ts-node": "^9.1.1",
@ -9422,6 +9435,11 @@
        "vary": "^1"
      }
    },
+    "crc-32": {
+      "version": "1.2.2",
+      "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz",
+      "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ=="
+    },
    "create-require": {
      "version": "1.1.1",
      "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz",