diff --git a/developer/src/kmldmlc/package.json b/developer/src/kmldmlc/package.json index 295bb94f8f..2bbe69a6f5 100644 --- a/developer/src/kmldmlc/package.json +++ b/developer/src/kmldmlc/package.json @@ -20,6 +20,7 @@ }, "dependencies": { "commander": "^3.0.0", + "crc-32": "^1.2.2", "restructure": "^3.0.0", "typescript": "^4.5.4", "xml2js": "^0.4.19" diff --git a/developer/src/kmldmlc/src/keyman/kmx/kmx-builder.ts b/developer/src/kmldmlc/src/keyman/kmx/kmx-builder.ts index 6165561afa..63565f313e 100644 --- a/developer/src/kmldmlc/src/keyman/kmx/kmx-builder.ts +++ b/developer/src/kmldmlc/src/keyman/kmx/kmx-builder.ts @@ -1,4 +1,6 @@ -import KMXFile from './kmx'; +import * as r from 'restructure'; +import * as crc32 from 'crc-32'; +import KMXFile, { GROUP, KEY, STORE } from './kmx'; // These type-checking structures are here to ensure that // we match the structures from kmx.ts in the generator @@ -35,56 +37,244 @@ interface BUILDER_COMP_KEYBOARD_KMXPLUSINFO { dwKMXPlusSize: number; }; +interface BUILDER_COMP_STORE { + dwSystemID: number; + dpName: number; + dpString: number; +}; + +interface BUILDER_COMP_KEY { + Key: number; + _padding: number; + Line: number; + ShiftFlags: number; + dpOutput: number; + dpContext: number; +}; + +interface BUILDER_COMP_GROUP { + dpName: number; + dpKeyArray: number; + dpMatch: number; + dpNoMatch: number; + cxKeyArray: number; + fUsingKeys: number; +}; + + export default class KMXBuilder { file: KMXFile; - constructor(file: KMXFile) { + base_keyboard: number = 0; + base_kmxplus: number = 0; + comp_header: BUILDER_COMP_KEYBOARD; + comp_kmxplus: BUILDER_COMP_KEYBOARD_KMXPLUSINFO; + comp_stores: {base: number, store: STORE, obj: BUILDER_COMP_STORE}[] = []; + comp_groups: {base: number, group: GROUP, obj: BUILDER_COMP_GROUP, keys: {base: number, key: KEY, obj: BUILDER_COMP_KEY}[]}[] = []; + writeDebug: boolean = false; + + constructor(file: KMXFile, writeDebug: boolean) { this.file = file; + this.writeDebug = writeDebug; } - compiledHeader(): BUILDER_COMP_KEYBOARD { - return { + calculateStringOffsetAndSize(string: string, base: number, requireString: boolean = false) { + if(string.length == 0 && !requireString) { + // Zero length strings take up no space in the file, and + // are treated as a 'null string' + return [0, base]; + } + return [base, base + string.length * 2 + 2]; // include trailing zero + } + + prepareFileBuffers() { + this.base_keyboard = 0; + this.base_kmxplus = 0; + + // Header + + this.comp_header = { dwIdentifier: KMXFile.FILEID_COMPILED, dwFileVersion: KMXFile.VERSION_160, dwCheckSum: 0, KeyboardID: 0, IsRegistered: 1, version: 0, - cxStoreArray: this.file.keyboard.stores.length, - cxGroupArray: this.file.keyboard.groups.length, + cxStoreArray: 0, + cxGroupArray: 0, dpStoreArray: 0, dpGroupArray: 0, StartGroup_ANSI: 0xFFFFFFFF, StartGroup_Unicode: 0xFFFFFFFF, - dwFlags: this.file.keyboard.isKMXPlus ? KMXFile.KF_KMXPLUS : 0, + dwFlags: 0, dwHotKey: 0, dpBitmapOffset: 0, dwBitmapSize: 0 }; + + let size = KMXFile.COMP_KEYBOARD_SIZE; + + if(this.file.keyboard.isKMXPlus) { + this.base_kmxplus = size; + + this.comp_kmxplus = { + dpKMXPlus: 0, + dwKMXPlusSize: 0 + }; + + size += KMXFile.COMP_KEYBOARD_KMXPLUSINFO_SIZE; + } + + // Stores + + this.comp_header.cxStoreArray = this.file.keyboard.stores.length; + this.comp_header.dpStoreArray = this.comp_header.cxStoreArray ? size : 0; + let storeBase = size; + size += this.file.keyboard.stores.length * KMXFile.COMP_STORE_SIZE; + for(let store of this.file.keyboard.stores) { + let comp_store: BUILDER_COMP_STORE = { + dwSystemID: store.dwSystemID, + dpName: 0, + dpString: 0 + }; + this.comp_stores.push({base: storeBase, store: store, obj: comp_store}); + if(this.writeDebug /*TODO: || store.isOption*/) { + [comp_store.dpName, size] = this.calculateStringOffsetAndSize(store.dpName, size); + } + + [comp_store.dpString, size] = this.calculateStringOffsetAndSize(store.dpString, size); + storeBase += KMXFile.COMP_STORE_SIZE; + } + + // Groups + + this.comp_header.cxGroupArray = this.file.keyboard.groups.length; + this.comp_header.dpGroupArray = this.comp_header.cxGroupArray ? size : 0; + let groupBase = size; + size += this.file.keyboard.groups.length * KMXFile.COMP_GROUP_SIZE; + for(let group of this.file.keyboard.groups) { + let comp_group: BUILDER_COMP_GROUP = { + dpName: 0, + dpKeyArray: 0, + dpMatch: 0, + dpNoMatch: 0, + cxKeyArray: group.keys.length, + fUsingKeys: group.fUsingKeys ? 1 : 0 + }; + + let comp_keys: {base: number, key: KEY, obj: BUILDER_COMP_KEY}[] = []; + + this.comp_groups.push({base: groupBase, group: group, obj: comp_group, keys: comp_keys}); + + if(this.writeDebug) { + [comp_group.dpName, size] = this.calculateStringOffsetAndSize(group.dpName, size); + } + [comp_group.dpMatch, size] = this.calculateStringOffsetAndSize(group.dpMatch, size); + [comp_group.dpNoMatch, size] = this.calculateStringOffsetAndSize(group.dpNoMatch, size); + + // Keys within a group + + comp_group.dpKeyArray = group.keys.length ? size : 0; + + let keyBase = size; + size += group.keys.length * KMXFile.COMP_KEY_SIZE; + for(let key of group.keys) { + let comp_key: BUILDER_COMP_KEY = { + Key: key.Key, + _padding: 0, + Line: key.Line, + ShiftFlags: key.ShiftFlags, + dpOutput: 0, + dpContext: 0 + }; + comp_keys.push({base: keyBase, key: key, obj: comp_key}); + [comp_key.dpOutput, size] = this.calculateStringOffsetAndSize(key.dpOutput, size, true); + [comp_key.dpContext, size] = this.calculateStringOffsetAndSize(key.dpContext, size, true); + keyBase += KMXFile.COMP_KEY_SIZE; + } + + groupBase += KMXFile.COMP_GROUP_SIZE; + } + + size += this.calculateBitmapSize(); + size += this.calculateKMXPlusSize(); + + return size; } - compiledKMXPlusHeader(): BUILDER_COMP_KEYBOARD_KMXPLUSINFO { - return { - dpKMXPlus: 0, - dwKMXPlusSize: 0 - }; + calculateBitmapSize() { + // TODO + return 0; } - concat(a: Uint8Array, b: Uint8Array): Uint8Array { - let c = new Uint8Array(a.length + b.length); - c.set(a); - c.set(b, a.length); - return c; + calculateKMXPlusSize() { + // TODO + return 0; + } + + setString(file: Uint8Array, pos: number, str: string, requireString: boolean = false): void { + if(requireString && !str.length) { + // Just write zero terminator, as r.String for a zero-length string + // seems to fail. + let sbuf = r.uint16le; + file.set(sbuf.toBuffer(0), pos); + } + else if(pos && str.length) { + let sbuf = new r.String(null, 'utf16le'); // null-terminated string + file.set(sbuf.toBuffer(str), pos); + } + } + + calculateCRC32(file: Uint8Array): number { + return ~crc32.buf(file, 0); } compile(): Uint8Array { - let file: Uint8Array = this.file.COMP_KEYBOARD.toBuffer(this.compiledHeader()); + const fileSize = this.prepareFileBuffers(); + let file: Uint8Array = new Uint8Array(fileSize); + + // Write headers + + const header = this.file.COMP_KEYBOARD.toBuffer(this.comp_header); + file.set(header, this.base_keyboard); + if(this.file.keyboard.isKMXPlus) { - let kmxplus: Uint8Array = this.file.COMP_KEYBOARD_KMXPLUSINFO.toBuffer(this.compiledKMXPlusHeader()); - file = this.concat(file, kmxplus); + const kmxplus: Uint8Array = this.file.COMP_KEYBOARD_KMXPLUSINFO.toBuffer(this.comp_kmxplus); + file.set(kmxplus, this.base_kmxplus); } - // TODO once we have the final buffer, we need to do a CRC32 of the entire contents - // TODO suggest using a single buffer that grows exponentially to save masses of allocs - // header.set + + // Write store array and data + + for(let store of this.comp_stores) { + file.set(this.file.COMP_STORE.toBuffer(store.obj), store.base); + if(this.writeDebug) { + this.setString(file, store.obj.dpName, store.store.dpName); + } + this.setString(file, store.obj.dpString, store.store.dpString); + } + + // Write group array and data + + for(let group of this.comp_groups) { + file.set(this.file.COMP_GROUP.toBuffer(group.obj), group.base); + if(this.writeDebug) { + this.setString(file, group.obj.dpName, group.group.dpName); + } + this.setString(file, group.obj.dpMatch, group.group.dpMatch); + this.setString(file, group.obj.dpNoMatch, group.group.dpNoMatch); + + for(let key of group.keys) { + file.set(this.file.COMP_KEY.toBuffer(key.obj), key.base); + // for back-compat reasons, these are never NULL strings + this.setString(file, key.obj.dpContext, key.key.dpContext, true); + this.setString(file, key.obj.dpOutput, key.key.dpOutput, true); + } + } + + // Finally, calculate and write the checksum + + this.comp_header.dwCheckSum = this.calculateCRC32(file); + file.set(this.file.COMP_KEYBOARD.toBuffer(this.comp_header), this.base_keyboard); + return file; } } \ No newline at end of file diff --git a/developer/src/kmldmlc/src/keyman/kmx/kmx.ts b/developer/src/kmldmlc/src/keyman/kmx/kmx.ts index cdcfeb519a..c453582926 100644 --- a/developer/src/kmldmlc/src/keyman/kmx/kmx.ts +++ b/developer/src/kmldmlc/src/keyman/kmx/kmx.ts @@ -263,10 +263,11 @@ export default class KMXFile { public static readonly K_MODIFIERFLAG = 0x007F; public static readonly K_NOTMODIFIERFLAG = 0xFF00; // I4548 - public static readonly KEYBOARDFILEHEADER_SIZE = 64; - public static readonly KEYBOARDFILESTORE_SIZE = 12; - public static readonly KEYBOARDFILEGROUP_SIZE = 24; - public static readonly KEYBOARDFILEKEY_SIZE = 20; + public static readonly COMP_KEYBOARD_SIZE = 64; + public static readonly COMP_KEYBOARD_KMXPLUSINFO_SIZE = 8; + public static readonly COMP_STORE_SIZE = 12; + public static readonly COMP_GROUP_SIZE = 24; + public static readonly COMP_KEY_SIZE = 20; /* In-memory representation of the keyboard */ @@ -281,13 +282,13 @@ export default class KMXFile { // Binary-correct structures matching kmx_file.h this.COMP_STORE = new r.Struct({ - dwSystemID: r.uint32, - dpName: r.uint32, - dpString: r.uint32 + dwSystemID: r.uint32le, + dpName: r.uint32le, + dpString: r.uint32le }); - if(this.COMP_STORE.size() != KMXFile.KEYBOARDFILESTORE_SIZE) { - throw "COMP_STORE size is "+this.COMP_STORE.size()+" but should be "+KMXFile.KEYBOARDFILESTORE_SIZE+" bytes"; + if(this.COMP_STORE.size() != KMXFile.COMP_STORE_SIZE) { + throw "COMP_STORE size is "+this.COMP_STORE.size()+" but should be "+KMXFile.COMP_STORE_SIZE+" bytes"; } this.COMP_KEY = new r.Struct({ @@ -299,8 +300,8 @@ export default class KMXFile { dpContext: r.uint32le }); - if(this.COMP_KEY.size() != KMXFile.KEYBOARDFILEKEY_SIZE) { - throw "COMP_KEY size is "+this.COMP_KEY.size()+" but should be "+KMXFile.KEYBOARDFILEKEY_SIZE+" bytes"; + if(this.COMP_KEY.size() != KMXFile.COMP_KEY_SIZE) { + throw "COMP_KEY size is "+this.COMP_KEY.size()+" but should be "+KMXFile.COMP_KEY_SIZE+" bytes"; } this.COMP_GROUP = new r.Struct({ @@ -312,8 +313,8 @@ export default class KMXFile { fUsingKeys: r.uint32le // group(xx) [using keys] <-- specified or not }); - if(this.COMP_GROUP.size() != KMXFile.KEYBOARDFILEGROUP_SIZE) { - throw "COMP_GROUP size is "+this.COMP_GROUP.size()+" but should be "+KMXFile.KEYBOARDFILEGROUP_SIZE+" bytes"; + if(this.COMP_GROUP.size() != KMXFile.COMP_GROUP_SIZE) { + throw "COMP_GROUP size is "+this.COMP_GROUP.size()+" but should be "+KMXFile.COMP_GROUP_SIZE+" bytes"; } this.COMP_KEYBOARD_KMXPLUSINFO = new r.Struct({ @@ -321,6 +322,10 @@ export default class KMXFile { dwKMXPlusSize: r.uint32le // 0044 size in bytes of entire KMXPlus data }); + if(this.COMP_KEYBOARD_KMXPLUSINFO.size() != KMXFile.COMP_KEYBOARD_KMXPLUSINFO_SIZE) { + throw "COMP_KEYBOARD_KMXPLUSINFO size is "+this.COMP_KEYBOARD_KMXPLUSINFO.size()+" but should be "+KMXFile.COMP_KEYBOARD_KMXPLUSINFO_SIZE+" bytes"; + } + this.COMP_KEYBOARD = new r.Struct({ dwIdentifier: r.uint32le, // 0000 Keyman compiled keyboard id @@ -348,8 +353,8 @@ export default class KMXFile { dwBitmapSize: r.uint32le // 003C size in bytes of the bitmaps }); - if(this.COMP_KEYBOARD.size() != KMXFile.KEYBOARDFILEHEADER_SIZE) { - throw "COMP_KEYBOARD size is "+this.COMP_KEYBOARD.size()+" but should be "+KMXFile.KEYBOARDFILEHEADER_SIZE+" bytes"; + if(this.COMP_KEYBOARD.size() != KMXFile.COMP_KEYBOARD_SIZE) { + throw "COMP_KEYBOARD size is "+this.COMP_KEYBOARD.size()+" but should be "+KMXFile.COMP_KEYBOARD_SIZE+" bytes"; } } } \ No newline at end of file diff --git a/developer/src/kmldmlc/src/keyman/ldmlkeyboard/compiler.ts b/developer/src/kmldmlc/src/keyman/ldmlkeyboard/compiler.ts index a3989ff323..1ff93e49b1 100644 --- a/developer/src/kmldmlc/src/keyman/ldmlkeyboard/compiler.ts +++ b/developer/src/kmldmlc/src/keyman/ldmlkeyboard/compiler.ts @@ -27,7 +27,7 @@ export default class Compiler { // TODO: transform LDMLXMLSourceFile to the kmx data // Use the builder to generate the binary output file - let builder = new KMXBuilder(kmx); + let builder = new KMXBuilder(kmx, true); return builder.compile(); } } diff --git a/package-lock.json b/package-lock.json index f2a76a6cae..8fc6dbd502 100644 --- a/package-lock.json +++ b/package-lock.json @@ -448,6 +448,7 @@ "license": "MIT", "dependencies": { "commander": "^3.0.0", + "crc-32": "^1.2.2", "restructure": "^3.0.0", "typescript": "^4.5.4", "xml2js": "^0.4.19" @@ -676,8 +677,8 @@ "typescript": "^4.5.4" }, "optionalDependencies": { - "hetrodo-node-hide-console-window-napi": "git+ssh://git@github.com/keymanapp/hetrodo-node-hide-console-window-napi.git#858b23036a9963b40ad6ff3c5bacd421e5839b92", - "node-windows-trayicon": "git+ssh://git@github.com/keymanapp/node-windows-trayicon.git#1e46786082213f3edcddd5953e33f5abdc7ea05f" + "hetrodo-node-hide-console-window-napi": "keymanapp/hetrodo-node-hide-console-window-napi#keyman-15.0", + "node-windows-trayicon": "keymanapp/node-windows-trayicon#keyman-15.0" } }, "developer/src/server/node_modules/@cspotcode/source-map-support": { @@ -2680,6 +2681,17 @@ "node": ">= 0.10" } }, + "node_modules/crc-32": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", + "bin": { + "crc32": "bin/crc32.njs" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/create-require": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", @@ -7465,11 +7477,11 @@ "chalk": "^4.1.2", "copyfiles": "^2.4.1", "express": "^4.17.2", - "hetrodo-node-hide-console-window-napi": "git+ssh://git@github.com/keymanapp/hetrodo-node-hide-console-window-napi.git#858b23036a9963b40ad6ff3c5bacd421e5839b92", + "hetrodo-node-hide-console-window-napi": "keymanapp/hetrodo-node-hide-console-window-napi#keyman-15.0", "mocha": "^9.1.4", "multer": "^1.4.4", "ngrok": "^4.2.2", - "node-windows-trayicon": "git+ssh://git@github.com/keymanapp/node-windows-trayicon.git#1e46786082213f3edcddd5953e33f5abdc7ea05f", + "node-windows-trayicon": "keymanapp/node-windows-trayicon#keyman-15.0", "open": "^8.4.0", "ts-node": "^10.4.0", "tsc-watch": "^4.5.0", @@ -7723,6 +7735,7 @@ "chai": "^4.3.4", "chalk": "^2.4.2", "commander": "^3.0.0", + "crc-32": "*", "mocha": "^8.4.0", "restructure": "^3.0.0", "ts-node": "^9.1.1", @@ -9422,6 +9435,11 @@ "vary": "^1" } }, + "crc-32": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==" + }, "create-require": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz",