From 5eb6c68400ee196ab8dbda97282e24da83178285 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Fri, 5 May 2023 09:14:15 +0700 Subject: [PATCH] feat(developer): 'working' kmw compiler --- .../keyman-touch-layout-file-writer.ts | 31 +- common/web/types/src/kmx/kmx-file-reader.ts | 54 +++- common/web/types/src/kmx/kmx.ts | 18 +- common/web/types/src/kvk/visual-keyboard.ts | 1 + common/web/types/src/main.ts | 2 +- .../src/kmc-kmn/src/compiler/compiler.ts | 12 +- developer/src/kmc-kmw/build.sh | 1 + developer/src/kmc-kmw/package.json | 4 +- .../src/compiler/javascript-strings.ts | 4 +- .../src/compiler/keymanweb-key-codes.ts | 271 +++++++++++++++++ .../src/compiler/validate-layout-file.ts | 280 ++++++++++++++++++ .../src/compiler/visual-keyboard-compiler.ts | 163 ++++++++++ .../src/compiler/write-compiled-keyboard.ts | 86 +++--- .../src/kmc-kmw/test/test-compiler-manual.ts | 12 +- developer/src/kmc-kmw/test/test-compiler.ts | 26 +- developer/src/kmc-kmw/test/tsconfig.json | 3 +- developer/src/kmc-kmw/test/util.ts | 19 ++ developer/src/kmc-kmw/tsconfig.json | 9 +- developer/src/kmcmpdll/Compiler.cpp | 2 +- developer/src/kmcmplib/include/kmcmplibapi.h | 3 +- developer/src/kmcmplib/src/Compiler.cpp | 15 +- .../src/kmcmplib/src/CompilerInterfaces.cpp | 9 +- developer/src/kmcmplib/tests/api-test.cpp | 2 +- developer/src/kmcmplib/tests/kmcompxtest.cpp | 2 +- 24 files changed, 938 insertions(+), 91 deletions(-) create mode 100644 developer/src/kmc-kmw/src/compiler/keymanweb-key-codes.ts create mode 100644 developer/src/kmc-kmw/src/compiler/validate-layout-file.ts create mode 100644 developer/src/kmc-kmw/src/compiler/visual-keyboard-compiler.ts create mode 100644 developer/src/kmc-kmw/test/util.ts diff --git a/common/web/types/src/keyman-touch-layout/keyman-touch-layout-file-writer.ts b/common/web/types/src/keyman-touch-layout/keyman-touch-layout-file-writer.ts index a86ed0441f..6ee01b7f8e 100644 --- a/common/web/types/src/keyman-touch-layout/keyman-touch-layout-file-writer.ts +++ b/common/web/types/src/keyman-touch-layout/keyman-touch-layout-file-writer.ts @@ -17,12 +17,19 @@ export class TouchLayoutFileWriter { * @param source TouchLayoutFile * @returns Uint8Array, the .keyman-touch-layout file */ - write(source: TouchLayoutFile): Uint8Array { - const output = JSON.stringify(source, null, this.options?.formatted ? 2 : undefined); + public write(source: TouchLayoutFile): Uint8Array { + const output = this.toJSONString(source); const encoder = new TextEncoder(); return encoder.encode(output); } + /** + * Gets the output as a JSON string + */ + public toJSONString(source: TouchLayoutFile): string { + return JSON.stringify(source, null, this.options?.formatted ? 2 : undefined); + } + /** * Compiles the touch layout file into a KeymanWeb-compatible JSON-style * object string. In the future, this may be optimized to remove unnecessary @@ -31,7 +38,11 @@ export class TouchLayoutFileWriter { * @param source * @returns string */ - compile(source: TouchLayoutFile): string { + public compile(source: TouchLayoutFile): string { + return this.toJSONString(this.fixup(source)); + } + + public fixup(source: TouchLayoutFile): TouchLayoutFile { // Deep copy the source source = JSON.parse(JSON.stringify(source)); @@ -40,13 +51,23 @@ export class TouchLayoutFileWriter { const fixupKey = (key: TouchLayoutKey | TouchLayoutSubKey) => { if(Object.hasOwn(key, 'pad')) (key.pad as any) = key.pad.toString(); - if(Object.hasOwn(key, 'sp')) (key.sp as any) = key.sp.toString(); + if(Object.hasOwn(key, 'sp')) { + if(key.sp == 0) { + delete key.sp; + } + else { + (key.sp as any) = key.sp.toString(); + } + } if(Object.hasOwn(key, 'width')) (key.width as any) = key.width.toString(); + if(Object.hasOwn(key, 'text') && key.text === '') delete key.text; }; const fixupPlatform = (platform: TouchLayoutPlatform) => { for(let layer of platform.layer) { for(let row of layer.row) { + // this matches the old spec for touch layout files + (row.id as any) = row.id.toString(); for(let key of row.key) { fixupKey(key); if(key.sk) { @@ -79,6 +100,6 @@ export class TouchLayoutFileWriter { fixupPlatform(source.tablet); } - return JSON.stringify(source, null, this.options?.formatted ? 2 : undefined); + return source; } }; \ No newline at end of file diff --git a/common/web/types/src/kmx/kmx-file-reader.ts b/common/web/types/src/kmx/kmx-file-reader.ts index 824e56c71d..45131ade77 100644 --- a/common/web/types/src/kmx/kmx-file-reader.ts +++ b/common/web/types/src/kmx/kmx-file-reader.ts @@ -13,11 +13,42 @@ export class KmxFileReader { return this.rString.fromBuffer(source.slice(offset)); } + private processSystemStore(store: STORE, result: KEYBOARD) { + switch(store.dwSystemID) { + case KMXFile.TSS_MNEMONIC: + result.isMnemonic = store.dpString == '1'; + break; + case KMXFile.TSS_KEYBOARDVERSION: + result.keyboardVersion = store.dpString; + break; + case KMXFile.TSS_BEGIN_NEWCONTEXT: + if(store.dpString.length == 3 && store.dpString.charCodeAt(0) == 0xFFFF && store.dpString.charCodeAt(1) == KMXFile.CODE_USE) { + result.startGroup.newContext = store.dpString.charCodeAt(2) - 1; + } + else { + // TODO: error + return false; + } + break; + case KMXFile.TSS_BEGIN_POSTKEYSTROKE: + if(store.dpString.length == 3 && store.dpString.charCodeAt(0) == 0xFFFF && store.dpString.charCodeAt(1) == KMXFile.CODE_USE) { + result.startGroup.postKeystroke = store.dpString.charCodeAt(2) - 1; + } + else { + // TODO: error + return false; + } + break; + } + return true; + } + public read(source: Uint8Array): KEYBOARD { let binaryKeyboard: BUILDER_COMP_KEYBOARD; let kmx = new KMXFile(); binaryKeyboard = kmx.COMP_KEYBOARD.fromBuffer(source); if(binaryKeyboard.dwIdentifier != KMXFile.FILEID_COMPILED) { + // TODO: error return null; } @@ -25,14 +56,17 @@ export class KmxFileReader { result.fileVersion = binaryKeyboard.dwFileVersion; result.flags = binaryKeyboard.dwFlags; result.hotkey = binaryKeyboard.dwHotKey; - result.keyboardVersion = '0'; //TODO result.startGroup = { - ansi: binaryKeyboard.StartGroup_ANSI, - unicode: binaryKeyboard.StartGroup_Unicode, + ansi: binaryKeyboard.StartGroup_ANSI == 0xFFFFFFFF ? -1 : binaryKeyboard.StartGroup_ANSI, + unicode: binaryKeyboard.StartGroup_Unicode == 0xFFFFFFFF ? -1 : binaryKeyboard.StartGroup_Unicode, newContext: -1, //TODO postKeystroke: -1 // TODO } + // Informative data + result.keyboardVersion = ''; + result.isMnemonic = false; + let offset = binaryKeyboard.dpStoreArray; for(let i = 0; i < binaryKeyboard.cxStoreArray; i++) { let binaryStore = kmx.COMP_STORE.fromBuffer(source.slice(offset)); @@ -41,6 +75,11 @@ export class KmxFileReader { store.dpName = this.readString(source, binaryStore.dpName); store.dpString = this.readString(source, binaryStore.dpString); result.stores.push(store); + + if(!this.processSystemStore(store, result)) { + return null; + } + offset += KMXFile.COMP_STORE_SIZE; } @@ -73,6 +112,15 @@ export class KmxFileReader { // TODO: KMXPlusFile + // Validate startGroup offsets + let gp: keyof KEYBOARD['startGroup']; + for(gp in result.startGroup) { + if(result.startGroup[gp] < -1 || result.startGroup[gp] >= result.groups.length) { + // TODO: error + return null; + } + } + return result; } }; \ No newline at end of file diff --git a/common/web/types/src/kmx/kmx.ts b/common/web/types/src/kmx/kmx.ts index cb7af49480..17e8b3749f 100644 --- a/common/web/types/src/kmx/kmx.ts +++ b/common/web/types/src/kmx/kmx.ts @@ -9,14 +9,13 @@ import * as r from 'restructure'; export class KEYBOARD { fileVersion?: number; // dwFileVersion (TSS_FILEVERSION) - keyboardVersion?: string; // version (TSS_KEYBOARDVERSION) startGroup: { - ansi: number; - unicode: number; - newContext: number; // TSS_BEGIN_NEWCONTEXT - postKeystroke: number; // TSS_BEGIN_POSTKEYSTROKE - } = {ansi:0xFFFFFFFF, unicode:0xFFFFFFFF, newContext:0xFFFFFFFF, postKeystroke:0xFFFFFFFF}; + ansi: number; // from COMP_KEYBOARD + unicode: number; // from COMP_KEYBOARD + newContext: number; // from TSS_BEGIN_NEWCONTEXT store + postKeystroke: number; // from TSS_BEGIN_POSTKEYSTROKE store + } = {ansi:-1, unicode:-1, newContext:-1, postKeystroke:-1}; flags?: number; hotkey?: number; @@ -24,6 +23,13 @@ export class KEYBOARD { //bitmap: groups: GROUP[] = []; stores: STORE[] = []; + + // Following values are extracted from stores[] but are + // informative only + + keyboardVersion?: string; // version (TSS_KEYBOARDVERSION) + isMnemonic: boolean; // TSS_MNEMONICLAYOUT store + }; export class STORE { diff --git a/common/web/types/src/kvk/visual-keyboard.ts b/common/web/types/src/kvk/visual-keyboard.ts index c1f1d5d7e9..4965a39b52 100644 --- a/common/web/types/src/kvk/visual-keyboard.ts +++ b/common/web/types/src/kvk/visual-keyboard.ts @@ -26,6 +26,7 @@ export class VisualKeyboardFont { name?: string; size?: number; color?: number; // unused + style?: string; // TODO: figure out style vs color issues }; export { BUILDER_KVK_KEY_FLAGS as VisualKeyboardKeyFlags } from "./kvk-file.js"; diff --git a/common/web/types/src/main.ts b/common/web/types/src/main.ts index 4dad5f96ab..8b5c9f8691 100644 --- a/common/web/types/src/main.ts +++ b/common/web/types/src/main.ts @@ -7,7 +7,7 @@ export { KmxFileReader } from './kmx/kmx-file-reader.js'; export * as VisualKeyboard from './kvk/visual-keyboard.js'; export { default as KMXPlusBuilder} from './kmx/kmx-plus-builder/kmx-plus-builder.js'; export { default as KvkFileReader } from './kvk/kvk-file-reader.js'; -export { default as KvksFileReader } from './kvk/kvks-file-reader.js'; +export { default as KvksFileReader, KVKSParseError } from './kvk/kvks-file-reader.js'; export { default as KvkFileWriter } from './kvk/kvk-file-writer.js'; export * as KvkFile from './kvk/kvk-file.js'; export * as KvksFile from './kvk/kvk-file.js'; diff --git a/developer/src/kmc-kmn/src/compiler/compiler.ts b/developer/src/kmc-kmn/src/compiler/compiler.ts index c112b4caf7..21ed62ba5f 100644 --- a/developer/src/kmc-kmn/src/compiler/compiler.ts +++ b/developer/src/kmc-kmn/src/compiler/compiler.ts @@ -31,13 +31,15 @@ export interface CompilerOptions { saveDebug?: boolean; compilerWarningsAsErrors?: boolean; warnDeprecatedCode?: boolean; + target?: 'kmx' | 'js'; }; const baseOptions: CompilerOptions = { shouldAddCompilerVersion: true, saveDebug: true, compilerWarningsAsErrors: false, - warnDeprecatedCode: true + warnDeprecatedCode: true, + target: 'kmx' }; /** @@ -63,7 +65,7 @@ export class Compiler { if(!this.wasmModule) { this.wasmModule = await loadWasmHost(); this.compileKeyboardFile = this.wasmModule.cwrap('kmcmp_Wasm_CompileKeyboardFile', 'boolean', ['string', 'string', - 'number', 'number', 'number', 'string']); + 'number', 'number', 'number', 'string', 'number']); this.setCompilerOptions = this.wasmModule.cwrap('kmcmp_Wasm_SetCompilerOptions', 'boolean', ['number']); } return this.compileKeyboardFile !== undefined && this.setCompilerOptions !== undefined; @@ -91,6 +93,9 @@ export class Compiler { } private runCompiler(infile: string, outfile: string, options: CompilerOptions): boolean { + const CKF_KEYMAN = 0; + const CKF_KEYMANWEB = 1; + try { if(!this.setCompilerOptions(options.shouldAddCompilerVersion)) { this.callbacks.reportMessage(CompilerMessages.Fatal_UnableToSetCompilerOptions()); @@ -101,7 +106,8 @@ export class Compiler { options.saveDebug ? 1 : 0, options.compilerWarningsAsErrors ? 1 : 0, options.warnDeprecatedCode ? 1 : 0, - this.callbackName); + this.callbackName, + options.target == 'js' ? CKF_KEYMANWEB : CKF_KEYMAN); } catch(e) { this.callbacks.reportMessage(CompilerMessages.Fatal_UnexpectedException({e:e})); return false; diff --git a/developer/src/kmc-kmw/build.sh b/developer/src/kmc-kmw/build.sh index 684f5f7911..be180cc725 100755 --- a/developer/src/kmc-kmw/build.sh +++ b/developer/src/kmc-kmw/build.sh @@ -19,6 +19,7 @@ cd "$THIS_SCRIPT_PATH" builder_describe "Build Keyman kmc KMW Keyboard Compiler module" \ "@/common/web/keyman-version" \ "@/common/web/types" \ + "@/developer/src/kmc-kmn" \ "configure" \ "build" \ "clean" \ diff --git a/developer/src/kmc-kmw/package.json b/developer/src/kmc-kmw/package.json index e00a3c7e31..fc4448b87e 100644 --- a/developer/src/kmc-kmw/package.json +++ b/developer/src/kmc-kmw/package.json @@ -24,6 +24,7 @@ }, "dependencies": { "@keymanapp/common-types": "*", + "@keymanapp/kmc-kmn": "*", "ajv": "^8.11.0", "restructure": "git+https://github.com/keymanapp/dependency-restructure.git#49d129cf0916d082a7278bb09296fb89cecfcc50", "semver": "^7.3.7", @@ -40,8 +41,7 @@ "chalk": "^2.4.2", "mocha": "^8.4.0", "ts-node": "^9.1.1", - "typescript": "^4.9.5", - "@keymanapp/kmc-kmn": "*" + "typescript": "^4.9.5" }, "mocha": { "spec": "build/test/**/test-*.js", diff --git a/developer/src/kmc-kmw/src/compiler/javascript-strings.ts b/developer/src/kmc-kmw/src/compiler/javascript-strings.ts index b4ae08829d..2504781b9b 100644 --- a/developer/src/kmc-kmw/src/compiler/javascript-strings.ts +++ b/developer/src/kmc-kmw/src/compiler/javascript-strings.ts @@ -324,7 +324,7 @@ export function JavaScript_ShiftAsString(fkp: KMX.KEY, FMnemonic: boolean): stri return ' '+FormatModifierAsBitflags(JavaScript_Shift(fkp, FMnemonic)); } -const VKeyNames = [ // from vkeys.h +export const VKeyNames = [ // from vkeys.h // Key Codes "K_?00", // &H0 "K_LBUTTON", // &H1 @@ -671,7 +671,7 @@ function FormatKeyForErrorMessage(fkp: KMX.KEY, FMnemonic: boolean): string { return result; } -function JavaScript_Key(fkp: KMX.KEY, FMnemonic: boolean): number { +export function JavaScript_Key(fkp: KMX.KEY, FMnemonic: boolean): number { let Result: number; if(!FMnemonic) { if(fkp.ShiftFlags & KMX.KMXFile.ISVIRTUALKEY) { diff --git a/developer/src/kmc-kmw/src/compiler/keymanweb-key-codes.ts b/developer/src/kmc-kmw/src/compiler/keymanweb-key-codes.ts new file mode 100644 index 0000000000..98e609d747 --- /dev/null +++ b/developer/src/kmc-kmw/src/compiler/keymanweb-key-codes.ts @@ -0,0 +1,271 @@ +export const + CKeymanWebKeyCodes: number[] = [ + 0xFF, // L"K_?00", // &H0 + 0xFF, // L"K_LBUTTON", // &H1 + 0xFF, // L"K_RBUTTON", // &H2 + 0xFF, // L"K_CANCEL", // &H3 + 0xFF, // L"K_MBUTTON", // &H4 + 0xFF, // L"K_?05", // &H5 + 0xFF, // L"K_?06", // &H6 + 0xFF, // L"K_?07", // &H7 + 0xFF, // L"K_BKSP", // &H8 + 0xFF, // L"K_TAB", // &H9 + 0xFF, // L"K_?0A", // &HA + 0xFF, // L"K_?0B", // &HB + 0xFF, // L"K_KP5", // &HC + 0xFF, // L"K_ENTER", // &HD + 0xFF, // L"K_?0E", // &HE + 0xFF, // L"K_?0F", // &HF + 0xFF, // L"K_SHIFT", // &H10 + 0xFF, // L"K_CONTROL", // &H11 + 0xFF, // L"K_ALT", // &H12 + 0xFF, // L"K_PAUSE", // &H13 + 0xFF, // L"K_CAPS", // &H14 + 0xFF, // L"K_KANJI?15", // &H15 + 0xFF, // L"K_KANJI?16", // &H16 + 0xFF, // L"K_KANJI?17", // &H17 + 0xFF, // L"K_KANJI?18", // &H18 + 0xFF, // L"K_KANJI?19", // &H19 + 0xFF, // L"K_?1A", // &H1A + 0xFF, // L"K_ESC", // &H1B + 0xFF, // L"K_KANJI?1C", // &H1C + 0xFF, // L"K_KANJI?1D", // &H1D + 0xFF, // L"K_KANJI?1E", // &H1E + 0xFF, // L"K_KANJI?1F", // &H1F + 0x40, // L"K_SPACE", // &H20 + 0xFF, // L"K_PGUP", // &H21 + 0xFF, // L"K_PGDN", // &H22 + 0xFF, // L"K_END", // &H23 + 0xFF, // L"K_HOME", // &H24 + 0xFF, // L"K_LEFT", // &H25 + 0xFF, // L"K_UP", // &H26 + 0xFF, // L"K_RIGHT", // &H27 + 0xFF, // L"K_DOWN", // &H28 + 0xFF, // L"K_SEL", // &H29 + 0xFF, // L"K_PRINT", // &H2A + 0xFF, // L"K_EXEC", // &H2B + 0xFF, // L"K_PRTSCN", // &H2C + 0xFF, // L"K_INS", // &H2D + 0xFF, // L"K_DEL", // &H2E + 0xFF, // L"K_HELP", // &H2F + 0x0A, // L"K_0", // &H30 + 0x01, // L"K_1", // &H31 + 0x02, // L"K_2", // &H32 + 0x03, // L"K_3", // &H33 + 0x04, // L"K_4", // &H34 + 0x05, // L"K_5", // &H35 + 0x06, // L"K_6", // &H36 + 0x07, // L"K_7", // &H37 + 0x08, // L"K_8", // &H38 + 0x09, // L"K_9", // &H39 + 0xFF, // L"K_?3A", // &H3A + 0xFF, // L"K_?3B", // &H3B + 0xFF, // L"K_?3C", // &H3C + 0xFF, // L"K_?3D", // &H3D + 0xFF, // L"K_?3E", // &H3E + 0xFF, // L"K_?3F", // &H3F + 0xFF, // L"K_?40", // &H40 + + 0x20, // L"K_A", // &H41 + 0x35, // L"K_B", // &H42 + 0x33, // L"K_C", // &H43 + 0x22, // L"K_D", // &H44 + 0x12, // L"K_E", // &H45 + 0x23, // L"K_F", // &H46 + 0x24, // L"K_G", // &H47 + 0x25, // L"K_H", // &H48 + 0x17, // L"K_I", // &H49 + 0x26, // L"K_J", // &H4A + 0x27, // L"K_K", // &H4B + 0x28, // L"K_L", // &H4C + 0x37, // L"K_M", // &H4D + 0x36, // L"K_N", // &H4E + 0x18, // L"K_O", // &H4F + 0x19, // L"K_P", // &H50 + 0x10, // L"K_Q", // &H51 + 0x13, // L"K_R", // &H52 + 0x21, // L"K_S", // &H53 + 0x14, // L"K_T", // &H54 + 0x16, // L"K_U", // &H55 + 0x34, // L"K_V", // &H56 + 0x11, // L"K_W", // &H57 + 0x32, // L"K_X", // &H58 + 0x15, // L"K_Y", // &H59 + 0x31, // L"K_Z", // &H5A + 0xFF, // L"K_?5B", // &H5B + 0xFF, // L"K_?5C", // &H5C + 0xFF, // L"K_?5D", // &H5D + 0xFF, // L"K_?5E", // &H5E + 0xFF, // L"K_?5F", // &H5F + 0xFF, // L"K_NP0", // &H60 + 0xFF, // L"K_NP1", // &H61 + 0xFF, // L"K_NP2", // &H62 + 0xFF, // L"K_NP3", // &H63 + 0xFF, // L"K_NP4", // &H64 + 0xFF, // L"K_NP5", // &H65 + 0xFF, // L"K_NP6", // &H66 + 0xFF, // L"K_NP7", // &H67 + 0xFF, // L"K_NP8", // &H68 + 0xFF, // L"K_NP9", // &H69 + 0xFF, // L"K_NPSTAR", // &H6A + 0xFF, // L"K_NPPLUS", // &H6B + 0xFF, // L"K_SEPARATOR", // &H6C + 0xFF, // L"K_NPMINUS", // &H6D + 0xFF, // L"K_NPDOT", // &H6E + 0xFF, // L"K_NPSLASH", // &H6F + 0xFF, // L"K_F1", // &H70 + 0xFF, // L"K_F2", // &H71 + 0xFF, // L"K_F3", // &H72 + 0xFF, // L"K_F4", // &H73 + 0xFF, // L"K_F5", // &H74 + 0xFF, // L"K_F6", // &H75 + 0xFF, // L"K_F7", // &H76 + 0xFF, // L"K_F8", // &H77 + 0xFF, // L"K_F9", // &H78 + 0xFF, // L"K_F10", // &H79 + 0xFF, // L"K_F11", // &H7A + 0xFF, // L"K_F12", // &H7B + 0xFF, // L"K_F13", // &H7C + 0xFF, // L"K_F14", // &H7D + 0xFF, // L"K_F15", // &H7E + 0xFF, // L"K_F16", // &H7F + 0xFF, // L"K_F17", // &H80 + 0xFF, // L"K_F18", // &H81 + 0xFF, // L"K_F19", // &H82 + 0xFF, // L"K_F20", // &H83 + 0xFF, // L"K_F21", // &H84 + 0xFF, // L"K_F22", // &H85 + 0xFF, // L"K_F23", // &H86 + 0xFF, // L"K_F24", // &H87 + + 0xFF, // L"K_?88", // &H88 + 0xFF, // L"K_?89", // &H89 + 0xFF, // L"K_?8A", // &H8A + 0xFF, // L"K_?8B", // &H8B + 0xFF, // L"K_?8C", // &H8C + 0xFF, // L"K_?8D", // &H8D + 0xFF, // L"K_?8E", // &H8E + 0xFF, // L"K_?8F", // &H8F + + 0xFF, // L"K_NUMLOCK", // &H90 + 0xFF, // L"K_SCROLL", // &H91 + + 0xFF, // L"K_?92", // &H92 + 0xFF, // L"K_?93", // &H93 + 0xFF, // L"K_?94", // &H94 + 0xFF, // L"K_?95", // &H95 + 0xFF, // L"K_?96", // &H96 + 0xFF, // L"K_?97", // &H97 + 0xFF, // L"K_?98", // &H98 + 0xFF, // L"K_?99", // &H99 + 0xFF, // L"K_?9A", // &H9A + 0xFF, // L"K_?9B", // &H9B + 0xFF, // L"K_?9C", // &H9C + 0xFF, // L"K_?9D", // &H9D + 0xFF, // L"K_?9E", // &H9E + 0xFF, // L"K_?9F", // &H9F + 0xFF, // L"K_?A0", // &HA0 + 0xFF, // L"K_?A1", // &HA1 + 0xFF, // L"K_?A2", // &HA2 + 0xFF, // L"K_?A3", // &HA3 + 0xFF, // L"K_?A4", // &HA4 + 0xFF, // L"K_?A5", // &HA5 + 0xFF, // L"K_?A6", // &HA6 + 0xFF, // L"K_?A7", // &HA7 + 0xFF, // L"K_?A8", // &HA8 + 0xFF, // L"K_?A9", // &HA9 + 0xFF, // L"K_?AA", // &HAA + 0xFF, // L"K_?AB", // &HAB + 0xFF, // L"K_?AC", // &HAC + 0xFF, // L"K_?AD", // &HAD + 0xFF, // L"K_?AE", // &HAE + 0xFF, // L"K_?AF", // &HAF + 0xFF, // L"K_?B0", // &HB0 + 0xFF, // L"K_?B1", // &HB1 + 0xFF, // L"K_?B2", // &HB2 + 0xFF, // L"K_?B3", // &HB3 + 0xFF, // L"K_?B4", // &HB4 + 0xFF, // L"K_?B5", // &HB5 + 0xFF, // L"K_?B6", // &HB6 + 0xFF, // L"K_?B7", // &HB7 + 0xFF, // L"K_?B8", // &HB8 + 0xFF, // L"K_?B9", // &HB9 + + 0x29, // L"K_COLON", // &HBA + 0x0C, // L"K_EQUAL", // &HBB + 0x38, // L"K_COMMA", // &HBC + 0x0B, // L"K_HYPHEN", // &HBD + 0x39, // L"K_PERIOD", // &HBE + 0x3A, // L"K_SLASH", // &HBF + 0x00, // L"K_BKQUOTE", // &HC0 + + 0x00, // L"K_?C1", // &HC1 + 0x00, // L"K_?C2", // &HC2 + 0x00, // L"K_?C3", // &HC3 + 0x00, // L"K_?C4", // &HC4 + 0x00, // L"K_?C5", // &HC5 + 0x00, // L"K_?C6", // &HC6 + 0x00, // L"K_?C7", // &HC7 + 0x00, // L"K_?C8", // &HC8 + 0x00, // L"K_?C9", // &HC9 + 0x00, // L"K_?CA", // &HCA + 0x00, // L"K_?CB", // &HCB + 0x00, // L"K_?CC", // &HCC + 0x00, // L"K_?CD", // &HCD + 0x00, // L"K_?CE", // &HCE + 0x00, // L"K_?CF", // &HCF + 0x00, // L"K_?D0", // &HD0 + 0x00, // L"K_?D1", // &HD1 + 0x00, // L"K_?D2", // &HD2 + 0x00, // L"K_?D3", // &HD3 + 0x00, // L"K_?D4", // &HD4 + 0x00, // L"K_?D5", // &HD5 + 0x00, // L"K_?D6", // &HD6 + 0x00, // L"K_?D7", // &HD7 + 0x00, // L"K_?D8", // &HD8 + 0x00, // L"K_?D9", // &HD9 + 0x00, // L"K_?DA", // &HDA + + 0x1A, // L"K_LBRKT", // &HDB + 0x1C, // L"K_BKSLASH", // &HDC + 0x1B, // L"K_RBRKT", // &HDD + 0x2A, // L"K_QUOTE", // &HDE + 0x00, // L"K_oDF", // &HDF + 0x00, // L"K_oE0", // &HE0 + 0x00, // L"K_oE1", // &HE1 + 0x30, // L"K_oE2", // &HE2 + 0x00, // L"K_oE3", // &HE3 + 0x00, // L"K_oE4", // &HE4 + + 0x00, // L"K_?E5", // &HE5 + + 0x00, // L"K_oE6", // &HE6 + + 0x00, // L"K_?E7", // &HE7 + 0x00, // L"K_?E8", // &HE8 + + 0x00, // L"K_oE9", // &HE9 + 0x00, // L"K_oEA", // &HEA + 0x00, // L"K_oEB", // &HEB + 0x00, // L"K_oEC", // &HEC + 0x00, // L"K_oED", // &HED + 0x00, // L"K_oEE", // &HEE + 0x00, // L"K_oEF", // &HEF + 0x00, // L"K_oF0", // &HF0 + 0x00, // L"K_oF1", // &HF1 + 0x00, // L"K_oF2", // &HF2 + 0x00, // L"K_oF3", // &HF3 + 0x00, // L"K_oF4", // &HF4 + 0x00, // L"K_oF5", // &HF5 + + 0x00, // L"K_?F6", // &HF6 + 0x00, // L"K_?F7", // &HF7 + 0x00, // L"K_?F8", // &HF8 + 0x00, // L"K_?F9", // &HF9 + 0x00, // L"K_?FA", // &HFA + 0x00, // L"K_?FB", // &HFB + 0x00, // L"K_?FC", // &HFC + 0x00, // L"K_?FD", // &HFD + 0x00, // L"K_?FE", // &HFE + 0x00 // L"K_?FF" // &HFF + ]; \ No newline at end of file diff --git a/developer/src/kmc-kmw/src/compiler/validate-layout-file.ts b/developer/src/kmc-kmw/src/compiler/validate-layout-file.ts new file mode 100644 index 0000000000..30d6f660d9 --- /dev/null +++ b/developer/src/kmc-kmw/src/compiler/validate-layout-file.ts @@ -0,0 +1,280 @@ +import { KMX, TouchLayout, TouchLayoutFileReader, TouchLayoutFileWriter } from "@keymanapp/common-types"; +import { callbacks, IsKeyboardVersion14OrLater, IsKeyboardVersion15OrLater } from "./compiler-globals.js"; +import { JavaScript_Key, VKeyNames } from "./javascript-strings.js"; + + +interface VLFOutput { + output: string; + result: boolean; +}; + +function IsValidUnicodeValue(ch: number): boolean { // I4198 + return ((ch >= 0x0020) && (ch <= 0x007F)) || + ((ch >= 0x00A0) && (ch <= 0x10FFFF)); +} + +enum TKeyIdType { Key_Invalid, Key_Constant, Key_Touch, Key_Unicode, Key_Unicode_Multi }; // I4142 + +function GetKeyIdUnicodeType(value: string): TKeyIdType { + let values = value.split('_'); + for(let v of values) { + if(!IsValidUnicodeValue(parseInt(v,16))) { + return TKeyIdType.Key_Invalid; + } + } + if(values.length > 1) { + return TKeyIdType.Key_Unicode_Multi; + } + return TKeyIdType.Key_Unicode; +} + +function KeyIdType(FId: string): TKeyIdType { // I4142 + FId = FId.toUpperCase(); + switch(FId.charAt(0)) { + case 'T': + return TKeyIdType.Key_Touch; + case 'U': + if(FId.startsWith('U_')) { + return GetKeyIdUnicodeType(FId.substring(2)); + } + default: + // Note: can't use indexOf because some VKeyNames are mixed case, e.g. K_oE2 + if(VKeyNames.find(key => key.toUpperCase() == FId)) { + return TKeyIdType.Key_Constant; + } + } + return TKeyIdType.Key_Invalid; +} + +enum TRequiredKey { K_LOPT, K_BKSP, K_ENTER }; // I4447 + +const + CRequiredKeys: TRequiredKey[] = [TRequiredKey.K_LOPT, TRequiredKey.K_BKSP, TRequiredKey.K_ENTER]; // I4447 + + // See also builder.js: specialCharacters; web/source/osk/oskKey.ts: specialCharacters +const + CSpecialText10: string = + '*Shift*\0*Enter*\0*Tab*\0*BkSp*\0*Menu*\0*Hide*\0*Alt*\0*Ctrl*\0*Caps*\0'+ + '*ABC*\0*abc*\0*123*\0*Symbol*\0*Currency*\0*Shifted*\0*AltGr*\0*TabLeft*', + + // these names were added in Keyman 14 + CSpecialText14: string = + '*LTREnter*\0*LTRBkSp*\0*RTLEnter*\0*RTLBkSp*\0*ShiftLock*\0*ShiftedLock*\0*ZWNJ*\0*ZWNJiOS*\0*ZWNJAndroid*', + CSpecialText14ZWNJ: string = + '*ZWNJ*\0*ZWNJiOS*\0*ZWNJAndroid*', + + CSpecialText14Map: string[][] = [ + ['*LTREnter*', '*Enter*'], + ['*LTRBkSp*', '*BkSp*'], + ['*RTLEnter*', '*Enter*'], + ['*RTLBkSp*', '*BkSp*'], + ['*ShiftLock*', '*Shift*'], + ['*ShiftedLock*', '*Shifted*'], + ['*ZWNJ*', '<|>'], + ['*ZWNJiOS*', '<|>'], + ['*ZWNJAndroid*', '<|>'] + ]; + +// TODO lifecycle + +function CheckKey(FPlatform: TouchLayout.TouchLayoutPlatform, + FId: string, FText: string, FNextLayer: string, FKeyType: TouchLayout.TouchLayoutKeySp, + FRequiredKeys: TRequiredKey[], FDictionary: string[]) { // I4119 + + // + // Check that each touch layer has K_LOPT, [K_ROPT,] K_BKSP, K_ENTER + // + + for(let key of CRequiredKeys) { + if(TRequiredKey[key].toLowerCase() == FId.toLowerCase()) { + FRequiredKeys.push(key); + break; + } + } + + // + // Check that each layer referenced exists + // + + if(typeof FNextLayer == 'string' && FNextLayer.length > 0) { + if(FPlatform.layer.find(l => l.id.toLowerCase() == FNextLayer.toLowerCase()) == undefined) { + // TODO: callbacks.reportMessage() ReportError(0, CWARN_TouchLayoutMissingLayer, 'Key "'+FId+'" on platform "'+FPlatform.Name+'", layer "'+FLayer.Id+'", platform "'+FPlatform.Name+'", references a missing layer "'+FNextLayer+'".'); + } + } + + // + // Check that the key has a valid id // I4142 + // + + if(FId.trim() == '') { + if(!(FKeyType in [TouchLayout.TouchLayoutKeySp.blank, TouchLayout.TouchLayoutKeySp.spacer]) && FNextLayer == '') { + // TODO: ReportError(0, CWARN_TouchLayoutUnidentifiedKey, 'A key on layer "'+FLayer.Id+'" has no identifier.'); + } + return; + } + + let FValid = KeyIdType(FId); + + if(FValid == TKeyIdType.Key_Invalid) { + // TODO: ReportError(0, CERR_TouchLayoutInvalidIdentifier, 'Key "'+FId+'" on "'+FPlatform.Name+'", layer "'+FLayer.Id+'" has an invalid identifier.'); + } + else if (FValid == TKeyIdType.Key_Unicode_Multi && !IsKeyboardVersion15OrLater()) { + // TODO: ReportError(0, CERR_TouchLayoutInvalidIdentifier, 'Key "'+FId+'" on "'+FPlatform.Name+'", layer "'+FLayer.Id+'" has a multi-part identifier which requires version 15.0 or newer.'); + } + + // + // Check that each custom key code has at least *a* rule associated with it + // + + if (FValid == TKeyIdType.Key_Touch && FNextLayer == '' && FKeyType in [TouchLayout.TouchLayoutKeySp.normal, TouchLayout.TouchLayoutKeySp.deadkey]) { + // Search for the key in the key dictionary - ignore K_LOPT, K_ROPT... + if(FDictionary.indexOf(FId) < 0) { + // TODO: ReportError(0, CWARN_TouchLayoutCustomKeyNotDefined, 'Key "'+FId+'" on layer "'+FLayer.Id+'", platform "'+FPlatform.Name+'", is a custom key but has no corresponding rule in the source.'); + } + } + + // + // Check that if the key has a *special* label, it is available in the target version + // + if(FText.startsWith('*') && FText.endsWith('*') && FText.length > 2) { + // Keyman versions before 14 do not support '*special*' labels on non-special keys. + // ZWNJ use, however, is safe because it will be transformed in function + // TransformSpecialKeys14 to '<|>', which does not require the custom OSK font. + if((CSpecialText10.includes(FText) || CSpecialText14.includes(FText)) && + !CSpecialText14ZWNJ.includes(FText) && + !IsKeyboardVersion14OrLater() && + !(FKeyType in [TouchLayout.TouchLayoutKeySp.special, TouchLayout.TouchLayoutKeySp.specialActive])) { + // TODO: ReportError(0, CWARN_TouchLayoutSpecialLabelOnNormalKey, + // Format('Key "%s" on layout "%s", platform "%s" does not have the key type "Special" or "Special (active)" but has the label "%s". This feature is only supported in Keyman 14 or later', [ + // FId, FLayer.Id, FPlatform.Name, FText + // ])); + } + } +} + +function CheckDictionaryKeyValidity(fk: KMX.KEYBOARD, FDictionary: string[]) { // I4142 + + // TODO: O(eeek) performance here + + for(let i = 0; i < FDictionary.length; i++) { + if(FDictionary[i] == '') { + continue; + } + + if(KeyIdType(FDictionary[i]) in [TKeyIdType.Key_Invalid, TKeyIdType.Key_Constant]) { + for(let fgp of fk.groups) { + if(fgp.fUsingKeys) { + for(let fkp of fgp.keys) { + if(JavaScript_Key(fkp, fk.isMnemonic) == i+256) { + // TODO: ReportError(fkp.Line, CERR_InvalidKeyCode, 'Invalid key identifier "'+FDictionary[i]+'"'); + } + } + } + } + } + } +} + +function TransformSpecialKeys14(FDebug: boolean, sLayoutFile: string): string { + // Rewrite Special key labels that are only supported in Keyman 14+ + // This code is a little ugly but effective. + if(!IsKeyboardVersion14OrLater()) { + for(let i = 0; i < CSpecialText14Map.length; i++) { + // Assumes the JSON output format will not change + if(FDebug) { + sLayoutFile = sLayoutFile.replace('"text": "'+CSpecialText14Map[i][0]+'"', '"text": this._v>13 ? "'+CSpecialText14Map[i][0]+'" : "'+CSpecialText14Map[i][1]+'"'); + } else { + sLayoutFile = sLayoutFile.replace('"text":"'+CSpecialText14Map[i][0]+'"', '"text":this._v>13?"'+CSpecialText14Map[i][0]+'":"'+CSpecialText14Map[i][1]+'"'); + } + } + } + return sLayoutFile; +} + +export function ValidateLayoutFile(fk: KMX.KEYBOARD, FDebug: boolean, sLayoutFile: string, sVKDictionary: string): VLFOutput { // I4060 // I4139 + +/* +var + FPlatform: TTouchLayoutPlatform; + FLayer: TTouchLayoutLayer; + FRow: TTouchLayoutRow; + FKey: TTouchLayoutKey; + FSubKey: TTouchLayoutSubKey; + FRequiredKeys: set of TRequiredKey; + FDictionary: TStringList; + FDirection: TTouchLayoutFlickDirection; + +*/ + + let FDictionary: string[] = sVKDictionary.split(/\s+/); + + CheckDictionaryKeyValidity(fk, FDictionary); // I4142 + + let reader = new TouchLayoutFileReader(); + let data = reader.read(callbacks.loadFile(sLayoutFile)); + if(!data) { + // TODO: ReportError(0, CERR_InvalidTouchLayoutFile, sMsg); + return {output:null, result: false}; + } + + let FTouchLayoutFont = ''; // I4872 + let pid: keyof TouchLayout.TouchLayoutFile; + for(pid in data) { + let platform = data[pid]; + + // Test that the font matches on all platforms // I4872 + + if(FTouchLayoutFont == '') { + FTouchLayoutFont = platform.font; + } + else if(platform.font.toLowerCase() != FTouchLayoutFont) { + // TODO: ReportError(0, CWARN_TouchLayoutFontShouldBeSameForAllPlatforms, 'The touch layout font should be the same for all platforms.'); + // TODO: why support multiple font values if it has to be the same across all platforms?! + } + + // Test that all required keys are present + for(let layer of platform.layer) { + let FRequiredKeys: TRequiredKey[] = []; + for(let row of layer.row) { + for(let key of row.key) { + CheckKey(platform, key.id, key.text, key.nextlayer, key.sp, FRequiredKeys, FDictionary); // I4119 + if(key.sk) { + for(let subkey of key.sk) { + CheckKey(platform, subkey.id, subkey.text, subkey.nextlayer, subkey.sp, FRequiredKeys, FDictionary); + } + } + let direction: keyof TouchLayout.TouchLayoutFlick; + if(key.flick) { + for(direction in key.flick) { + CheckKey(platform, key.flick[direction].id, key.flick[direction].text, + key.flick[direction].nextlayer, key.flick[direction].sp, FRequiredKeys, FDictionary); + } + } + + if(key.multitap) { + for(let subkey of key.multitap) { + CheckKey(platform, subkey.id, subkey.text, subkey.nextlayer, subkey.sp, FRequiredKeys, FDictionary); + } + } + } + } + + if(FRequiredKeys.length != CRequiredKeys.length) { + // TODO: ReportError(0, CWARN_TouchLayoutMissingRequiredKeys, 'Layer "'+FLayer.Id+'" on platform "'+FPlatform.Name+'" is missing the required key(s) '+RequiredKeysToString(CRequiredKeys-FRequiredKeys)+'.'); + } + } + } + + // If not debugging, then this strips out formatting for a big saving in file size + // This also normalises any values such as Pad or Width which should be strings + let writer = new TouchLayoutFileWriter({formatted: FDebug}); + + sLayoutFile = writer.compile(data); + + sLayoutFile = TransformSpecialKeys14(FDebug, sLayoutFile); + + return { + output: sLayoutFile, + result: true + } +} \ No newline at end of file diff --git a/developer/src/kmc-kmw/src/compiler/visual-keyboard-compiler.ts b/developer/src/kmc-kmw/src/compiler/visual-keyboard-compiler.ts new file mode 100644 index 0000000000..6dc7cf3178 --- /dev/null +++ b/developer/src/kmc-kmw/src/compiler/visual-keyboard-compiler.ts @@ -0,0 +1,163 @@ +import { KMX, KvkFile, VisualKeyboard } from "@keymanapp/common-types"; +import { FTabStop, nl } from "./compiler-globals.js"; +import { CKeymanWebKeyCodes } from "./keymanweb-key-codes.js"; +import { RequotedString } from "./write-compiled-keyboard.js"; + +interface VKFFResult { + result: string; + displayUnderling: boolean; +} + +export function VisualKeyboardFromFile(visualKeyboard: VisualKeyboard.VisualKeyboard, debug: boolean): VKFFResult { + let fbold = ''; // TODO 'bold ' if visualKeyboard.header.unicodeFont.style + let fitalic = ''; // TODO 'italic ' if visualKeyboard.header.unicodeFont.style + let f102 = visualKeyboard.header.flags & KvkFile.BUILDER_KVK_HEADER_FLAGS.kvkh102 ? '1' : '0'; + + let result = `{F:'${fbold}${fitalic} 1em "${RequotedString(visualKeyboard.header.unicodeFont.name)}"',K102:${f102}}` + + `;` + VisualKeyboardToKLS(visualKeyboard) + + ';' + BuildBKFromKLS(debug); + + return { + result: result, + // TODO: this can go in caller, later + displayUnderling: !!(visualKeyboard.header.flags & KvkFile.BUILDER_KVK_HEADER_FLAGS.kvkhDisplayUnderlying) + } +} + +function WideQuote(s: string): string { + let result = ''; + for(let i = 0; i < s.length; i++) { + if(s[i] == '"' || s[i] == '\\') { + result += '\\' + s[i]; + } else { + result += s[i]; + } + } + return result; +} + +function VkShiftStateToKmxShiftState(ShiftState: number): number { + + interface TVKToKMX { + VK: number; KMX: number; + } + + const Map: TVKToKMX[] = [ + {VK: KvkFile.BUILDER_KVK_SHIFT_STATE.KVKS_SHIFT, KMX: KMX.KMXFile.K_SHIFTFLAG}, + {VK: KvkFile.BUILDER_KVK_SHIFT_STATE.KVKS_CTRL, KMX: KMX.KMXFile.K_CTRLFLAG}, + {VK: KvkFile.BUILDER_KVK_SHIFT_STATE.KVKS_ALT, KMX: KMX.KMXFile.K_ALTFLAG}, + {VK: KvkFile.BUILDER_KVK_SHIFT_STATE.KVKS_LCTRL, KMX: KMX.KMXFile.LCTRLFLAG}, + {VK: KvkFile.BUILDER_KVK_SHIFT_STATE.KVKS_RCTRL, KMX: KMX.KMXFile.RCTRLFLAG}, + {VK: KvkFile.BUILDER_KVK_SHIFT_STATE.KVKS_LALT, KMX: KMX.KMXFile.LALTFLAG}, + {VK: KvkFile.BUILDER_KVK_SHIFT_STATE.KVKS_RALT, KMX: KMX.KMXFile.RALTFLAG} + ]; + + let result = 0; + for(let i = 0; i < Map.length; i++) { + if (ShiftState & Map[i].VK) { + result |= Map[i].KMX; + } + } + + return result; +} + + +function VKShiftToLayerName(shift: number): string { + + const masks: string[] = [ + 'leftctrl', + 'rightctrl', + 'leftalt', + 'rightalt', + 'shift', + 'ctrl', + 'alt' + ]; + + shift = VkShiftStateToKmxShiftState(shift); + if(shift == 0) { + return 'default'; + } + + let result = ''; + for(let i = 0; i < masks.length; i++) { + if(shift & (1 << i)) { + result += masks[i] + '-'; + } + } + return result.substring(0, result.length - 1); +} + + +function VisualKeyboardToKLS(FVK: VisualKeyboard.VisualKeyboard): string { + + interface TLayer { + shift: number; + name: string; + keys: string[]; + }; + + let layers: TLayer[] = []; + + // Discover the layers used in the visual keyboard + for(let key of FVK.keys) { + if(key.flags & KvkFile.BUILDER_KVK_KEY_FLAGS.kvkkUnicode) { + // Find the index of the key in KMW VK arrays + let n = CKeymanWebKeyCodes[key.vkey]; + if(n == 0xFF) { + continue; + } + + let layer = layers.find(layer => layer.shift == key.shift); + if(!layer) { + // 0-64 covers all possible VirtualKeyCodes in CKemyanWebKeyCodes + layer = { shift: key.shift, name: '', keys: new Array(65)}; + layers.push(layer); + } + layer.keys[n] = key.text; + } + } + + // Build the layer array + + let result = nl+FTabStop+'this.KV.KLS={'+nl; + + for(let i = 0; i < layers.length; i++) { + let layer = layers[i]; + result += `${FTabStop}${FTabStop}"${VKShiftToLayerName(layer.shift)}": [`; + for(let j = 0; j < layer.keys.length - 1; j++) { + result += '"'+WideQuote(layer.keys[j] ?? '')+'",'; + } + result += '"'+WideQuote(layer.keys[layer.keys.length-1] ?? '')+'"]'; + if(i < layers.length - 1) { + result += ',' + nl; + } + } + result += nl+FTabStop+'}'; + return result; +} + +function BuildBKFromKLS(debug: boolean): string { + const func = + 'function(x){var e=Array.apply(null,Array(65)).map(String.prototype.valueOf,"")'+ + ',r=[],v,i,m=[\'default\',\'shift\',\'ctrl\',\'shift-ctrl\',\'alt\',\'shift-alt\','+ + '\'ctrl-alt\',\'shift-ctrl-alt\'];for(i=m.length-1;i>=0;i--)if((v=x[m[i]])||r.length)'+ + 'r=(v?v:e).slice().concat(r);return r}'; + const func_debug = + 'function(x){'+nl+ + ' var'+nl+ + ' empty=Array.apply(null, Array(65)).map(String.prototype.valueOf,""),'+nl+ + ' result=[], v, i,'+nl+ + ' modifiers=[\'default\',\'shift\',\'ctrl\',\'shift-ctrl\',\'alt\',\'shift-alt\',\'ctrl-alt\',\'shift-ctrl-alt\'];'+nl+ + ' for(i=modifiers.length-1;i>=0;i--) {'+nl+ + ' v = x[modifiers[i]];'+nl+ + ' if(v || result.length > 0) {'+nl+ + ' result=(v ? v : empty).slice().concat(result);'+nl+ + ' }'+nl+ + ' }'+nl+ + ' return result;'+nl+ + ' }'; + + return nl+FTabStop+'this.KV.BK=('+(debug ? func_debug : func)+')(this.KV.KLS)'; +} diff --git a/developer/src/kmc-kmw/src/compiler/write-compiled-keyboard.ts b/developer/src/kmc-kmw/src/compiler/write-compiled-keyboard.ts index 0175072aeb..7e89aa7cdd 100644 --- a/developer/src/kmc-kmw/src/compiler/write-compiled-keyboard.ts +++ b/developer/src/kmc-kmw/src/compiler/write-compiled-keyboard.ts @@ -1,10 +1,13 @@ -import { KMX, CompilerCallbacks } from "@keymanapp/common-types"; +import { KVKSParseError, VisualKeyboard } from "@keymanapp/common-types"; +import { KMX, CompilerCallbacks, KvkFileReader, KvksFileReader } from "@keymanapp/common-types"; import { ExpandSentinel, incxstr, xstrlen } from "../util/util.js"; // import { KEY, KEYBOARD, KMX.KMXFile, STORE } from "../../../../../common/web/types/src/kmx/kmx.js"; import { options, nl, FTabStop, setupGlobals, IsKeyboardVersion10OrLater } from "./compiler-globals.js"; import CompilerOptions from "./compiler-options.js"; import { JavaScript_ContextMatch, JavaScript_KeyAsString, JavaScript_Name, JavaScript_OutputString, JavaScript_Rules, JavaScript_Shift, JavaScript_ShiftAsString, JavaScript_Store, zeroPadHex } from './javascript-strings.js'; import { CERR_InvalidBegin, CWARN_DontMixChiralAndNonChiralModifiers, ReportError } from "./messages.js"; +import { ValidateLayoutFile } from "./validate-layout-file.js"; +import { VisualKeyboardFromFile } from "./visual-keyboard-compiler.js"; export let FFix183_LadderLength: number = 100; // TODO: option @@ -12,7 +15,7 @@ function requote(s: string): string { return "'" + s.replaceAll(/(['\\])/, "\\$1") + "'"; } -function RequotedString(s: string, RequoteSingleQuotes: boolean = false): string { +export function RequotedString(s: string, RequoteSingleQuotes: boolean = false): string { // TODO: use a JSON encode let i: number = 0; while(i < s.length) { @@ -35,12 +38,12 @@ function RequotedString(s: string, RequoteSingleQuotes: boolean = false): string return s; } -export function WriteCompiledKeyboard(callbacks: CompilerCallbacks, name: string, keyboard: KMX.KEYBOARD, FDebug: boolean = false): string { +export function WriteCompiledKeyboard(callbacks: CompilerCallbacks, kmnfile: string, kmxfile: string, name: string, keyboard: KMX.KEYBOARD, FDebug: boolean = false): string { let opts: CompilerOptions = { addCompilerVersion: false, debug: FDebug }; - setupGlobals(callbacks, opts, FDebug?' ':'', FDebug?'\n':'', keyboard); + setupGlobals(callbacks, opts, FDebug?' ':'', FDebug?'\r\n':'', keyboard); // let fgp: GROUP; @@ -172,26 +175,17 @@ export function WriteCompiledKeyboard(callbacks: CompilerCallbacks, name: string } if (sLayoutFile != '') { // I3483 - // TODO: Load sLayoutFile from file - /*try - with TStringList.Create do - try - LoadFromFile(ExtractFilePath(FInFile) + sLayoutFile, TEncoding.UTF8); - sLayoutFile := Text; - if not ValidateLayoutFile(sLayoutFile, sVKDictionary) then // I4060 - begin - sLayoutFile := ''; - end; - finally - Free; - end; - except - on E:EFOpenError do // I3683 - begin - ReportError(0, CWARN_TouchLayoutFileMissing, E.Message); // I4061 - sLayoutFile := ''; - end; - end;*/ + let path = callbacks.resolveFilename(kmnfile, sLayoutFile); + + let result = ValidateLayoutFile(keyboard, options.debug, path, sVKDictionary); + if(!result.result) { + sLayoutFile = ''; + // TODO: error + // ReportError(0, CWARN_TouchLayoutFileInvalid, 'Touch layout file is not valid'); + } else { + // TODO: reusing the same variable here is ugly + sLayoutFile = result.output; + } } // Default to hide underlying layout characters. This is overridden by touch @@ -200,22 +194,34 @@ export function WriteCompiledKeyboard(callbacks: CompilerCallbacks, name: string let fDisplayUnderlying = false; if (sVisualKeyboard != '') { - //TODO: Load sVisualKeyboard from file - /* - try - // The Keyman .kmx compiler will change the value of this store from a - // .kvks to a .kvk during the build. Earlier in the build, the visual keyboard - // would have been compiled, so we need to account for that and use that file. + // TODO: stop reusing sVisualKeyboard for both filename and content + let path = callbacks.resolveFilename(kmnfile, sVisualKeyboard); - sVisualKeyboard := VisualKeyboardFromFile(ExtractFilePath(FOutFile) + sVisualKeyboard, fDisplayUnderlying); - except - on E:EFOpenError do // I3947 - begin - ReportError(0, CWARN_VisualKeyboardFileMissing, E.Message); // I4061 - sVisualKeyboard := 'null'; - end; - end; - */ + let kvk: VisualKeyboard.VisualKeyboard; + if(path.match(/\.kvks$/i)) { + let reader = new KvksFileReader(); + let source = reader.read(callbacks.loadFile(path)); + reader.validate(source, callbacks.loadSchema("kvks")); // TODO: handle exceptions + let errors: KVKSParseError[]; + kvk = reader.transform(source, errors); + // TODO: log errors + } + else { + // Note: very old keyboard sources may still have .kvk as an xml + // file, but we'll treat that as an error rather than silently + // falling back to KvksFileReader + let reader = new KvkFileReader(); + kvk = reader.read(callbacks.loadFile(path)); + } + + let result = VisualKeyboardFromFile(kvk, options.debug); + if(!result.result) { + // TODO: error + sVisualKeyboard = 'null'; + } + else { + sVisualKeyboard = result.result; + } } else { sVisualKeyboard = 'null'; @@ -236,7 +242,7 @@ export function WriteCompiledKeyboard(callbacks: CompilerCallbacks, name: string // Following line caches the Keyman major version `${FTabStop}this._v=(typeof keyman!="undefined"&&typeof keyman.version=="string")?parseInt(keyman.version,10):9;${nl}` + `${FTabStop}this.KI="${sName}";${nl}` + - `${FTabStop}this.KN="${RequotedString(sFullName)}";${nl}`, nl, + `${FTabStop}this.KN="${RequotedString(sFullName)}";${nl}` + `${FTabStop}this.KMINVER="${(keyboard.fileVersion & KMX.KMXFile.VERSION_MASK_MAJOR) >> 8}.${keyboard.fileVersion & KMX.KMXFile.VERSION_MASK_MINOR}";${nl}` + `${FTabStop}this.KV=${sVisualKeyboard};${nl}` + `${FTabStop}this.KDU=${fDisplayUnderlying?'1':'0'};${nl}` + diff --git a/developer/src/kmc-kmw/test/test-compiler-manual.ts b/developer/src/kmc-kmw/test/test-compiler-manual.ts index af2589b833..9f4f5ea4db 100644 --- a/developer/src/kmc-kmw/test/test-compiler-manual.ts +++ b/developer/src/kmc-kmw/test/test-compiler-manual.ts @@ -5,6 +5,7 @@ import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers'; import { Compiler } from '@keymanapp/kmc-kmn'; import { KMX, KmxFileReader } from '@keymanapp/common-types'; import { WriteCompiledKeyboard } from '../src/compiler/write-compiled-keyboard.js'; +import { extractTouchLayout } from './util.js'; const __dirname = dirname(fileURLToPath(import.meta.url)).replace(/\\/g, '/'); const fixturesDir = __dirname + '/../../test/fixtures/'; @@ -28,7 +29,8 @@ if(!await kmxCompiler.init()) { // TODO: runToMemory, add option to kmxCompiler to store debug-data for conversion to .js (e.g. store metadata, group readonly metadata, etc) if(!kmxCompiler.run(infile, outfile, callbacks, { shouldAddCompilerVersion: false, - saveDebug: true + saveDebug: true, + target: 'js' })) { callbacks.printMessages(); process.exit(1); @@ -37,12 +39,16 @@ if(!kmxCompiler.run(infile, outfile, callbacks, { const reader = new KmxFileReader(); const keyboard: KMX.KEYBOARD = reader.read(callbacks.loadFile(outfile)); -const js = WriteCompiledKeyboard(callbacks, 'khmer_angkor', keyboard, true); +const js = WriteCompiledKeyboard(callbacks, infile, outfile, 'khmer_angkor', keyboard, true); callbacks.printMessages(); const fjs = fs.readFileSync(fixtureName, 'utf8'); -if(fjs !== js) { + +const expected = extractTouchLayout(fjs); +const actual = extractTouchLayout(js); + +if(expected.js !== actual.js) { fs.writeFileSync(testOutfile, js); console.error('JS not equal'); process.exit(1); diff --git a/developer/src/kmc-kmw/test/test-compiler.ts b/developer/src/kmc-kmw/test/test-compiler.ts index 7fb0efd9ce..7332d71bda 100644 --- a/developer/src/kmc-kmw/test/test-compiler.ts +++ b/developer/src/kmc-kmw/test/test-compiler.ts @@ -8,6 +8,7 @@ import fs from 'fs'; import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers'; import { Compiler } from '@keymanapp/kmc-kmn'; import { KMX, KmxFileReader } from '@keymanapp/common-types'; +import { extractTouchLayout } from './util.js'; const __dirname = dirname(fileURLToPath(import.meta.url)).replace(/\\/g, '/'); const fixturesDir = __dirname + '/../../test/fixtures/'; @@ -37,19 +38,28 @@ describe('Compiler class', function() { const kmxCompiler = new Compiler(); assert.isTrue(await kmxCompiler.init()); - // TODO: runToMemory, add option to kmxCompiler to store debug-data for conversion to .js (e.g. store metadata, group readonly metadata, etc) - assert.isTrue(kmxCompiler.run(infile, outfile, callbacks)); + // TODO: runToMemory, add option to kmxCompiler to store debug-data for conversion to .js (e.g. store metadata, group readonly metadata, visual keyboard source filename, etc) + assert.isTrue(kmxCompiler.run(infile, outfile, callbacks, { + shouldAddCompilerVersion: false, + saveDebug: true, + target: 'js' + })); const reader = new KmxFileReader(); const keyboard: KMX.KEYBOARD = reader.read(callbacks.loadFile(outfile)); - const js = WriteCompiledKeyboard(callbacks, 'khmer_angkor', keyboard, true); - // const js = compiler.compile('khmer_angkor', keyboard); + const js = WriteCompiledKeyboard(callbacks, infile, outfile, 'khmer_angkor', keyboard, true); const fjs = fs.readFileSync(fixtureName, 'utf8'); - if(fjs !== js) { - fs.writeFileSync(testOutfile, js); - assert.fail('JS not equal'); - } + + const expected = extractTouchLayout(fjs); + const actual = extractTouchLayout(js); + + fs.writeFileSync(testOutfile + '.strip.js', actual.js); + fs.writeFileSync(fixtureName + '.strip.js', expected.js); + fs.writeFileSync(testOutfile, js); + + assert.deepEqual(actual.js, expected.js); + assert.deepEqual(JSON.parse(actual.touchLayout), JSON.parse(expected.touchLayout)); }); }); \ No newline at end of file diff --git a/developer/src/kmc-kmw/test/tsconfig.json b/developer/src/kmc-kmw/test/tsconfig.json index abdb3a7cfe..3159c14bfd 100644 --- a/developer/src/kmc-kmw/test/tsconfig.json +++ b/developer/src/kmc-kmw/test/tsconfig.json @@ -14,7 +14,8 @@ }, }, "include": [ - "**/test-*.ts" + "**/test-*.ts", + "util.ts" ], "references": [ { "path": "../../../../common/web/types/" }, diff --git a/developer/src/kmc-kmw/test/util.ts b/developer/src/kmc-kmw/test/util.ts new file mode 100644 index 0000000000..f63e8dd26e --- /dev/null +++ b/developer/src/kmc-kmw/test/util.ts @@ -0,0 +1,19 @@ + +interface ETLResult { + js: string; + touchLayout: string; +} + +export function extractTouchLayout(js: string): ETLResult|null { + let m = /KVKL=(?.+?);[\r\n]/ds.exec(js); + if(!m) { + return null; + } + + let kvkl = (m).indices.groups.kvkl; + + return { + js: js.substring(0, kvkl[0]) + 'null' + js.substring(kvkl[1]), + touchLayout: m.groups?.['kvkl'] ?? '' + }; +} diff --git a/developer/src/kmc-kmw/tsconfig.json b/developer/src/kmc-kmw/tsconfig.json index fb2abd93e3..4fabe4a39d 100644 --- a/developer/src/kmc-kmw/tsconfig.json +++ b/developer/src/kmc-kmw/tsconfig.json @@ -5,21 +5,18 @@ "outDir": "build/src/", "rootDir": "src/", "baseUrl": ".", - "allowSyntheticDefaultImports": true, // for ajv - "paths": { - // "@keymanapp/keyman-version": ["../../../common/web/keyman-version/keyman-version.mts"], "@keymanapp/common-types": ["../../../common/web/types/src/main"], - // "@keymanapp/": ["core/include/ldml/ldml-keyboard-constants"], + "@keymanapp/kmc-kmn": ["../kmc-kmn/src/main"], }, }, "include": [ "src/**/*.ts" -, "src/compiler/__keymanweb-compiler.ts.tmp" ], + ], "references": [ - // { "path": "../../../common/web/keyman-version/tsconfig.esm.json" }, { "path": "../../../common/web/types/" }, + { "path": "../kmc-kmn/" }, { "path": "../../../core/include/ldml/"}, ] } diff --git a/developer/src/kmcmpdll/Compiler.cpp b/developer/src/kmcmpdll/Compiler.cpp index 89defad84d..847a1ff78c 100644 --- a/developer/src/kmcmpdll/Compiler.cpp +++ b/developer/src/kmcmpdll/Compiler.cpp @@ -334,7 +334,7 @@ extern "C" BOOL __declspec(dllexport) CompileKeyboardFileEx(PSTR pszInfile, PSTR if ( flag_use_new_kmcomp ) { - return kmcmp_CompileKeyboardFile(pszInfile, pszOutfile, ASaveDebug, ACompilerWarningsAsErrors,AWarnDeprecatedCode, kmcmpMsgproc, (void*) pMsgProc); + return kmcmp_CompileKeyboardFile(pszInfile, pszOutfile, ASaveDebug, ACompilerWarningsAsErrors,AWarnDeprecatedCode, kmcmpMsgproc, (void*) pMsgProc, Target); } //printf("---> stayed in CompileKeyboardFile() of kmcmpdll\n"); diff --git a/developer/src/kmcmplib/include/kmcmplibapi.h b/developer/src/kmcmplib/include/kmcmplibapi.h index be0f477ab9..e825550f70 100644 --- a/developer/src/kmcmplib/include/kmcmplibapi.h +++ b/developer/src/kmcmplib/include/kmcmplibapi.h @@ -32,7 +32,8 @@ EXTERN bool kmcmp_CompileKeyboardFile( bool ACompilerWarningsAsErrors, bool AWarnDeprecatedCode, kmcmp_CompilerMessageProc pMsgproc, - void* AmsgprocContext + void* AmsgprocContext, + int target // CKF_KEYMAN || CKF_KEYMANWEB ); /* Compile target */ diff --git a/developer/src/kmcmplib/src/Compiler.cpp b/developer/src/kmcmplib/src/Compiler.cpp index 06e707e7ea..48d97120d5 100644 --- a/developer/src/kmcmplib/src/Compiler.cpp +++ b/developer/src/kmcmplib/src/Compiler.cpp @@ -1042,12 +1042,21 @@ KMX_DWORD ProcessSystemStore(PFILE_KEYBOARD fk, KMX_DWORD SystemID, PFILE_STORE pp2[4] = 0; } - delete[] sp->dpString; - sp->dpString = q; + if(CompileTarget == CKF_KEYMAN) { + // When we compile to kmx, we want to save this info into + // the .kmx, but for KMW, we need the original source file + // for the KMW compiler process + delete[] sp->dpString; + sp->dpString = q; + } - if ((msg = CheckFilenameConsistency( (sp->dpString), FALSE)) != CERR_None) { + if ((msg = CheckFilenameConsistency(q, FALSE)) != CERR_None) { return msg; } + + if(CompileTarget == CKF_KEYMANWEB) { + delete[] q; + } } break; case TSS_KMW_RTL: diff --git a/developer/src/kmcmplib/src/CompilerInterfaces.cpp b/developer/src/kmcmplib/src/CompilerInterfaces.cpp index 1eb9b6eaab..95f01353f4 100644 --- a/developer/src/kmcmplib/src/CompilerInterfaces.cpp +++ b/developer/src/kmcmplib/src/CompilerInterfaces.cpp @@ -53,7 +53,7 @@ EXTERN bool kmcmp_Wasm_SetCompilerOptions(int ShouldAddCompilerVersion) { EXTERN bool kmcmp_Wasm_CompileKeyboardFile(char* pszInfile, char* pszOutfile, int ASaveDebug, int ACompilerWarningsAsErrors, - int AWarnDeprecatedCode, char* msgProc + int AWarnDeprecatedCode, char* msgProc, int target ) { return kmcmp_CompileKeyboardFile( pszInfile, @@ -62,14 +62,15 @@ EXTERN bool kmcmp_Wasm_CompileKeyboardFile(char* pszInfile, ACompilerWarningsAsErrors, AWarnDeprecatedCode, wasm_CompilerMessageProc, - msgProc + msgProc, + target ); } #endif EXTERN bool kmcmp_CompileKeyboardFile(char* pszInfile, char* pszOutfile, bool ASaveDebug, bool ACompilerWarningsAsErrors, - bool AWarnDeprecatedCode, kmcmp_CompilerMessageProc pMsgproc, void* AmsgprocContext + bool AWarnDeprecatedCode, kmcmp_CompilerMessageProc pMsgproc, void* AmsgprocContext, int Target ) { FILE* fp_in = NULL; FILE* fp_out = NULL; @@ -81,7 +82,7 @@ EXTERN bool kmcmp_CompileKeyboardFile(char* pszInfile, kmcmp::FCompilerWarningsAsErrors = ACompilerWarningsAsErrors; // I4865 AWarnDeprecatedCode_GLOBAL_LIB = AWarnDeprecatedCode; - kmcmp::CompileTarget = CKF_KEYMAN; + kmcmp::CompileTarget = Target; if (!pMsgproc || !pszInfile || !pszOutfile) SetError(CERR_BadCallParams); diff --git a/developer/src/kmcmplib/tests/api-test.cpp b/developer/src/kmcmplib/tests/api-test.cpp index 4c46b1f1a4..4c08272d4b 100644 --- a/developer/src/kmcmplib/tests/api-test.cpp +++ b/developer/src/kmcmplib/tests/api-test.cpp @@ -62,7 +62,7 @@ void test_kmcmp_CompileKeyboardFile() { fclose(fp); // It should fail when a zero-byte file is passed in - assert(!kmcmp_CompileKeyboardFile(kmn_file, kmx_file, true, false, true, msgproc, nullptr)); + assert(!kmcmp_CompileKeyboardFile(kmn_file, kmx_file, true, false, true, msgproc, nullptr, CKF_KEYMAN)); assert(error_vec.size() == 1); assert(error_vec[0] == CERR_CannotReadInfile); diff --git a/developer/src/kmcmplib/tests/kmcompxtest.cpp b/developer/src/kmcmplib/tests/kmcompxtest.cpp index 1ce542917d..fc1e23a514 100644 --- a/developer/src/kmcmplib/tests/kmcompxtest.cpp +++ b/developer/src/kmcmplib/tests/kmcompxtest.cpp @@ -78,7 +78,7 @@ int main(int argc, char *argv[]) return __LINE__; } - if(kmcmp_CompileKeyboardFile(kmn_file, kmx_file, true, false, true, msgproc, nullptr)) { + if(kmcmp_CompileKeyboardFile(kmn_file, kmx_file, true, false, true, msgproc, nullptr, CKF_KEYMAN)) { char* testname = strrchr( (char*) kmn_file, '/') + 1; if(strncmp(testname, pfirst5, 5) == 0){ return __LINE__; // exit code: CERR_ in Name + no Error found