From efad9361054ea8d377e0ebaec00de7919dc2a8a6 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 16 Mar 2026 07:21:38 +0100 Subject: [PATCH] feat(common): add KMX+ file reader in Typescript Add a KMX+ file reader and tests, including a round-trip builder:reader test. Move basic-17.txt, basic-19.txt, and basic.xml to common test resources folder so they can be used by unit tests in common and developer. Test-bot: skip --- .../test/keyboards/kmx-plus}/basic-17.txt | 29 +- .../test/keyboards/kmx-plus}/basic-19.txt | 27 +- .../test/keyboards/kmx-plus}/basic.xml | 0 common/web/types/build.sh | 2 +- common/web/types/package.json | 1 + .../src/kmx/kmx-plus/kmx-plus-file-reader.ts | 481 ++++++++++++++++++ .../types/src/kmx/kmx-plus/kmx-plus-file.ts | 322 +++++++++++- common/web/types/src/kmx/kmx-plus/kmx-plus.ts | 43 +- common/web/types/src/main.ts | 5 + common/web/types/src/util/errors.ts | 9 +- common/web/types/tests/helpers/index.ts | 15 +- .../kmx-plus/kmx-plus-file-reader.tests.ts | 409 +++++++++++++++ common/web/types/tests/tsconfig.json | 1 + core/include/ldml/keyman_core_ldml.h | 1 + core/include/ldml/keyman_core_ldml.ts | 41 +- developer/src/common/web/utils/.c8rc.json | 13 + developer/src/common/web/utils/build.sh | 2 +- .../common/web/utils/test/helpers/index.ts | 20 +- .../kmx-plus-builder.tests.ts | 37 ++ .../src/kmc-ldml/test/compiler-e2e.tests.ts | 16 +- developer/src/kmc-ldml/test/helpers/index.ts | 16 +- .../kmc-ldml/test/metadata-compiler.tests.ts | 6 +- .../test/visual-keyboard-compiler.tests.ts | 4 +- package-lock.json | 2 + 24 files changed, 1432 insertions(+), 70 deletions(-) rename {developer/src/kmc-ldml/test/fixtures => common/test/keyboards/kmx-plus}/basic-17.txt (97%) rename {developer/src/kmc-ldml/test/fixtures => common/test/keyboards/kmx-plus}/basic-19.txt (98%) rename {developer/src/kmc-ldml/test/fixtures => common/test/keyboards/kmx-plus}/basic.xml (100%) create mode 100644 common/web/types/src/kmx/kmx-plus/kmx-plus-file-reader.ts create mode 100644 common/web/types/tests/kmx/kmx-plus/kmx-plus-file-reader.tests.ts create mode 100644 developer/src/common/web/utils/.c8rc.json create mode 100644 developer/src/common/web/utils/test/kmx/kmx-plus-builder/kmx-plus-builder.tests.ts diff --git a/developer/src/kmc-ldml/test/fixtures/basic-17.txt b/common/test/keyboards/kmx-plus/basic-17.txt similarity index 97% rename from developer/src/kmc-ldml/test/fixtures/basic-17.txt rename to common/test/keyboards/kmx-plus/basic-17.txt index 3d9891c9cb..99159142b7 100644 --- a/developer/src/kmc-ldml/test/fixtures/basic-17.txt +++ b/common/test/keyboards/kmx-plus/basic-17.txt @@ -154,6 +154,7 @@ block(bksp) index(strNull,strNull,2) # KMX_DWORD str mapFrom 0 index(strNull,strNull,2) # KMX_DWORD str mapTo 0 +block(endbksp) # ---------------------------------------------------------------------------------------------------- # disp @@ -225,12 +226,12 @@ block(elemOrdrFrom) 00 00 00 00 # TODO-LDML: uset #0 # KMX_DWORD element; [uset] // str: output string or UTF-32LE codepoint 02 00 37 00 # KMX_DWORD flags; // flag and order values - unicodeset 45 1A 00 00 # KMX_DWORD element; 'ᩅ' // str: output string or UTF-32LE codepoint - 00 00 0A 00 # KMX_DWORD flags; // flag and order values - unicodeset + 00 00 0A 00 # KMX_DWORD flags; // flag and order values - cp block(elemOrdrBefore) # before="\u{1A6B}" 6b 1a 00 00 # KMX_DWORD element; 'ᩫ' // str: output string or UTF-32LE codepoint - 00 00 00 00 # KMX_DWORD flags; // flag and order values - unicodeset + 00 00 00 00 # KMX_DWORD flags; // flag and order values - cp block(endelem) @@ -380,6 +381,8 @@ block(keys) # struct COMP_KMXPLUS_KEYS { dd 00 00 00 00 00 00 00 02 00 00 00 # gap (reserved) de 00 00 00 00 00 00 00 02 00 00 00 # gap (reserved) +block(endkeys) + # ---------------------------------------------------------------------------------------------------- # layr # ---------------------------------------------------------------------------------------------------- @@ -559,10 +562,10 @@ block(endstrs) # end of strs block block(tran) # struct COMP_KMXPLUS_TRAN { 74 72 61 6e # KMX_DWORD header.ident; // 0000 Section name - tran - diff(tran,tranEnd) # KMX_DWORD header.size; // 0004 Section length + diff(tran,endtran) # KMX_DWORD header.size; // 0004 Section length diff(tranGroupStart,tranTransformStart,12) # KMX_DWORD groupCount; diff(tranTransformStart,tranReorderStart,16) # KMX_DWORD transformCount; - diff(tranReorderStart,tranEnd,8) # KMX_DWORD reorderCount; + diff(tranReorderStart,endtran,8) # KMX_DWORD reorderCount; block(tranGroupStart) # COMP_KMXPLUS_TRAN_GROUP # group 0 @@ -606,11 +609,11 @@ block(tran) # struct COMP_KMXPLUS_TRAN { index(elemNull,elemOrdrFrom) # KMXPLUS_ELEM elements; index(elemNull,elemOrdrBefore) # KMXPLUS_ELEM before; -block(tranEnd) +block(endtran) block(uset) 75 73 65 74 # KMX_DWORD header.ident; // 0000 Section name - uset - diff(uset,usetEnd) # KMX_DWORD header.size; // 0004 Section length + diff(uset,enduset) # KMX_DWORD header.size; // 0004 Section length 01 00 00 00 # lists 01 00 00 00 # elements # lists @@ -621,17 +624,17 @@ block(uset) # range @0 75 1A 00 00 # start 79 1A 00 00 # end - block(usetEnd) + block(enduset) block(vars) # struct COMP_KMXPLUS_VARS { 76 61 72 73 # KMX_DWORD header.ident; // 0000 Section name - vars - diff(vars,varsEnd) # KMX_DWORD header.size; // 0004 Section length + diff(vars,endvars) # KMX_DWORD header.size; // 0004 Section length 01 00 00 00 # KMX_DWORD markers - list 1 ['a'] - diff(varsBegin,varsEnd,16) # KMX_DWORD varCount + diff(varsBegin,endvars,16) # KMX_DWORD varCount block(varsBegin) # var 0 - 00 00 00 00 # KMX_DWORD type = str + 00 00 00 00 # KMX_DWORD type = string index(strNull,strA,2) # KMXPLUS_STR id 'a' index(strNull,strAmarker,2) # KMXPLUS_STR value '\m{a}' 00 00 00 00 # KMXPLUS_ELEM @@ -649,10 +652,10 @@ block(vars) # struct COMP_KMXPLUS_VARS { 00 00 00 00 # KMXPLUS_ELEM elem # var 3 - 02 00 00 00 # KMX_DWORD type = string + 02 00 00 00 # KMX_DWORD type = uset index(strNull,strVus,2) # KMXPLUS_STR id 'vus' index(strNull,strUSet,2) # KMXPLUS_STR value '[abc]' - 00 00 00 00 # KMXPLUS_ELEM elem -block(varsEnd) + 00 00 00 00 # KMXPLUS_ELEM elem == uset.list[0] +block(endvars) block(eof) # end of file diff --git a/developer/src/kmc-ldml/test/fixtures/basic-19.txt b/common/test/keyboards/kmx-plus/basic-19.txt similarity index 98% rename from developer/src/kmc-ldml/test/fixtures/basic-19.txt rename to common/test/keyboards/kmx-plus/basic-19.txt index 75806e7293..62542dafc4 100644 --- a/developer/src/kmc-ldml/test/fixtures/basic-19.txt +++ b/common/test/keyboards/kmx-plus/basic-19.txt @@ -163,6 +163,7 @@ block(bksp) index(strNull,strNull,2) # KMX_DWORD str mapFrom 0 index(strNull,strNull,2) # KMX_DWORD str mapTo 0 +block(endbksp) # ---------------------------------------------------------------------------------------------------- # disp (v19) @@ -236,12 +237,12 @@ block(elemOrdrFrom) 00 00 00 00 # TODO-LDML: uset #0 # KMX_DWORD element; [uset] // str: output string or UTF-32LE codepoint 02 00 37 00 # KMX_DWORD flags; // flag and order values - unicodeset 45 1A 00 00 # KMX_DWORD element; 'ᩅ' // str: output string or UTF-32LE codepoint - 00 00 0A 00 # KMX_DWORD flags; // flag and order values - unicodeset + 00 00 0A 00 # KMX_DWORD flags; // flag and order values - cp block(elemOrdrBefore) # before="\u{1A6B}" 6b 1a 00 00 # KMX_DWORD element; 'ᩫ' // str: output string or UTF-32LE codepoint - 00 00 00 00 # KMX_DWORD flags; // flag and order values - unicodeset + 00 00 00 00 # KMX_DWORD flags; // flag and order values - cp block(endelem) @@ -392,6 +393,8 @@ block(keys) # struct COMP_KMXPLUS_KEYS { dd 00 00 00 00 00 00 00 02 00 00 00 # gap (reserved) de 00 00 00 00 00 00 00 02 00 00 00 # gap (reserved) +block(endkeys) + # ---------------------------------------------------------------------------------------------------- # layr # ---------------------------------------------------------------------------------------------------- @@ -581,11 +584,11 @@ block(endstrs) # end of strs block block(tran) # struct COMP_KMXPLUS_TRAN { 74 72 61 6e # KMX_DWORD header.ident; // 0000 Section name - tran - diff(tran,tranEnd) # KMX_DWORD header.size; // 0004 Section length + diff(tran,endtran) # KMX_DWORD header.size; // 0004 Section length 00 11 00 00 # KMX_DWORD header.version; // 0008 Section version, 17 = 0x11 diff(tranGroupStart,tranTransformStart,12) # KMX_DWORD groupCount; diff(tranTransformStart,tranReorderStart,16) # KMX_DWORD transformCount; - diff(tranReorderStart,tranEnd,8) # KMX_DWORD reorderCount; + diff(tranReorderStart,endtran,8) # KMX_DWORD reorderCount; block(tranGroupStart) # COMP_KMXPLUS_TRAN_GROUP # group 0 @@ -629,11 +632,11 @@ block(tran) # struct COMP_KMXPLUS_TRAN { index(elemNull,elemOrdrFrom) # KMXPLUS_ELEM elements; index(elemNull,elemOrdrBefore) # KMXPLUS_ELEM before; -block(tranEnd) +block(endtran) block(uset) 75 73 65 74 # KMX_DWORD header.ident; // 0000 Section name - uset - diff(uset,usetEnd) # KMX_DWORD header.size; // 0004 Section length + diff(uset,enduset) # KMX_DWORD header.size; // 0004 Section length 00 11 00 00 # KMX_DWORD header.version; // 0008 Section version, 17 = 0x11 01 00 00 00 # lists 01 00 00 00 # elements @@ -645,18 +648,18 @@ block(uset) # range @0 75 1A 00 00 # start 79 1A 00 00 # end - block(usetEnd) + block(enduset) block(vars) # struct COMP_KMXPLUS_VARS { 76 61 72 73 # KMX_DWORD header.ident; // 0000 Section name - vars - diff(vars,varsEnd) # KMX_DWORD header.size; // 0004 Section length + diff(vars,endvars) # KMX_DWORD header.size; // 0004 Section length 00 11 00 00 # KMX_DWORD header.version; // 0008 Section version, 17 = 0x11 01 00 00 00 # KMX_DWORD markers - list 1 ['a'] - diff(varsBegin,varsEnd,16) # KMX_DWORD varCount + diff(varsBegin,endvars,16) # KMX_DWORD varCount block(varsBegin) # var 0 - 00 00 00 00 # KMX_DWORD type = str + 00 00 00 00 # KMX_DWORD type = string index(strNull,strA,2) # KMXPLUS_STR id 'a' index(strNull,strAmarker,2) # KMXPLUS_STR value '\m{a}' 00 00 00 00 # KMXPLUS_ELEM @@ -674,10 +677,10 @@ block(vars) # struct COMP_KMXPLUS_VARS { 00 00 00 00 # KMXPLUS_ELEM elem # var 3 - 02 00 00 00 # KMX_DWORD type = string + 02 00 00 00 # KMX_DWORD type = uset index(strNull,strVus,2) # KMXPLUS_STR id 'vus' index(strNull,strUSet,2) # KMXPLUS_STR value '[abc]' 00 00 00 00 # KMXPLUS_ELEM elem -block(varsEnd) +block(endvars) block(eof) # end of file diff --git a/developer/src/kmc-ldml/test/fixtures/basic.xml b/common/test/keyboards/kmx-plus/basic.xml similarity index 100% rename from developer/src/kmc-ldml/test/fixtures/basic.xml rename to common/test/keyboards/kmx-plus/basic.xml diff --git a/common/web/types/build.sh b/common/web/types/build.sh index 08419fa35d..f05a907f8e 100755 --- a/common/web/types/build.sh +++ b/common/web/types/build.sh @@ -82,4 +82,4 @@ function do_configure() { builder_run_action clean rm -rf ./build/ ./tsconfig.tsbuildinfo ./src/schemas/ ./node_modules/ ./obj/ builder_run_action configure do_configure builder_run_action build tsc --build -builder_run_action test typescript_run_eslint_mocha_tests 60 +builder_run_action test typescript_run_eslint_mocha_tests diff --git a/common/web/types/package.json b/common/web/types/package.json index 4c2e2bfd2e..bb259c52a5 100644 --- a/common/web/types/package.json +++ b/common/web/types/package.json @@ -36,6 +36,7 @@ "restructure": "3.0.1" }, "devDependencies": { + "@keymanapp/hextobin": "*", "@types/mocha": "^5.2.7", "@types/node": "^20.4.1", "ajv": "^8.12.0", diff --git a/common/web/types/src/kmx/kmx-plus/kmx-plus-file-reader.ts b/common/web/types/src/kmx/kmx-plus/kmx-plus-file-reader.ts new file mode 100644 index 0000000000..12d235ffa0 --- /dev/null +++ b/common/web/types/src/kmx/kmx-plus/kmx-plus-file-reader.ts @@ -0,0 +1,481 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * KMX+ file reader: load a KMX+ file into KMXPlus structures + */ + +import * as r from 'restructure'; +import { KMXPlusVersion, SectionIdent, constants } from "@keymanapp/ldml-keyboard-constants"; +import { KeymanTypesError } from "../../util/errors.js"; +import { ListIndex } from '../../ldml-keyboard/string-list.js'; +import { UnicodeSet, UnicodeSetParser } from '../../ldml-keyboard/unicodeset-parser-api.js'; +import { ICOMP_PLUS_DISP_v17, ICOMP_PLUS_DISP_v19, ICOMP_PLUS_ELEM, ICOMP_PLUS_ELEM_ELEMENT, ICOMP_PLUS_KEYS, ICOMP_PLUS_LAYR_v17, ICOMP_PLUS_LAYR_v19, ICOMP_PLUS_LIST, ICOMP_PLUS_LOCA, ICOMP_PLUS_META, ICOMP_PLUS_SECT, ICOMP_PLUS_SectionHeader, ICOMP_PLUS_STRS, ICOMP_PLUS_TRAN, ICOMP_PLUS_USET, ICOMP_PLUS_VARS, KMXPlusFileFormat } from "./kmx-plus-file.js"; +import * as KMXPlus from "./kmx-plus.js"; +import { ElemElement, ElementString } from './element-string.js'; +import { KMXFile } from '../kmx.js'; + +export class KMXPlusFileReaderError extends KeymanTypesError { + constructor(message?: string, options?: ErrorOptions) { + super(message, options); + this.name = this.constructor.name; + } +} + +export const KMXPLUS_FILE_READER_ERROR = { + SOURCE_IS_REQUIRED: () => "source is required", + FILE_IS_TOO_SHORT: () => "file is too short", + UNRECOGNIZED_MAGIC: () => "header magic bytes should be 'sect' or 'sec2'", + VERSION_SHOULD_MATCH_CONSTRUCTOR: (o:{version: KMXPlusVersion, constructorVersion: KMXPlusVersion}) => `Expected version '${o.version}' to match constructor version '${o.constructorVersion}'`, + MISSING_SECT: () => "Missing 'sect' section", + EXPECTED_SECT_OR_SEC2: () => "Expected 'sect' or 'sec2' section", + UNKNOWN_ELEMENT_TYPE: (o:{type:string}) => `Internal Error: Unknown element type 0x${o.type}`, + UNKNOWN_VAR_TYPE: (o:{type:number, id:number}) => `Unrecognized var type ${o.type} for ${o.id}`, + UNKNOWN_GROUP_TYPE: (o:{type:number}) => `Unrecognized group type ${o.type}`, +}; + +export class KMXPlusFileReader { + private format: KMXPlusFileFormat; + + constructor(private version?: KMXPlusVersion) { + if(this.version) { + this.format = new KMXPlusFileFormat(this.version); + } + } + + public readFromKmx(input: Uint8Array) { + // This is for a valid KMX+ files, results undefined if not a valid KMX+ file + const kmx = new KMXFile(); + const binaryKmxPlusHeader = kmx.COMP_KEYBOARD_KMXPLUSINFO.fromBuffer(input.slice(KMXFile.COMP_KEYBOARD_SIZE)); + return this.read(input.slice(binaryKmxPlusHeader.dpKMXPlus, binaryKmxPlusHeader.dpKMXPlus + binaryKmxPlusHeader.dwKMXPlusSize)); + } + + /** + * Read the KMX+ data into memory + * @param source KMX+ data blob starting at the sect/sec2 section -- does not include the KMX wrapper + * @returns in-memory representation of the KMX+ data + */ + public read(source: Uint8Array): KMXPlus.KMXPlusData { + if(!source) { + throw new KMXPlusFileReaderError(KMXPLUS_FILE_READER_ERROR.SOURCE_IS_REQUIRED()); + } + + const version = this.findVersionFromMagic(source); + if(!this.version) { + this.version = version; + this.format = new KMXPlusFileFormat(version); + } else if(version !== this.version) { + throw new KMXPlusFileReaderError(KMXPLUS_FILE_READER_ERROR.VERSION_SHOULD_MATCH_CONSTRUCTOR({version, constructorVersion: this.version})); + } + + const sect = this.format.COMP_PLUS_SECT.fromBuffer(source) as ICOMP_PLUS_SECT; + if(!sect) { + throw new KMXPlusFileReaderError(KMXPLUS_FILE_READER_ERROR.MISSING_SECT()); + } + + if(sect.header.ident != constants.hex_section_id('sect') && sect.header.ident != constants.hex_section_id('sec2')) { + throw new KMXPlusFileReaderError(KMXPLUS_FILE_READER_ERROR.EXPECTED_SECT_OR_SEC2()); + } + + if(source.length < sect.total) { + throw new KMXPlusFileReaderError(KMXPLUS_FILE_READER_ERROR.FILE_IS_TOO_SHORT()); + } + + const kmx: KMXPlus.KMXPlusData = { + sect: {}, + }; + + // strs, list, uset, elem are dependency sections, must be read in this order + this.readSection('strs', sect, source, kmx); + this.readSection('list', sect, source, kmx); + this.readSection('uset', sect, source, kmx); + this.readSection('elem', sect, source, kmx); + // read remaining sections in alphabetical order + this.readSection('bksp', sect, source, kmx); + this.readSection('disp', sect, source, kmx); + this.readSection('keys', sect, source, kmx); + this.readSection('layr', sect, source, kmx); + this.readSection('loca', sect, source, kmx); + this.readSection('meta', sect, source, kmx); + // no need to re-read sect/sec2 + this.readSection('tran', sect, source, kmx); + this.readSection('vars', sect, source, kmx); + + return kmx; + } + + private sectionReaders: {[index in SectionIdent]: any} = { + bksp: this.readBkspSection, + disp: this.readDispSection, + elem: this.readElemSection, + keys: this.readKeysSection, + layr: this.readLayrSection, + list: this.readListSection, + loca: this.readLocaSection, + meta: this.readMetaSection, + sect: null, // this is handled directly by read() + strs: this.readStrsSection, + tran: this.readTranSection, + uset: this.readUsetSection, + vars: this.readVarsSection, + }; + + private readSection(identString: SectionIdent, sect: any, source: Uint8Array, kmx: KMXPlus.KMXPlusData) { + const ident = constants.hex_section_id(identString); + for(let i = 0; i < sect.count; i++) { + if(sect.items[i].sect == ident) { + return this.readSectionData(source.slice(sect.items[i].offset), kmx); + } + } + return null; + } + + private readSectionData(sect: Uint8Array, kmx: KMXPlus.KMXPlusData) { + const header = this.format.COMP_PLUS_SectionHeader.fromBuffer(sect) as ICOMP_PLUS_SectionHeader; + const ident = constants.str_section_id(header.ident) as SectionIdent; + const reader = this.sectionReaders[ident]; + if(typeof reader != 'function') { + throw new Error(`Unsupported section ${ident}`); + } + + kmx[ident] = reader.bind(this)(sect, kmx); + } + + private readString(id: number, kmx: KMXPlus.KMXPlusData) { + return new KMXPlus.StrsItem(kmx.strs.strings[id].value); + } + + private readBkspSection(sect: Uint8Array, kmx: KMXPlus.KMXPlusData) { + return this._readTranSection(new KMXPlus.Bksp(), sect, kmx) as KMXPlus.Bksp; + } + + private readDispSection(sect: Uint8Array, kmx: KMXPlus.KMXPlusData) { + const result = new KMXPlus.Disp(); + if(this.version == KMXPlusVersion.Version17) { + const disp = this.format.COMP_PLUS_DISP_v17.fromBuffer(sect) as ICOMP_PLUS_DISP_v17; + result.baseCharacter = this.readString(disp.baseCharacter ?? 0, kmx); + for(const item of disp.items) { + const resultItem: KMXPlus.DispItem = { + to: this.readString(item.to, kmx), + display: this.readString(item.display, kmx), + flags: item.to ? 0 : KMXPlus.DispItemFlags.isId, + id: this.readString(item.id, kmx), + toId: null, + }; + resultItem.toId = item.to ? resultItem.to : resultItem.id; + result.disps.push(resultItem); + } + } else { + const disp = this.format.COMP_PLUS_DISP_v19.fromBuffer(sect) as ICOMP_PLUS_DISP_v19; + result.baseCharacter = this.readString(disp.baseCharacter ?? 0, kmx); + for(const item of disp.items) { + const toId = this.readString(item.toId, kmx); + const resultItem: KMXPlus.DispItem = { + to: item.flags & KMXPlus.DispItemFlags.isId ? this.readString(0, kmx) : toId, + display: this.readString(item.display, kmx), + flags: item.flags, + id: item.flags & KMXPlus.DispItemFlags.isId ? toId : this.readString(0, kmx), + toId, + }; + result.disps.push(resultItem); + } + } + return result; + } + + private readElemSection(sect: Uint8Array, kmx: KMXPlus.KMXPlusData) { + const elem = this.format.COMP_PLUS_ELEM.fromBuffer(sect) as ICOMP_PLUS_ELEM; + const result = new KMXPlus.Elem({}); + result.strings = []; // remove the default elem, as we'll be adding it on load + for(const string of elem.strings) { + let offset = string.offset; + const resultString = new ElementString(); + for(let i = 0; i < string.length; i++) { + const element = this.format.COMP_PLUS_ELEM_ELEMENT.fromBuffer(sect.slice(offset, offset + constants.length_elem_item_element)) as ICOMP_PLUS_ELEM_ELEMENT; + const type = (element.flags & constants.elem_flags_type); + const resultElement = new ElemElement(); + resultElement.flags = element.flags & constants.elem_flags_flags_mask; + resultElement.order = (element.flags & constants.elem_flags_order_mask) >> constants.elem_flags_order_bitshift; + resultElement.tertiary = (element.flags & constants.elem_flags_tertiary_mask) >> constants.elem_flags_tertiary_bitshift; + if (type === constants.elem_flags_type_char) { + resultElement.value = new KMXPlus.CharStrsItem(String.fromCodePoint(element.element)); + } else if (type == constants.elem_flags_type_str) { + resultElement.value = this.readString(element.element, kmx); + } else if (type == constants.elem_flags_type_uset) { + resultElement.uset = kmx.uset.usets[element.element]; + resultElement.value = kmx.strs.strings[0]; + // TODO-EMBED-OSK-IN-KMX: loading usets is incomplete - see TODO-LDML + } else { + throw new KMXPlusFileReaderError(KMXPLUS_FILE_READER_ERROR.UNKNOWN_ELEMENT_TYPE({type:type.toString(16)})); + } + resultString.push(resultElement); + offset += constants.length_elem_item_element; + } + result.strings.push(resultString); + } + return result; + } + + private readKeysSection(sect: Uint8Array, kmx: KMXPlus.KMXPlusData) { + const keys = this.format.COMP_PLUS_KEYS.fromBuffer(sect) as ICOMP_PLUS_KEYS; + const result = new KMXPlus.Keys(kmx.strs); + + result.flicks = []; // remove the default flick, as we'll be adding it on load + for(const flicks of keys.flicks) { + const resultFlicks = new KMXPlus.KeysFlicks(this.readString(flicks.id, kmx)); + for(let i = flicks.flick; i < flicks.flick + flicks.count; i++) { + const flick = keys.flick[i]; + const resultFlick = new KMXPlus.KeysFlick(); + resultFlick.directions = kmx.list.lists[flick.directions]; + resultFlick.keyId = this.readString(flick.to, kmx); + resultFlicks.flicks.push(resultFlick); + } + result.flicks.push(resultFlicks); + } + + for(const key of keys.keys) { + const resultKey = new KMXPlus.KeysKeys(); + resultKey.flags = key.flags; + resultKey.id = this.readString(key.id, kmx); + if(key.flags & KMXPlus.KeysKeysFlags.extend) { + resultKey.to = this.readString(key.to, kmx); + } else { + resultKey.to = new KMXPlus.StrsItem(String.fromCodePoint(key.to), key.to); + } + resultKey.flicks = result.flicks[key.flicks].id.value; + resultKey.longPress = kmx.list.lists[key.longPress]; + resultKey.longPressDefault = this.readString(key.longPressDefault, kmx); + resultKey.multiTap = kmx.list.lists[key.multiTap]; + resultKey.switch = this.readString(key.switch, kmx); + resultKey.width = key.width; + result.keys.push(resultKey); + } + + for(const kmap of keys.kmap) { + const resultKmap = new KMXPlus.KeysKmap(); + resultKmap.vkey = kmap.vkey; + resultKmap.mod = kmap.mod; + resultKmap.key = result.keys[kmap.key].id.value; + result.kmap.push(resultKmap); + } + return result; + } + + private readLayrSection(sect: Uint8Array, kmx: KMXPlus.KMXPlusData) { + const result = new KMXPlus.Layr(); + if(this.version == KMXPlusVersion.Version17) { + const layr = this.format.COMP_PLUS_LAYR_v17.fromBuffer(sect) as ICOMP_PLUS_LAYR_v17; + for(const form of layr.forms) { + const resultForm = new KMXPlus.LayrForm(); + resultForm.baseLayout = this.readString(0, kmx); + resultForm.flags = 0; + resultForm.hardware = this.readString(form.hardware, kmx); + resultForm.minDeviceWidth = form.minDeviceWidth; + resultForm.fontFaceName = this.readString(0, kmx); + resultForm.fontSizePct = 100; + for(let i = 0; i < form.count; i++) { + const layer = layr.layers[form.layer + i]; + const resultLayer = new KMXPlus.LayrEntry(); + resultLayer.id = this.readString(layer.id, kmx); + resultLayer.mod = layer.mod; + for(let j = 0; j < layer.count; j++) { + const row = layr.rows[layer.row + j]; + const resultRow = new KMXPlus.LayrRow(); + for(let k = 0; k < row.count; k++) { + resultRow.keys.push(this.readString(layr.keys[row.key + k].key, kmx)); + } + resultLayer.rows.push(resultRow); + } + resultForm.layers.push(resultLayer); + } + result.forms.push(resultForm); + } + } else { + const layr = this.format.COMP_PLUS_LAYR_v19.fromBuffer(sect) as ICOMP_PLUS_LAYR_v19; + for(const form of layr.forms) { + const resultForm = new KMXPlus.LayrForm(); + resultForm.baseLayout = this.readString(form.baseLayout, kmx); + resultForm.flags = form.flags; + resultForm.hardware = this.readString(form.hardware, kmx); + resultForm.minDeviceWidth = form.minDeviceWidth; + resultForm.fontFaceName = this.readString(form.fontFaceName, kmx); + resultForm.fontSizePct = form.fontSizePct; + for(let i = 0; i < form.count; i++) { + const layer = layr.layers[form.layer + i]; + const resultLayer = new KMXPlus.LayrEntry(); + resultLayer.id = this.readString(layer.id, kmx); + resultLayer.mod = layer.mod; + for(let j = 0; j < layer.count; j++) { + const row = layr.rows[layer.row + j]; + const resultRow = new KMXPlus.LayrRow(); + for(let k = 0; k < row.count; k++) { + resultRow.keys.push(this.readString(layr.keys[row.key + k].key, kmx)); + } + resultLayer.rows.push(resultRow); + } + resultForm.layers.push(resultLayer); + } + result.forms.push(resultForm); + } + } + return result; + } + + private readListSection(sect: Uint8Array, kmx: KMXPlus.KMXPlusData) { + const list = this.format.COMP_PLUS_LIST.fromBuffer(sect) as ICOMP_PLUS_LIST; + const result = new KMXPlus.List(kmx.strs); + result.lists = []; // remove the default list, as we'll be adding it on load + for(const listItem of list.lists) { + const resultList = new KMXPlus.ListItem(); + for(let i = 0; i < listItem.count; i++) { + const item = new ListIndex(kmx.strs.strings[list.indices[listItem.index + i].str]); + resultList.push(item); + } + result.lists.push(resultList); + } + return result; + } + + private readLocaSection(sect: Uint8Array, kmx: KMXPlus.KMXPlusData) { + const loca = this.format.COMP_PLUS_LOCA.fromBuffer(sect) as ICOMP_PLUS_LOCA; + const result = new KMXPlus.Loca(); + for(const item of loca.items) { + result.locales.push(this.readString(item, kmx)); + } + return result; + } + + private readMetaSection(sect: Uint8Array, kmx: KMXPlus.KMXPlusData) { + const meta = this.format.COMP_PLUS_META.fromBuffer(sect) as ICOMP_PLUS_META; + const result = new KMXPlus.Meta(); + result.author = this.readString(meta.author, kmx); + result.conform = this.readString(meta.conform, kmx); + result.indicator = this.readString(meta.indicator, kmx); + result.layout = this.readString(meta.layout, kmx); + result.name = this.readString(meta.name, kmx); + result.settings = meta.settings; + result.version = this.readString(meta.version, kmx); + return result; + } + + private readStrsSection(sect: Uint8Array, kmx: KMXPlus.KMXPlusData) { + const strs = this.format.COMP_PLUS_STRS.fromBuffer(sect) as ICOMP_PLUS_STRS; + const result = new KMXPlus.Strs(); + result.strings = []; // remove the default string, as we'll be adding it on load + const strReader = new r.String(null, 'utf16le'); + for(const str of strs.items) { + const buffer = strReader.fromBuffer(sect.slice(str.offset, str.offset + str.length * 2)); + result.strings.push(new KMXPlus.StrsItem(buffer)); + } + return result; + } + + private readTranSection(sect: Uint8Array, kmx: KMXPlus.KMXPlusData) { + return this._readTranSection(new KMXPlus.Tran(), sect, kmx); + } + + private _readTranSection(result: KMXPlus.Tran, sect: Uint8Array, kmx: KMXPlus.KMXPlusData) { + const tran = this.format.COMP_PLUS_TRAN.fromBuffer(sect) as ICOMP_PLUS_TRAN; + for(const group of tran.groups) { + const resultGroup = new KMXPlus.TranGroup(); + resultGroup.type = group.type; + if(group.type == constants.tran_group_type_transform) { + for(let i = group.index; i < group.index + group.count; i++) { + const transform = tran.transforms[i]; + const resultTransform = new KMXPlus.TranTransform(); + resultTransform.from = this.readString(transform.from, kmx); + resultTransform.to = this.readString(transform.to, kmx); + resultTransform.mapFrom = this.readString(transform.mapFrom, kmx); + resultTransform.mapTo = this.readString(transform.mapTo, kmx); + resultGroup.transforms.push(resultTransform); + } + } else if(group.type == constants.tran_group_type_reorder) { + for(let i = group.index; i < group.index + group.count; i++) { + const reorder = tran.reorders[i]; + const resultReorder = new KMXPlus.TranReorder(); + resultReorder.before = kmx.elem.strings[reorder.before]; + resultReorder.elements = kmx.elem.strings[reorder.elements]; + resultGroup.reorders.push(resultReorder); + } + } else { + throw new KMXPlusFileReaderError(KMXPLUS_FILE_READER_ERROR.UNKNOWN_GROUP_TYPE(group)); + } + result.groups.push(resultGroup); + } + return result; + } + + private readUsetSection(sect: Uint8Array, kmx: KMXPlus.KMXPlusData) { + const uset = this.format.COMP_PLUS_USET.fromBuffer(sect) as ICOMP_PLUS_USET; + const result = new KMXPlus.Uset(); + for(const item of uset.usets) { + const pattern = this.readString(item.pattern, kmx); + const unicodeSet: UnicodeSet = new UnicodeSet(pattern.value, uset.ranges.slice(item.range, item.range + item.count).map(r => [r.start, r.end])); + const resultUsetItem = new KMXPlus.UsetItem(unicodeSet, pattern); + result.usets.push(resultUsetItem); + } + return result; + } + + private readVarsSection(sect: Uint8Array, kmx: KMXPlus.KMXPlusData) { + const vars = this.format.COMP_PLUS_VARS.fromBuffer(sect) as ICOMP_PLUS_VARS; + const result = new KMXPlus.Vars(); + result.markers = kmx.list.lists[vars.markers]; + for(const v of vars.varEntries) { + const id = this.readString(v.id, kmx); + const value = this.readString(v.value, kmx); + if(v.type == constants.vars_entry_type_string) { + const str = new KMXPlus.StringVarItem(id.value, value.value, kmx); + result.strings.push(str); + } else if(v.type == constants.vars_entry_type_set) { + const set = new KMXPlus.SetVarItem(id.value, value.value.split(' '), kmx); + result.sets.push(set); + } else if(v.type == constants.vars_entry_type_unicodeSet) { + // TODO-EMBED-OSK-IN-KMX: seems like we need to re-parse the unicode set + // on load -- data is not available see also + // /docs/file-formats/kmx-plus-file-format.md#L563 + const usetparser: UnicodeSetParser = { + sizeUnicodeSet: (pattern: string, compileContext?: any): number => -1, // skip parsing + parseUnicodeSet: null + }; + const uset = new KMXPlus.UnicodeSetItem(id.value, value.value, { ...kmx, usetparser }, usetparser); + uset.unicodeSet = null; + result.usets.push(uset); + } else { + throw new KMXPlusFileReaderError(KMXPLUS_FILE_READER_ERROR.UNKNOWN_VAR_TYPE(v)); + } + } + return result; + } + + private findVersionFromMagic(source: Uint8Array) { + if(source.length < constants.length_sect) { + throw new KMXPlusFileReaderError(KMXPLUS_FILE_READER_ERROR.FILE_IS_TOO_SHORT()); + } + + if(source[0] == 0x73 && source[1] == 0x65 && source[2] == 0x63 && source[3] == 0x74) { // 'sect' + return KMXPlusVersion.Version17; + } + + if(source[0] == 0x73 && source[1] == 0x65 && source[2] == 0x63 && source[3] == 0x32) { // 'sec2' + return KMXPlusVersion.Version19; + } + + throw new KMXPlusFileReaderError(KMXPLUS_FILE_READER_ERROR.UNRECOGNIZED_MAGIC()); + } + + /** @internal */ + public unitTestEndpoints = { + readBkspSection: this.readBkspSection.bind(this), + readDispSection: this.readDispSection.bind(this), + readElemSection: this.readElemSection.bind(this), + readKeysSection: this.readKeysSection.bind(this), + readLayrSection: this.readLayrSection.bind(this), + readListSection: this.readListSection.bind(this), + readLocaSection: this.readLocaSection.bind(this), + readMetaSection: this.readMetaSection.bind(this), + readStrsSection: this.readStrsSection.bind(this), + readTranSection: this.readTranSection.bind(this), + readUsetSection: this.readUsetSection.bind(this), + readVarsSection: this.readVarsSection.bind(this), + } +} diff --git a/common/web/types/src/kmx/kmx-plus/kmx-plus-file.ts b/common/web/types/src/kmx/kmx-plus/kmx-plus-file.ts index 924f9b9f9d..fdf1118529 100644 --- a/common/web/types/src/kmx/kmx-plus/kmx-plus-file.ts +++ b/common/web/types/src/kmx/kmx-plus/kmx-plus-file.ts @@ -10,6 +10,318 @@ import * as r from 'restructure'; import KMXFile = KMX.KMXFile; import { KMXPlusVersion } from '@keymanapp/ldml-keyboard-constants'; +/* interfaces that match the COMP_PLUS_* structs -- for type safety */ + +export type IIDENT = number; +export type ISTR_REF = number; +export type ISTR_OR_CHAR32_OR_USET = number; + +export interface ICOMP_PLUS_SectionHeader { + ident: IIDENT; + size: number; + version?: number; +}; + +// 'sect' + +export interface ICOMP_PLUS_SECT_ITEM { + sect: number; + offset: number; +}; + +export interface ICOMP_PLUS_SECT { + header: ICOMP_PLUS_SectionHeader; + total: number; + count: number; + items: ICOMP_PLUS_SECT_ITEM[]; +}; + +// 'bksp' - see 'tran' + +// 'disp' + +export interface ICOMP_PLUS_DISP_ITEM_v17 { + to: ISTR_REF; + id: ISTR_REF; + display: ISTR_REF; +}; + +export interface ICOMP_PLUS_DISP_v17 { + header: ICOMP_PLUS_SectionHeader; + count: number; + baseCharacter: ISTR_REF; + items: ICOMP_PLUS_DISP_ITEM_v17[]; +}; + +export interface ICOMP_PLUS_DISP_ITEM_v19 { + toId: ISTR_REF; + display: ISTR_REF; + flags: number; +}; + +export interface ICOMP_PLUS_DISP_v19 { + header: ICOMP_PLUS_SectionHeader; + count: number; + baseCharacter: ISTR_REF; + items: ICOMP_PLUS_DISP_ITEM_v19[]; +}; + +// 'elem' + +export interface ICOMP_PLUS_ELEM_ELEMENT { + element: ISTR_OR_CHAR32_OR_USET; + flags: number; +}; + +export interface ICOMP_PLUS_ELEM_STRING { + offset: number; + length: number; +}; + +export interface ICOMP_PLUS_ELEM { + header: ICOMP_PLUS_SectionHeader; + count: number; + strings: ICOMP_PLUS_ELEM_STRING[]; + // + variable subtable: Element data (see KMXPlusBuilder.emitElements()) +}; + +// 'keys' + +export type ISTR_OR_CHAR32 = number; +export type ILIST_REF = number; + +export interface ICOMP_PLUS_KEYS_FLICK { + directions: ILIST_REF; // list + to: ISTR_OR_CHAR32; // str | codepoint +}; + +export interface ICOMP_PLUS_KEYS_FLICKS { + count: number; + flick: number; + id: ISTR_REF; // str +}; + +export interface ICOMP_PLUS_KEYS_KEY { + to: ISTR_OR_CHAR32; // str | codepoint + flags: number; + id: ISTR_REF; // str + switch: ISTR_REF; // str + width: number; // width*10 ( 1 = 0.1 keys) + longPress: ILIST_REF; // list index + longPressDefault: ISTR_REF; // str + multiTap: ILIST_REF; // list index + flicks: number; // index into flicks table +}; + +export interface ICOMP_PLUS_KEYS_KMAP { + vkey: number; + mod: number; + key: number; // index into 'keys' subtable +}; + +export interface ICOMP_PLUS_KEYS { + header: ICOMP_PLUS_SectionHeader; + keyCount: number; + flicksCount: number; + flickCount: number; + kmapCount: number; + keys: ICOMP_PLUS_KEYS_KEY[]; + flicks: ICOMP_PLUS_KEYS_FLICKS[]; + flick: ICOMP_PLUS_KEYS_FLICK[]; + kmap: ICOMP_PLUS_KEYS_KMAP[]; +}; + +export interface ICOMP_PLUS_LAYR_ENTRY { + id: ISTR_REF; // str + mod: number; // bitfield + row: number; // index into rows + count: number; +}; + +export interface ICOMP_PLUS_LAYR_KEY { + key: ISTR_REF; // str: key id +}; + +export interface ICOMP_PLUS_LAYR_FORM_v17 { + hardware: ISTR_REF; // str: hardware name + layer: number; // index into layers + count: number; + minDeviceWidth: number; // integer: millimeters +}; + +export interface ICOMP_PLUS_LAYR_FORM_v19 { + hardware: ISTR_REF; // str: hardware name + layer: number; // index into layers + count: number; + minDeviceWidth: number; // integer: millimeters + baseLayout: ISTR_REF; // v19: str: identifier for base layout (reserved) + fontFaceName: ISTR_REF; // v19: str: font face name + fontSizePct: number; // v19: font size in % of default size + flags: number; // v19: flags +}; + +export interface ICOMP_PLUS_LAYR_ROW { + key: number; + count: number; +}; + +export interface ICOMP_PLUS_LAYR_v17 { + header: ICOMP_PLUS_SectionHeader; + formCount: number; + layerCount: number; + rowCount: number; + keyCount: number; + forms: ICOMP_PLUS_LAYR_FORM_v17[]; + layers: ICOMP_PLUS_LAYR_ENTRY[]; + rows: ICOMP_PLUS_LAYR_ROW[]; + keys: ICOMP_PLUS_LAYR_KEY[]; +}; + +export interface ICOMP_PLUS_LAYR_v19 { + header: ICOMP_PLUS_SectionHeader; + formCount: number; + layerCount: number; + rowCount: number; + keyCount: number; + forms: ICOMP_PLUS_LAYR_FORM_v19[]; + layers: ICOMP_PLUS_LAYR_ENTRY[]; + rows: ICOMP_PLUS_LAYR_ROW[]; + keys: ICOMP_PLUS_LAYR_KEY[]; +}; + +// 'list' + +export interface ICOMP_PLUS_LIST_LIST { + index: number; + count: number; +}; + +export interface ICOMP_PLUS_LIST_INDEX { + str: ISTR_REF; // str +}; + +export interface ICOMP_PLUS_LIST { + header: ICOMP_PLUS_SectionHeader; + listCount: number; + indexCount: number; + lists: ICOMP_PLUS_LIST_LIST[]; + indices: ICOMP_PLUS_LIST_INDEX[]; +}; + +// 'loca' + +export type ICOMP_PLUS_LOCA_ITEM = ISTR_REF; //str + +export interface ICOMP_PLUS_LOCA { + header: ICOMP_PLUS_SectionHeader; + count: number; + items: ICOMP_PLUS_LOCA_ITEM[]; +}; + +// 'meta' + +export interface ICOMP_PLUS_META { + header: ICOMP_PLUS_SectionHeader; + author: ISTR_REF; //str + conform: ISTR_REF; //str + layout: ISTR_REF; //str + name: ISTR_REF; //str + indicator: ISTR_REF; //str + version: ISTR_REF; //str + settings: number; //new r.Bitfield(number, ['normalizationDisabled']) +}; + +// 'strs' + +export interface ICOMP_PLUS_STRS_ITEM { + // While we use length which is number of utf-16 code units excluding null terminator, + // we always write a null terminator, so we can get restructure to do that for us here + offset: number; //? new r.Pointer(number, new r.String(null, 'utf16le')), + length: number; +}; + +export interface ICOMP_PLUS_STRS { + header: ICOMP_PLUS_SectionHeader; + count: number; + items: ICOMP_PLUS_STRS_ITEM[]; + // + variable subtable: String data (see KMXPlusBuilder.emitStrings()) +}; + +// 'tran' + +export interface ICOMP_PLUS_TRAN_GROUP { + type: number; //type of group + count: number; //number of items + index: number; //index into subtable +}; + +export type IELEM_REF = number; + +export interface ICOMP_PLUS_TRAN_TRANSFORM { + from: ISTR_REF; //str + to: ISTR_REF; //str + mapFrom: IELEM_REF; //elem + mapTo: IELEM_REF; //elem +}; + +export interface ICOMP_PLUS_TRAN_REORDER { + elements: IELEM_REF; //elem + before: IELEM_REF; //elem +}; + +export interface ICOMP_PLUS_TRAN { + header: ICOMP_PLUS_SectionHeader; + groupCount: number; + transformCount: number; + reorderCount: number; + groups: ICOMP_PLUS_TRAN_GROUP[]; + transforms: ICOMP_PLUS_TRAN_TRANSFORM[]; + reorders: ICOMP_PLUS_TRAN_REORDER[]; +}; + +// 'uset' + +export interface ICOMP_PLUS_USET_USET { + range: number; + count: number; + pattern: ISTR_REF; // str +}; + +export type ICHAR32 = number; + +export interface ICOMP_PLUS_USET_RANGE { + start: ICHAR32; + end: ICHAR32; +}; + +export interface ICOMP_PLUS_USET { + header: ICOMP_PLUS_SectionHeader; + usetCount: number; + rangeCount: number; + usets: ICOMP_PLUS_USET_USET[]; + ranges: ICOMP_PLUS_USET_RANGE[]; +}; + +// 'vars' + +export interface ICOMP_PLUS_VARS_ITEM { + type: number; + id: ISTR_REF; // str + value: ISTR_REF; // str + elem: IELEM_REF; +}; + +export interface ICOMP_PLUS_VARS { + header: ICOMP_PLUS_SectionHeader; + markers: ILIST_REF; + varCount: number; + varEntries: ICOMP_PLUS_VARS_ITEM[]; +}; + +// Aliases + +export type ICOMP_PLUS_BKSP = ICOMP_PLUS_TRAN; + /** * Binary representation of KMX+ data, using Restructure. These structures * should be directly used only by KMX+ file readers and writers; in general, @@ -78,7 +390,7 @@ export class KMXPlusFileFormat extends KMXFile { public readonly COMP_PLUS_VARS: any; public readonly COMP_PLUS_VARS_ITEM: any; - private readonly COMP_PLUS_SectionHeader: any; + public readonly COMP_PLUS_SectionHeader: any; constructor(public readonly version: KMXPlusVersion) { super(); @@ -138,7 +450,7 @@ export class KMXPlusFileFormat extends KMXFile { this.COMP_PLUS_DISP_v17 = new r.Struct({ header: this.COMP_PLUS_SectionHeader, count: r.uint32le, - baseCharacter: CHAR32, + baseCharacter: STR_REF, items: new r.Array(this.COMP_PLUS_DISP_ITEM_v17, 'count'), }); @@ -151,7 +463,7 @@ export class KMXPlusFileFormat extends KMXFile { this.COMP_PLUS_DISP_v19 = new r.Struct({ header: this.COMP_PLUS_SectionHeader, count: r.uint32le, - baseCharacter: CHAR32, + baseCharacter: STR_REF, items: new r.Array(this.COMP_PLUS_DISP_ITEM_v19, 'count'), }); @@ -181,14 +493,14 @@ export class KMXPlusFileFormat extends KMXFile { // 'layr' this.COMP_PLUS_LAYR_ENTRY = new r.Struct({ - id: r.uint32le, // str + id: STR_REF, // str mod: r.uint32le, // bitfield row: r.uint32le, // index into rows count: r.uint32le, }); this.COMP_PLUS_LAYR_KEY = new r.Struct({ - key: r.uint32le, // str: key id + key: STR_REF, // str: key id }); this.COMP_PLUS_LAYR_FORM_v17 = new r.Struct({ diff --git a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts index 2aa6a06a84..2868949551 100644 --- a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts +++ b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts @@ -3,7 +3,7 @@ * * KMX+ file format structures and helper functions */ -import { constants } from '@keymanapp/ldml-keyboard-constants'; +import { constants, KMXPlusVersion, SectionIdent } from '@keymanapp/ldml-keyboard-constants'; import { ElementString } from './element-string.js'; import { ListItem } from '../../ldml-keyboard/string-list.js'; import * as util from '../../util/util.js'; @@ -22,6 +22,7 @@ import { KMXPlusFileFormat } from './kmx-plus-file.js'; // Reader in kmx-loader.ts export class Section { + static dependencies: SectionIdent[] = []; } /** @@ -46,6 +47,8 @@ export class Elem extends Section { super(); this.strings.push(ElementString.fromStrings(sections, {}, '')); // C7043: null element string } + static override dependencies: SectionIdent[] = ['strs', 'uset']; + /** * @param source if a string array, does not get reinterpreted as UnicodeSet. This is used with vars, etc. Or pass `["str"]` for an explicit 1-element elem. * If it is a string, will be interpreted per reorder element ruls. @@ -68,6 +71,7 @@ export class Elem extends Section { export class Loca extends Section { locales: StrsItem[] = []; + static override dependencies: SectionIdent[] = ['strs']; }; // 'meta' @@ -86,6 +90,8 @@ export class Meta extends Section { version: StrsItem; // semver version string, defaults to "0" settings: KeyboardSettings; + static override dependencies: SectionIdent[] = ['strs']; + /** convenience for checking settings */ get normalizationDisabled() { return this?.settings & KeyboardSettings.normalizationDisabled; @@ -265,6 +271,8 @@ export class Vars extends Section { sets: SetVarItem[] = []; usets: UnicodeSetItem[] = []; + static override dependencies: SectionIdent[] = ['strs', 'list', 'uset', 'elem']; + /** * * @returns false if any invalid variables @@ -374,13 +382,12 @@ export class Vars extends Section { * Common base for variable sections * See Variable */ -export class VarsItem extends Section { +export class VarsItem { id: StrsItem; value: StrsItem; compileContext?: any; constructor(id: string, value: string, sections: DependencySections, compileContext?: any) { - super(); this.id = sections.strs.allocString(id); this.value = sections.strs.allocString(value, { unescape: true }); this.compileContext = compileContext; @@ -392,6 +399,7 @@ export class VarsItem extends Section { }; export class UnicodeSetItem extends VarsItem { + // TODO-EMBED-OSK-IN-KMX: second usetparser ref (outside `sections`) is never used, cleanup constructor(id: string, value: string, sections: DependencySections, usetparser: UnicodeSetParser, compileContext?: any) { super(id, value, sections, compileContext); const needRanges = sections.usetparser.sizeUnicodeSet(value); @@ -453,6 +461,7 @@ export class Tran extends Section { get id() { return constants.section.tran; } + static override dependencies: SectionIdent[] = ['strs', 'list', 'uset', 'elem']; }; export class UsetItem { @@ -465,6 +474,7 @@ export class UsetItem { export class Uset extends Section { usets: UsetItem[] = []; + static override dependencies: SectionIdent[] = ['strs']; allocUset(set: UnicodeSet, sections: DependencySections, compileContext?: any) : UsetItem { // match the same pattern let result = this.usets.find(s => set.pattern == s.uset.pattern); @@ -575,6 +585,7 @@ export interface DispItem { export class Disp extends Section { baseCharacter: StrsItem; disps: DispItem[] = []; + static override dependencies: SectionIdent[] = ['strs']; }; // 'layr' @@ -624,6 +635,7 @@ export class LayrForm { export class Layr extends Section { forms: LayrForm[] = []; + static override dependencies: SectionIdent[] = ['strs']; }; export enum KeysKeysFlags { @@ -682,6 +694,7 @@ export class Keys extends Section { const nullFlicks = new KeysFlicks(strs.allocString('')); this.flicks.push(nullFlicks); // C7043: null element string } + static override dependencies: SectionIdent[] = ['strs', 'list']; }; export class List extends Section { @@ -723,6 +736,7 @@ export class List extends Section { super(); this.lists.push(ListItem.fromStrings([], {}, { strs })); // C7043: null element string } + static override dependencies: SectionIdent[] = ['strs']; lists: ListItem[] = []; }; @@ -751,4 +765,27 @@ export interface KMXPlusData { export class KMXPlusFile extends KMXPlusFileFormat { /* File in-memory data */ public kmxplus: KMXPlusData = { }; + static createEmptyMinimalKMXPlusFile(version: KMXPlusVersion): KMXPlusFile { + const kmx = new KMXPlusFile(version); + const strs = kmx.kmxplus.strs = new Strs(); + kmx.kmxplus.layr = new Layr(); + kmx.kmxplus.elem = new Elem(kmx.kmxplus); + kmx.kmxplus.disp = new Disp(); + kmx.kmxplus.keys = new Keys(strs); + kmx.kmxplus.list = new List(strs); + kmx.kmxplus.loca = new Loca(); + + kmx.kmxplus.meta = new Meta(); + kmx.kmxplus.meta.author = strs.allocString(); + kmx.kmxplus.meta.conform = strs.allocString(); + kmx.kmxplus.meta.indicator = strs.allocString(); + kmx.kmxplus.meta.layout = strs.allocString(); + kmx.kmxplus.meta.name = strs.allocString(); + kmx.kmxplus.meta.settings = 0; + kmx.kmxplus.meta.version = strs.allocString(); + + // Are there other sections we need? + + return kmx; + } }; diff --git a/common/web/types/src/main.ts b/common/web/types/src/main.ts index 924f109562..f527985070 100644 --- a/common/web/types/src/main.ts +++ b/common/web/types/src/main.ts @@ -1,3 +1,7 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + */ + export * as KMX from './kmx/kmx.js'; export { KmxFileReader, KmxFileReaderError } from './kmx/kmx-file-reader.js'; export * as KeymanTargets from './kmx/keyman-targets.js'; @@ -7,6 +11,7 @@ export { default as KvkFileReader } from './kvk/kvk-file-reader.js'; export { default as KvkFileWriter } from './kvk/kvk-file-writer.js'; export * as KvkFile from './kvk/kvk-file.js'; +export { KMXPlusFileReader } from './kmx/kmx-plus/kmx-plus-file-reader.js'; export { USVirtualKeyCodes, usVirtualKeyName } from './consts/virtual-key-constants.js'; export * as Constants from './consts/virtual-key-constants.js'; diff --git a/common/web/types/src/util/errors.ts b/common/web/types/src/util/errors.ts index b4b35af32f..368f2eaf00 100644 --- a/common/web/types/src/util/errors.ts +++ b/common/web/types/src/util/errors.ts @@ -1,6 +1,11 @@ -/** +/* + * Keyman is copyright (C) SIL Global. MIT License. + * * Base class for all common/web/types errors thrown */ export class KeymanTypesError extends Error { - + constructor(message?: string, options?: ErrorOptions) { + super(message, options); + this.name = this.constructor.name; + } } \ No newline at end of file diff --git a/common/web/types/tests/helpers/index.ts b/common/web/types/tests/helpers/index.ts index 717b2e7519..bd3a1336cb 100644 --- a/common/web/types/tests/helpers/index.ts +++ b/common/web/types/tests/helpers/index.ts @@ -2,7 +2,7 @@ import * as path from "path"; import { fileURLToPath } from "url"; /** - * Builds a path to the fixture with the given path components. + * Builds a path to the common/web/types fixture with the given path components. * * e.g., makePathToFixture('basic.xml') * @@ -11,3 +11,16 @@ import { fileURLToPath } from "url"; export function makePathToFixture(...components: string[]): string { return fileURLToPath(new URL(path.join('..', '..', '..', 'tests', 'fixtures', ...components), import.meta.url)); } + +/** + * Builds a path to the /common/test file with the given path components. Note + * that this links to the base of /common/test, not /common/test/fixtures, + * because the /common/test folder currently has a mix of paths. + * + * e.g., makePathToFixture('basic.xml') + * + * @param components One or more path components. + */ +export function makePathToCommonFixture(...components: string[]): string { + return fileURLToPath(new URL(path.join('..', '..', '..', '..', '..', 'test', ...components), import.meta.url)); +} diff --git a/common/web/types/tests/kmx/kmx-plus/kmx-plus-file-reader.tests.ts b/common/web/types/tests/kmx/kmx-plus/kmx-plus-file-reader.tests.ts new file mode 100644 index 0000000000..cd1bd3a7a8 --- /dev/null +++ b/common/web/types/tests/kmx/kmx-plus/kmx-plus-file-reader.tests.ts @@ -0,0 +1,409 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * KMX+ file reader unit tests + */ + +import 'mocha'; +import { assert } from 'chai'; +import { hextobinFromFile } from '@keymanapp/hextobin'; +import { constants, KMXPlusVersion, SECTION_IDENTS, SectionIdent } from '@keymanapp/ldml-keyboard-constants'; +import { KMXPlus } from '../../../src/main.js'; +import { KMXPLUS_FILE_READER_ERROR, KMXPlusFileReader } from '../../../src/kmx/kmx-plus/kmx-plus-file-reader.js'; +import { makePathToCommonFixture } from '../../helpers/index.js'; + +function sectToSectionClass(ident: SectionIdent) { + return ident[0].toUpperCase()+ident.slice(1); +} + +function sectToSectionReader(ident: SectionIdent) { + return `read${sectToSectionClass(ident)}Section`; +} + +describe('KMXPlusFileReader', function() { + + it('should throw when no data is provided', function() { + const reader = new KMXPlusFileReader(); + assert.throws(() => reader.read(null), KMXPLUS_FILE_READER_ERROR.SOURCE_IS_REQUIRED()); + }); + + it('should throw when the input file is too short', function() { + const reader = new KMXPlusFileReader(); + const file = new Uint8Array([0,0,0]); + assert.throws(() => reader.read(file), KMXPLUS_FILE_READER_ERROR.FILE_IS_TOO_SHORT()); + }); + + it('should throw when the input file has invalid magic', function() { + const reader = new KMXPlusFileReader(); + const file = new Uint8Array([0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]); + assert.throws(() => reader.read(file), KMXPLUS_FILE_READER_ERROR.UNRECOGNIZED_MAGIC()); + }); + + [[17, KMXPlusVersion.Version17], [19, KMXPlusVersion.Version19]].forEach( ([versionMajor, version]) => { + function preloadSections(sections: SectionIdent[], thisSection: SectionIdent) { + const kmx: KMXPlus.KMXPlusData = {}; + const path = makePathToCommonFixture('keyboards', 'kmx-plus', `basic-${versionMajor}.txt`); + const reader = new KMXPlusFileReader(version); + for(const section of [...sections, thisSection]) { + const fixture = hextobinFromFile(path, null, { silent: true, startBlock: section, endBlock: 'end'+section }); + kmx[section] = (reader.unitTestEndpoints as any)[sectToSectionReader(section)](fixture, kmx); + } + return kmx; + } + + SECTION_IDENTS.forEach((ident: SectionIdent) => { + if(ident == 'sect') { + // we don't have a separate test for 'sect' -- it is effectively tested + // automatically by any other section test + return; + } + + it(`should parse a v${versionMajor} '${ident}' section`, async function() { + const kmx = preloadSections((KMXPlus as any)[sectToSectionClass(ident)].dependencies, ident); + test_endpoints[ident](kmx[ident]); + }); + + }); + + it(`should read a v${versionMajor} KMX+ file into memory`, async function() { + // Note: see developer/src/common/web/utils for a round-trip-test + const path = makePathToCommonFixture('keyboards', 'kmx-plus', `basic-${versionMajor}.txt`); + const input = hextobinFromFile(path, null, { silent: true }); + const reader = new KMXPlusFileReader(); + const result = reader.readFromKmx(input); + assert.isObject(result); + for(const ident of SECTION_IDENTS) { + test_endpoints[ident](result[ident]); + } + }); + }); + + const test_endpoints: {[index in SectionIdent]: Function} = { + 'sect': function test_sect(sect: KMXPlus.Sect) { + // nothing to test + assert.isTrue(true); + }, + + 'bksp': function test_bksp(bksp: KMXPlus.Bksp) { + assert.equal(bksp.id, 'bksp'); + assert.lengthOf(bksp.groups, 1); + assert.equal(bksp.groups[0].type, constants.tran_group_type_transform); + assert.lengthOf(bksp.groups[0].transforms, 1); + assert.lengthOf(bksp.groups[0].reorders, 0); + + assert.equal(bksp.groups[0].transforms[0].from.value, '^e'); + assert.equal(bksp.groups[0].transforms[0].to.value, ''); + assert.equal(bksp.groups[0].transforms[0].mapFrom.value, ''); + assert.equal(bksp.groups[0].transforms[0].mapTo.value, ''); + }, + + 'disp': function test_disp(disp: KMXPlus.Disp) { + assert.isObject(disp); + assert.equal(disp.baseCharacter.value, 'e'); + assert.isArray(disp.disps); + assert.lengthOf(disp.disps, 2); // counted from basic-17.txt + + assert.equal(disp.disps[0].display.value, '^'); + assert.equal(disp.disps[0].flags, 0); + assert.equal(disp.disps[0].id.value, ''); + assert.equal(disp.disps[0].to.value, 'a'); + assert.equal(disp.disps[0].toId.value, 'a'); + + assert.equal(disp.disps[1].display.value, '^e'); + assert.equal(disp.disps[1].flags, KMXPlus.DispItemFlags.isId); + assert.equal(disp.disps[1].id.value, 'e'); + assert.equal(disp.disps[1].to.value, ''); + assert.equal(disp.disps[1].toId.value, 'e'); + }, + + 'elem': function test_elem(elem: KMXPlus.Elem) { + assert.lengthOf(elem.strings, 4); + + assert.lengthOf(elem.strings[0], 0); + + // 'a b c' set + + assert.lengthOf(elem.strings[1], 3); + + assert.equal(elem.strings[1][0].flags, constants.elem_flags_type_char); + assert.equal(elem.strings[1][0].order, 0); + assert.equal(elem.strings[1][0].tertiary, 0); + assert.isUndefined(elem.strings[1][0].uset); + assert.equal(elem.strings[1][0].value.value, 'a'); + + assert.equal(elem.strings[1][1].flags, constants.elem_flags_type_char); + assert.equal(elem.strings[1][1].order, 0); + assert.equal(elem.strings[1][1].tertiary, 0); + assert.isUndefined(elem.strings[1][1].uset); + assert.equal(elem.strings[1][1].value.value, 'b'); + + assert.equal(elem.strings[1][2].flags, constants.elem_flags_type_char); + assert.equal(elem.strings[1][2].order, 0); + assert.equal(elem.strings[1][2].tertiary, 0); + assert.isUndefined(elem.strings[1][2].uset); + assert.equal(elem.strings[1][2].value.value, 'c'); + + // from="\u{1A60}[\u1A75-\u1A79]\u{1A45}" order="10 55 10" + + assert.lengthOf(elem.strings[2], 3); + + assert.equal(elem.strings[2][0].flags, constants.elem_flags_type_char); + assert.equal(elem.strings[2][0].order, 10); + assert.equal(elem.strings[2][0].tertiary, 0); + assert.isUndefined(elem.strings[2][0].uset); + assert.equal(elem.strings[2][0].value.value, '\u{1A60}'); + + assert.equal(elem.strings[2][1].flags, constants.elem_flags_type_uset); + assert.equal(elem.strings[2][1].order, 55); + assert.equal(elem.strings[2][1].tertiary, 0); + assert.equal(elem.strings[2][1].uset.str.value, '[\\u1A75-\\u1A79]'); + assert.equal(elem.strings[2][1].value.value, ''); + + assert.equal(elem.strings[2][2].flags, constants.elem_flags_type_char); + assert.equal(elem.strings[2][2].order, 10); + assert.equal(elem.strings[2][2].tertiary, 0); + assert.isUndefined(elem.strings[2][2].uset); + assert.equal(elem.strings[2][2].value.value, '\u{1A45}'); + + // before="\u{1A6B}" + + assert.lengthOf(elem.strings[3], 1); + + assert.equal(elem.strings[3][0].flags, constants.elem_flags_type_char); + assert.equal(elem.strings[3][0].order, 0); + assert.equal(elem.strings[3][0].tertiary, 0); + assert.isUndefined(elem.strings[3][0].uset); + assert.equal(elem.strings[3][0].value.value, '\u{1A6B}'); + }, + + 'keys': function test_keys(keys: KMXPlus.Keys) { + assert.isObject(keys); + assert.isArray(keys.flicks); + assert.isArray(keys.keys); + assert.isArray(keys.kmap); + assert.lengthOf(keys.flicks, 1); // counted from basic-17.txt + assert.equal(keys.flicks[0].id.value, ''); + assert.lengthOf(keys.flicks[0].flicks, 0); + + assert.lengthOf(keys.keys, 5); // counted from basic-17.txt + assert.lengthOf(keys.kmap, 48); // counted from basic-17.txt + + assert.equal(keys.keys[0].id.value, 'a'); + assert.equal(keys.keys[0].to.value, 'a'); + assert.equal(keys.keys[0].flags, 0); + assert.lengthOf(keys.keys[0].flicks, 0); + assert.lengthOf(keys.keys[0].longPress, 0); + assert.equal(keys.keys[0].longPressDefault.value, ''); + assert.equal(keys.keys[0].multiTap.length, 0); + assert.equal(keys.keys[0].switch.value, ''); + assert.equal(keys.keys[0].width, 10); + + assert.equal(keys.keys[1].id.value, 'e'); + assert.equal(keys.keys[1].to.value, 'e'); + assert.equal(keys.keys[1].flags, 0); + assert.lengthOf(keys.keys[1].flicks, 0); + assert.lengthOf(keys.keys[1].longPress, 0); + assert.equal(keys.keys[1].longPressDefault.value, ''); + assert.equal(keys.keys[1].multiTap.length, 0); + assert.equal(keys.keys[1].switch.value, ''); + assert.equal(keys.keys[1].width, 10); + + assert.equal(keys.keys[2].id.value, 'gap (reserved)'); + assert.equal(keys.keys[2].to.value, ''); + assert.equal(keys.keys[2].flags, KMXPlus.KeysKeysFlags.gap | KMXPlus.KeysKeysFlags.extend); + assert.lengthOf(keys.keys[2].flicks, 0); + assert.lengthOf(keys.keys[2].longPress, 0); + assert.equal(keys.keys[2].longPressDefault.value, ''); + assert.equal(keys.keys[2].multiTap.length, 0); + assert.equal(keys.keys[2].switch.value, ''); + assert.equal(keys.keys[2].width, 10); + + assert.equal(keys.keys[3].id.value, 'hmaqtugha'); + assert.equal(keys.keys[3].to.value, 'ħ'); + assert.equal(keys.keys[3].flags, 0); + assert.lengthOf(keys.keys[3].flicks, 0); + assert.lengthOf(keys.keys[3].longPress, 2); + assert.equal(keys.keys[3].longPress[0].value.value, 'a'); + assert.equal(keys.keys[3].longPress[1].value.value, 'e'); + assert.equal(keys.keys[3].longPressDefault.value, ''); + assert.equal(keys.keys[3].multiTap.length, 0); + assert.equal(keys.keys[3].switch.value, ''); + assert.equal(keys.keys[3].width, 10); + + assert.equal(keys.keys[4].id.value, 'that'); + assert.equal(keys.keys[4].to.value, 'ថា'); + assert.equal(keys.keys[4].flags, KMXPlus.KeysKeysFlags.extend); + assert.lengthOf(keys.keys[4].flicks, 0); + assert.lengthOf(keys.keys[4].longPress, 0); + assert.equal(keys.keys[4].longPressDefault.value, ''); + assert.equal(keys.keys[4].multiTap.length, 0); + assert.equal(keys.keys[4].switch.value, ''); + assert.equal(keys.keys[4].width, 10); + + // TODO-EMBED-OSK-IN-KMX: add flicks + + assert.equal(keys.kmap[0].vkey, 0x20); + assert.equal(keys.kmap[0].mod, 0); + assert.equal(keys.kmap[0].key, 'gap (reserved)'); + + assert.equal(keys.kmap[2].vkey, 0x31); + assert.equal(keys.kmap[2].mod, 0); + assert.equal(keys.kmap[2].key, 'that'); + }, + + 'layr': function test_layr(layr: KMXPlus.Layr) { + assert.isObject(layr); + assert.isArray(layr.forms); + assert.lengthOf(layr.forms, 1); // counted from basic-17.txt + + assert.equal(layr.forms[0].baseLayout.value, ''); + assert.equal(layr.forms[0].flags, 0); + assert.equal(layr.forms[0].fontFaceName.value, ''); + assert.equal(layr.forms[0].fontSizePct, 100); + assert.equal(layr.forms[0].hardware.value, 'us'); + assert.equal(layr.forms[0].minDeviceWidth, 123); + + assert.lengthOf(layr.forms[0].layers, 1); + assert.equal(layr.forms[0].layers[0].id.value, ''); + assert.equal(layr.forms[0].layers[0].mod, 0); + + assert.lengthOf(layr.forms[0].layers[0].rows, 1); + assert.lengthOf(layr.forms[0].layers[0].rows[0].keys, 2); + + assert.equal(layr.forms[0].layers[0].rows[0].keys[0].value, 'hmaqtugha'); + assert.equal(layr.forms[0].layers[0].rows[0].keys[1].value, 'that'); + }, + + 'list': function test_list(list: KMXPlus.List) { + assert.isObject(list); + assert.isArray(list.lists); + assert.lengthOf(list.lists, 3); // counted from basic-17.txt + + assert.lengthOf(list.lists[0], 0); + assert.lengthOf(list.lists[1], 1); + assert.lengthOf(list.lists[2], 2); + + assert.equal(list.lists[1][0].value.value, 'a'); + + assert.equal(list.lists[2][0].value.value, 'a'); + assert.equal(list.lists[2][1].value.value, 'e'); + }, + + 'loca': function test_loca(loca: KMXPlus.Loca) { + assert.isObject(loca); + assert.lengthOf(loca.locales, 1); + assert.equal(loca.locales[0].value, 'mt'); + }, + + 'meta': function test_meta(meta: KMXPlus.Meta) { + assert.isObject(meta); + assert.equal(meta.author.value, 'srl295'); + assert.equal(meta.conform.value, '45'); + assert.equal(meta.indicator.value, '🙀'); + assert.equal(meta.layout.value, 'qwerty'); + assert.equal(meta.name.value, 'TestKbd'); + assert.equal(meta.settings, 0); + assert.equal(meta.version.value, '1.0.0'); + }, + + 'strs': function test_strs(strs: KMXPlus.Strs) { + assert.isObject(strs); + assert.isArray(strs.strings); + assert.lengthOf(strs.strings, 29); // counted from basic-17.txt + // Check a few strings + assert.equal(strs.strings[0].value, ''); + assert.equal(strs.strings[1].value, '1.0.0'); + assert.equal(strs.strings[2].value, '45'); + assert.equal(strs.strings[3].value, 'TestKbd'); + assert.equal(strs.strings[27].value, '🙀'); + }, + + 'tran': function test_tran(tran: KMXPlus.Tran) { + assert.equal(tran.id, 'tran'); + assert.lengthOf(tran.groups, 3); + assert.equal(tran.groups[0].type, constants.tran_group_type_transform); + assert.lengthOf(tran.groups[0].transforms, 2); + assert.lengthOf(tran.groups[0].reorders, 0); + + assert.equal(tran.groups[0].transforms[0].from.value, '^a'); + assert.equal(tran.groups[0].transforms[0].to.value, '\u{0061}\u{0302}'); // 'â' + assert.equal(tran.groups[0].transforms[0].mapFrom.value, ''); // TODO-LDML: no usage of mapFrom, mapTo in our tests? + assert.equal(tran.groups[0].transforms[0].mapTo.value, ''); + + assert.equal(tran.groups[0].transforms[1].from.value, 'a'); + assert.equal(tran.groups[0].transforms[1].to.value, '\u{FFFF}\u{0008}\u{0001}'); // UC_SENTINEL CODE_DEADKEY U+0001 + assert.equal(tran.groups[0].transforms[1].mapFrom.value, ''); + assert.equal(tran.groups[0].transforms[1].mapTo.value, ''); + + assert.equal(tran.groups[1].type, constants.tran_group_type_transform); + assert.lengthOf(tran.groups[1].transforms, 1); + assert.lengthOf(tran.groups[1].reorders, 0); + + assert.equal(tran.groups[1].transforms[0].from.value, '\\uffff\\u0008\\u0001'); + assert.equal(tran.groups[1].transforms[0].to.value, ''); + assert.equal(tran.groups[1].transforms[0].mapFrom.value, ''); + assert.equal(tran.groups[1].transforms[0].mapTo.value, ''); + + assert.equal(tran.groups[2].type, constants.tran_group_type_reorder); + assert.lengthOf(tran.groups[2].transforms, 0); + assert.lengthOf(tran.groups[2].reorders, 1); + + // Verify that we are connected to the correct elem, no need to check every aspect of elem + assert.lengthOf(tran.groups[2].reorders[0].before, 1); + assert.equal(tran.groups[2].reorders[0].before[0].value.value, '\u{1A6B}'); + assert.lengthOf(tran.groups[2].reorders[0].elements, 3); + assert.equal(tran.groups[2].reorders[0].elements[0].value.value, '\u{1A60}'); + }, + + 'uset': function test_uset(uset: KMXPlus.Uset) { + assert.lengthOf(uset.usets, 1); + assert.equal(uset.usets[0].str.value, '[\\u1A75-\\u1A79]'); + assert.equal(uset.usets[0].uset.pattern, '[\\u1A75-\\u1A79]'); + assert.lengthOf(uset.usets[0].uset.ranges, 1); + assert.lengthOf(uset.usets[0].uset.ranges[0], 2); + assert.equal(uset.usets[0].uset.ranges[0][0], 0x1A75); // from + assert.equal(uset.usets[0].uset.ranges[0][1], 0x1A79); // from + + // TODO-EMBED-OSK-IN-KMX: other usets and variables not used in transforms in basic-17.kmx? + }, + + 'vars': function test_vars(vars: KMXPlus.Vars) { + assert.lengthOf(vars.markers, 1); + assert.equal(vars.markers[0].value.value, 'a'); + + assert.lengthOf(vars.strings, 2); + assert.equal(vars.strings[0].id.value, 'a'); + assert.equal(vars.strings[0].value.value, '\\m{a}'); + assert.equal(vars.strings[1].id.value, 'vst'); + assert.equal(vars.strings[1].value.value, 'abc'); + + assert.lengthOf(vars.sets, 1); + assert.equal(vars.sets[0].id.value, 'vse'); + assert.equal(vars.sets[0].value.value, 'a b c'); + assert.lengthOf(vars.sets[0].items, 3); + + assert.equal(vars.sets[0].items[0].flags, 0); + assert.isUndefined(vars.sets[0].items[0].uset); + assert.equal(vars.sets[0].items[0].value.value, 'a'); + + assert.equal(vars.sets[0].items[1].flags, 0); + assert.isUndefined(vars.sets[0].items[1].uset); + assert.equal(vars.sets[0].items[1].value.value, 'b'); + + assert.equal(vars.sets[0].items[2].flags, 0); + assert.isUndefined(vars.sets[0].items[2].uset); + assert.equal(vars.sets[0].items[2].value.value, 'c'); + + assert.lengthOf(vars.usets, 1); + assert.equal(vars.usets[0].id.value, 'vus'); + // TODO-LDML: uset variable never stored in uset data? should use `elem` prop + // assert.equal(vars.usets[0].unicodeSet.pattern, '[abc]'); + // assert.lengthOf(vars.usets[0].unicodeSet.ranges, 1); + // assert.lengthOf(vars.usets[0].unicodeSet.ranges[0], 2); + // assert.equal(vars.usets[0].unicodeSet.ranges[0][0], 0x0061); + // assert.equal(vars.usets[0].unicodeSet.ranges[0][1], 0x0063); + assert.equal(vars.usets[0].value.value, '[abc]'); + } + }; + +}); \ No newline at end of file diff --git a/common/web/types/tests/tsconfig.json b/common/web/types/tests/tsconfig.json index 45e5311fdc..9c0bdc711f 100644 --- a/common/web/types/tests/tsconfig.json +++ b/common/web/types/tests/tsconfig.json @@ -16,6 +16,7 @@ "references": [ { "path": "../../keyman-version" }, { "path": "../../../../core/include/ldml/"}, + { "path": "../../../tools/hextobin/" }, { "path": "../" }, ], } diff --git a/core/include/ldml/keyman_core_ldml.h b/core/include/ldml/keyman_core_ldml.h index ef08f8651b..12a6875489 100644 --- a/core/include/ldml/keyman_core_ldml.h +++ b/core/include/ldml/keyman_core_ldml.h @@ -101,6 +101,7 @@ #define LDML_DISP_KEY_CAP_ZWNJGENERIC 0x79 #define LDML_DISP_KEY_CAP_ZWNJIOS 0x75 #define LDML_DISP_KEY_CAP_ZWSP 0x81 +#define LDML_ELEM_FLAGS_FLAGS_MASK 0xFFFF #define LDML_ELEM_FLAGS_ORDER_BITSHIFT 0x10 #define LDML_ELEM_FLAGS_ORDER_MASK 0xFF0000 #define LDML_ELEM_FLAGS_PREBASE 0x8 diff --git a/core/include/ldml/keyman_core_ldml.ts b/core/include/ldml/keyman_core_ldml.ts index 83a3f12683..7c8ee18ac0 100644 --- a/core/include/ldml/keyman_core_ldml.ts +++ b/core/include/ldml/keyman_core_ldml.ts @@ -16,26 +16,32 @@ // It is not updated automatically. +/** + * Names of the possible sections + */ +export const SECTION_IDENTS = [ + // Keep this sorted, but with `sect` as the first entry. + 'sect', + 'bksp', + 'disp', + 'elem', + 'keys', + 'layr', + 'list', + 'loca', + 'meta', + 'strs', + 'tran', + 'uset', + 'vars' +] as const; + /** * Defines the section identifiers and ensures that we include each and every * one of them in the `sections` block and gives us a type which we can iterate * through. */ -export type SectionIdent = -// Keep this sorted, but with `sect` as the first entry. - 'sect' | - 'bksp' | - 'disp' | - 'elem' | - 'keys' | - 'layr' | - 'list' | - 'loca' | - 'meta' | - 'strs' | - 'tran' | - 'uset' | - 'vars'; +export type SectionIdent = typeof SECTION_IDENTS[number]; type SectionMap = { @@ -340,6 +346,11 @@ class Constants { */ readonly elem_flags_prebase = 0x00000008; + /** + * bitwise mask for bit flags in elem[elemstr][element].flags. + */ + readonly elem_flags_flags_mask = 0x0000FFFF; + /** * bitwise mask for order in elem[elemstr][element].flags. * diff --git a/developer/src/common/web/utils/.c8rc.json b/developer/src/common/web/utils/.c8rc.json new file mode 100644 index 0000000000..5fcf16280a --- /dev/null +++ b/developer/src/common/web/utils/.c8rc.json @@ -0,0 +1,13 @@ +{ + "reporter": ["text", "text-summary"], + "reports-dir": "build/coverage", + "src": [ + "src/**" + ], + "include": [ + "src/**" + ], + "exclude": [ + "src/ttfmeta/**" + ] +} diff --git a/developer/src/common/web/utils/build.sh b/developer/src/common/web/utils/build.sh index fd94a8ab66..dd985b18c6 100755 --- a/developer/src/common/web/utils/build.sh +++ b/developer/src/common/web/utils/build.sh @@ -47,4 +47,4 @@ function do_build() { builder_run_action clean rm -rf ./build/ builder_run_action configure node_select_version_and_npm_ci builder_run_action build do_build -builder_run_action test typescript_run_eslint_mocha_tests 40 +builder_run_action test typescript_run_eslint_mocha_tests 60 diff --git a/developer/src/common/web/utils/test/helpers/index.ts b/developer/src/common/web/utils/test/helpers/index.ts index 25fd6856c7..09025373c6 100644 --- a/developer/src/common/web/utils/test/helpers/index.ts +++ b/developer/src/common/web/utils/test/helpers/index.ts @@ -6,10 +6,11 @@ import * as fs from "node:fs"; import { fileURLToPath } from 'node:url'; /** - * Builds a path to the fixture with the given path components. + * Builds a path to the /developer/src/common/web/utils fixture with the given + * path components. * - * e.g., makePathToFixture('example.qaa.trivial') - * e.g., makePathToFixture('example.qaa.trivial', 'model.ts') + * e.g., makePathToFixture('example.qaa.trivial') e.g., + * makePathToFixture('example.qaa.trivial', 'model.ts') * * @param components One or more path components. */ @@ -17,6 +18,19 @@ import { fileURLToPath } from 'node:url'; return fileURLToPath(new URL(path.join('..', '..', '..', 'test', 'fixtures', ...components), import.meta.url)); } +/** + * Builds a path to the /common/test file with the given path components. Note + * that this links to the base of /common/test, not /common/test/fixtures, + * because the /common/test folder currently has a mix of paths. + * + * e.g., makePathToFixture('basic.xml') + * + * @param components One or more path components. + */ +export function makePathToCommonFixture(...components: string[]): string { + return fileURLToPath(new URL(path.join('..', '..', '..', '..', '..', '..', '..', '..', 'common', 'test', ...components), import.meta.url)); +} + export function loadFile(filename: string | URL): Buffer { return fs.readFileSync(filename); } diff --git a/developer/src/common/web/utils/test/kmx/kmx-plus-builder/kmx-plus-builder.tests.ts b/developer/src/common/web/utils/test/kmx/kmx-plus-builder/kmx-plus-builder.tests.ts new file mode 100644 index 0000000000..06157ccb24 --- /dev/null +++ b/developer/src/common/web/utils/test/kmx/kmx-plus-builder/kmx-plus-builder.tests.ts @@ -0,0 +1,37 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * KMX+ file reader builder unit tests + */ + +import 'mocha'; +import { assert } from 'chai'; +import { hextobinFromFile } from '@keymanapp/hextobin'; +import { KMXPlusVersion } from '@keymanapp/ldml-keyboard-constants'; +import { makePathToCommonFixture } from '../../helpers/index.js'; +import { KMX, KMXPlus, KMXPlusFileReader } from '@keymanapp/common-types'; +import KMXPlusBuilder from '../../../src/types/kmx/kmx-plus-builder/kmx-plus-builder.js'; + +describe('KMXPlusBuilder', function() { + [[17, KMXPlusVersion.Version17], [19, KMXPlusVersion.Version19]].forEach( ([versionMajor, version]) => { + it(`should round trip a v${versionMajor} KMX+ file into memory, using KMXPlusFileReader, and back`, async function() { + // TODO-LDML: this really is not sufficient to verify that the builders + // are writing out what they should be writing but it is a bit + // of a sanity check, verifying against the reader, so much + // better than nothing! + const path = makePathToCommonFixture('keyboards', 'kmx-plus', `basic-${versionMajor}.txt`); + const input = hextobinFromFile(path, null, { silent: true }); + const reader = new KMXPlusFileReader(); + const file = new KMXPlus.KMXPlusFile(version); + file.kmxplus = reader.readFromKmx(input); + const writer = new KMXPlusBuilder(file); + const output = writer.compile(); + + const kmx = new KMX.KMXFile(); + const binaryKmxPlusHeader = kmx.COMP_KEYBOARD_KMXPLUSINFO.fromBuffer(input.slice(KMX.KMXFile.COMP_KEYBOARD_SIZE)); + const kmxplusInput = input.slice(binaryKmxPlusHeader.dpKMXPlus, binaryKmxPlusHeader.dpKMXPlus + binaryKmxPlusHeader.dwKMXPlusSize); + + assert.deepEqual(output, kmxplusInput); + }); + }); +}); \ No newline at end of file diff --git a/developer/src/kmc-ldml/test/compiler-e2e.tests.ts b/developer/src/kmc-ldml/test/compiler-e2e.tests.ts index 515b27d804..94d1e9dee3 100644 --- a/developer/src/kmc-ldml/test/compiler-e2e.tests.ts +++ b/developer/src/kmc-ldml/test/compiler-e2e.tests.ts @@ -2,7 +2,7 @@ import * as fs from 'node:fs'; import 'mocha'; import {assert} from 'chai'; import { hextobinFromFile } from '@keymanapp/hextobin'; -import {compileKeyboard, compilerTestCallbacks, compilerTestOptions, makePathToFixture, scrubContextFromMessages} from './helpers/index.js'; +import {compileKeyboard, compilerTestCallbacks, compilerTestOptions, makePathToCommonFixture, makePathToFixture, scrubContextFromMessages} from './helpers/index.js'; import { compareXml } from './helpers/compareXml.js'; import { LdmlKeyboardCompiler } from '../src/compiler/compiler.js'; import { kmxToXml } from '../src/util/serialize.js'; @@ -37,8 +37,8 @@ describe('compiler-tests', function() { // Let's build basic.xml // It should match basic.kmx (built from basic.txt) - const inputFilename = makePathToFixture('basic.xml'); - const binaryFilename = makePathToFixture(`basic-${vernum}.txt`); + const inputFilename = makePathToCommonFixture('keyboards', 'kmx-plus', 'basic.xml'); + const binaryFilename = makePathToCommonFixture('keyboards', 'kmx-plus', `basic-${vernum}.txt`); // Compare output const expected = hextobinFromFile(binaryFilename, undefined, {silent:true}); @@ -55,8 +55,8 @@ describe('compiler-tests', function() { assert.isNotNull(kmx); assert.isNotNull(kmx.data); if(debug) { - fs.writeFileSync(makePathToFixture(`basic-${vernum}-actual.kmx`), kmx.data); - fs.writeFileSync(makePathToFixture(`basic-${vernum}-expected.kmx`), expected); + fs.writeFileSync(makePathToCommonFixture(`basic-${vernum}-actual.kmx`), kmx.data); + fs.writeFileSync(makePathToCommonFixture(`basic-${vernum}-expected.kmx`), expected); } assert.deepEqual(kmx.data, expected); @@ -66,7 +66,7 @@ describe('compiler-tests', function() { }); it('should not build a v19 file with incorrect section versions for sect, disp, and layr', async function() { - const inputFilename = makePathToFixture('basic.xml'); + const inputFilename = makePathToCommonFixture('keyboards', 'kmx-plus', 'basic.xml'); const kmxPlusBuilder = await runKmxPlusCompiler(inputFilename, KMX.KMX_Version.VERSION_190); assert.equal(kmxPlusBuilder.sect.sect?.header.ident, constants.sectionid_sec2); @@ -87,7 +87,7 @@ describe('compiler-tests', function() { }); it('should not build a v17 file with incorrect section versions', async function() { - const inputFilename = makePathToFixture('basic.xml'); + const inputFilename = makePathToCommonFixture('keyboards', 'kmx-plus', 'basic.xml'); const kmxPlusBuilder = await runKmxPlusCompiler(inputFilename, KMX.KMX_Version.VERSION_170); assert.equal(kmxPlusBuilder.sect.sect?.header.ident, constants.hex_section_id(constants.section.sect)); @@ -146,7 +146,7 @@ describe('compiler-tests', function() { this.timeout(4000); // Let's build basic.xml // It should match basic.kmx (built from basic.txt) - const inputFilename = makePathToFixture('basic.xml'); + const inputFilename = makePathToCommonFixture('keyboards', 'kmx-plus', 'basic.xml'); // Compile the keyboard const kmx = await compileKeyboard(inputFilename, {...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false}); diff --git a/developer/src/kmc-ldml/test/helpers/index.ts b/developer/src/kmc-ldml/test/helpers/index.ts index 8b89a58003..9513b2f499 100644 --- a/developer/src/kmc-ldml/test/helpers/index.ts +++ b/developer/src/kmc-ldml/test/helpers/index.ts @@ -25,7 +25,7 @@ import { VarsCompiler } from '../../src/compiler/vars.js'; import { KMXPlusVersion } from '@keymanapp/ldml-keyboard-constants'; /** - * Builds a path to the fixture with the given path components. + * Builds a path to the /developer/src/kmc-ldml fixture with the given path components. * * e.g., makePathToFixture('basic.xml') * @@ -35,6 +35,20 @@ export function makePathToFixture(...components: string[]): string { return fileURLToPath(new URL(path.join('..', '..', '..', 'test', 'fixtures', ...components), import.meta.url)); } +/** + * Builds a path to the /common/test file with the given path components. Note + * that this links to the base of /common/test, not /common/test/fixtures, + * because the /common/test folder currently has a mix of paths. + * + * e.g., makePathToFixture('basic.xml') + * + * @param components One or more path components. + */ +export function makePathToCommonFixture(...components: string[]): string { + return fileURLToPath(new URL(path.join('..', '..', '..', '..', '..', '..', 'common', 'test', ...components), import.meta.url)); +} + + export const compilerTestCallbacks = new TestCompilerCallbacks(); export const compilerTestOptions: LdmlCompilerOptions = { diff --git a/developer/src/kmc-ldml/test/metadata-compiler.tests.ts b/developer/src/kmc-ldml/test/metadata-compiler.tests.ts index bd72944fae..5e5631e1ec 100644 --- a/developer/src/kmc-ldml/test/metadata-compiler.tests.ts +++ b/developer/src/kmc-ldml/test/metadata-compiler.tests.ts @@ -1,6 +1,6 @@ import 'mocha'; import { assert } from 'chai'; -import { checkMessages, compileKeyboard, compilerTestOptions, makePathToFixture } from './helpers/index.js'; +import { checkMessages, compileKeyboard, compilerTestOptions, makePathToCommonFixture } from './helpers/index.js'; import { KMX } from '@keymanapp/common-types'; import KEYMAN_VERSION from '@keymanapp/keyman-version'; @@ -10,7 +10,7 @@ describe('kmx metadata compiler', function () { this.slow(500); // 0.5 sec -- json schema validation takes a while it('should compile metadata with debug and compiler version', async function() { - const inputFilename = makePathToFixture('basic.xml'); + const inputFilename = makePathToCommonFixture('keyboards', 'kmx-plus', 'basic.xml'); // Compile the keyboard const kmx = await compileKeyboard(inputFilename, {...compilerTestOptions, saveDebug:true, shouldAddCompilerVersion:true}); @@ -45,7 +45,7 @@ describe('kmx metadata compiler', function () { }); it('should compile metadata with no compiler version', async function() { - const inputFilename = makePathToFixture('basic.xml'); + const inputFilename = makePathToCommonFixture('keyboards', 'kmx-plus', 'basic.xml'); // Compile the keyboard const kmx = await compileKeyboard(inputFilename, {...compilerTestOptions, saveDebug:true, shouldAddCompilerVersion:false}); diff --git a/developer/src/kmc-ldml/test/visual-keyboard-compiler.tests.ts b/developer/src/kmc-ldml/test/visual-keyboard-compiler.tests.ts index c6b08aaacc..ee95e392c2 100644 --- a/developer/src/kmc-ldml/test/visual-keyboard-compiler.tests.ts +++ b/developer/src/kmc-ldml/test/visual-keyboard-compiler.tests.ts @@ -9,7 +9,7 @@ import { stripIndent } from 'common-tags'; import { KMX, KvkFileWriter, VisualKeyboard } from '@keymanapp/common-types'; import { hextobinFromFile } from '@keymanapp/hextobin'; -import { checkMessages, compilerTestCallbacks, compilerTestOptions, makePathToFixture } from './helpers/index.js'; +import { checkMessages, compilerTestCallbacks, compilerTestOptions, makePathToCommonFixture, makePathToFixture } from './helpers/index.js'; import { LdmlKeyboardVisualKeyboardCompiler } from '../src/compiler/visual-keyboard-compiler.js'; import { LDMLKeyboardXMLSourceFileReader } from '@keymanapp/developer-utils'; @@ -22,7 +22,7 @@ describe('visual-keyboard-compiler', function() { // Let's build basic.xml // It should match basic.kvk (built from basic-kvk.txt) - const inputFilename = makePathToFixture('basic.xml'); + const inputFilename = makePathToCommonFixture('keyboards', 'kmx-plus', 'basic.xml'); const binaryFilename = makePathToFixture('basic-kvk.txt'); // Compile the visual keyboard diff --git a/package-lock.json b/package-lock.json index 61a304996b..904268b72a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -149,6 +149,7 @@ "restructure": "3.0.1" }, "devDependencies": { + "@keymanapp/hextobin": "*", "@types/mocha": "^5.2.7", "@types/node": "^20.4.1", "ajv": "^8.12.0", @@ -241,6 +242,7 @@ "license": "MIT", "dependencies": { "@keymanapp/common-types": "*", + "@keymanapp/ldml-keyboard-constants": "*", "eventemitter3": "^5.0.0", "fast-xml-parser": "^5.3.4", "path-browserify": "^1.0.1",