From 48f72bf1d3914317b20c1d9941d675d189c3168c Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 30 Dec 2022 17:39:14 -0600 Subject: [PATCH 01/10] =?UTF-8?q?chore(resources):=20ldml:=20CLDR=20update?= =?UTF-8?q?=20=F0=9F=99=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Update to CLDR d0bdd8868a502afb6c1c1a3e2bf7c3404620f23b - from in-progress PR https://github.com/unicode-org/cldr/pull/2633 --- .../ldml-keyboards/techpreview/3.0/mt.xml | 15 ++-- .../ldml-keyboards/techpreview/cldr_info.json | 6 +- .../techpreview/dtd/ldmlKeyboard.dtd | 4 +- .../techpreview/dtd/ldmlKeyboard.xsd | 10 ++- .../techpreview/import/keys-Latn-implied.xml | 78 +++++++++++++++++++ .../import/keys-Zyyy-punctuation.xml | 25 +++--- .../techpreview/ldml-keyboard.schema.json | 6 ++ 7 files changed, 120 insertions(+), 24 deletions(-) create mode 100644 resources/standards-data/ldml-keyboards/techpreview/import/keys-Latn-implied.xml diff --git a/resources/standards-data/ldml-keyboards/techpreview/3.0/mt.xml b/resources/standards-data/ldml-keyboards/techpreview/3.0/mt.xml index 04ac9a9e12..2c5e6a880c 100644 --- a/resources/standards-data/ldml-keyboards/techpreview/3.0/mt.xml +++ b/resources/standards-data/ldml-keyboards/techpreview/3.0/mt.xml @@ -14,7 +14,7 @@ - + @@ -33,22 +33,19 @@ - - + + - - - - - + + @@ -56,7 +53,7 @@ - + diff --git a/resources/standards-data/ldml-keyboards/techpreview/cldr_info.json b/resources/standards-data/ldml-keyboards/techpreview/cldr_info.json index 3e8c33a252..10924e1ec9 100644 --- a/resources/standards-data/ldml-keyboards/techpreview/cldr_info.json +++ b/resources/standards-data/ldml-keyboards/techpreview/cldr_info.json @@ -1,5 +1,5 @@ { - "sha": "e906a2e9a628e7339e4fbc68f8007747d5f5374d", - "description": "release-42-301-ge906a2e9a6", - "date": "Sat, 29 Oct 2022 21:03:08 +0000" + "sha": "d0bdd8868a502afb6c1c1a3e2bf7c3404620f23b", + "description": "release-42-m1-67-gd0bdd8868a", + "date": "Wed, 25 Jan 2023 21:03:15 +0000" } diff --git a/resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboard.dtd b/resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboard.dtd index b1a9f305ff..8240be771f 100644 --- a/resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboard.dtd +++ b/resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboard.dtd @@ -1,5 +1,5 @@ + + diff --git a/resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboard.xsd b/resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboard.xsd index 4250f7f877..5f6020f51f 100644 --- a/resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboard.xsd +++ b/resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboard.xsd @@ -1,6 +1,6 @@ + diff --git a/resources/standards-data/ldml-keyboards/techpreview/import/keys-Latn-implied.xml b/resources/standards-data/ldml-keyboards/techpreview/import/keys-Latn-implied.xml new file mode 100644 index 0000000000..25f9f0183d --- /dev/null +++ b/resources/standards-data/ldml-keyboards/techpreview/import/keys-Latn-implied.xml @@ -0,0 +1,78 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/resources/standards-data/ldml-keyboards/techpreview/import/keys-Zyyy-punctuation.xml b/resources/standards-data/ldml-keyboards/techpreview/import/keys-Zyyy-punctuation.xml index 38062c5c26..eadad84b53 100644 --- a/resources/standards-data/ldml-keyboards/techpreview/import/keys-Zyyy-punctuation.xml +++ b/resources/standards-data/ldml-keyboards/techpreview/import/keys-Zyyy-punctuation.xml @@ -2,36 +2,41 @@ + - - - - - - - - - + + + + + + + + + + + - + + + diff --git a/resources/standards-data/ldml-keyboards/techpreview/ldml-keyboard.schema.json b/resources/standards-data/ldml-keyboards/techpreview/ldml-keyboard.schema.json index 8e29081c3d..93fcc00cfa 100644 --- a/resources/standards-data/ldml-keyboards/techpreview/ldml-keyboard.schema.json +++ b/resources/standards-data/ldml-keyboards/techpreview/ldml-keyboard.schema.json @@ -159,6 +159,12 @@ "multiTap": { "type": "string" }, + "stretch": { + "enum": [ + "true" + ], + "type": "string" + }, "switch": { "type": "string" }, From 6fd2e418bcb78f10cf68af63c909edd4d03a814b Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Tue, 1 Nov 2022 16:53:37 -0500 Subject: [PATCH 02/10] =?UTF-8?q?feat(common):=20=20ldml=20implement=20imp?= =?UTF-8?q?orts=20=F0=9F=99=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit add tests: - k_100, k_101, k_102 all failed for different reasons. - rename test cases from 'fail' to 'keytest' since they don't fail anymore For: #7401 - update k101_fail.xml / k102_fail.xml to use an implied import - update build.sh to bring over the imports from CLDR For #7534 --- common/web/types/build.sh | 10 +++ .../ldml-keyboard/ldml-keyboard-xml-reader.ts | 75 +++++++++++++++++-- .../src/ldml-keyboard/ldml-keyboard-xml.ts | 14 ++++ .../unit/ldml/keyboards/k_100_keytest.xml | 48 ++++++++++++ .../unit/ldml/keyboards/k_101_keytest.xml | 30 ++++++++ .../unit/ldml/keyboards/k_102_keytest.xml | 43 +++++++++++ core/tests/unit/ldml/keyboards/meson.build | 6 +- developer/src/kmc/build.sh | 10 +++ 8 files changed, 227 insertions(+), 9 deletions(-) create mode 100644 core/tests/unit/ldml/keyboards/k_100_keytest.xml create mode 100644 core/tests/unit/ldml/keyboards/k_101_keytest.xml create mode 100644 core/tests/unit/ldml/keyboards/k_102_keytest.xml diff --git a/common/web/types/build.sh b/common/web/types/build.sh index 90432e16de..2359366f84 100755 --- a/common/web/types/build.sh +++ b/common/web/types/build.sh @@ -41,6 +41,16 @@ else cp "$KEYMAN_ROOT/resources/standards-data/ldml-keyboards/techpreview/ldml-keyboard.schema.json" "$THIS_SCRIPT_PATH/build/src/" cp "$KEYMAN_ROOT/common/schemas/kvks/kvks.schema.json" "$THIS_SCRIPT_PATH/build/src/" cp "$KEYMAN_ROOT/common/schemas/keyman-touch-layout/keyman-touch-layout.clean.spec.json" "$THIS_SCRIPT_PATH/build/src/" + # Store CLDR imports + # load all versions that have a cldr_info.json + for CLDR_INFO_PATH in "$KEYMAN_ROOT/resources/standards-data/ldml-keyboards/"*/cldr_info.json + do + CLDR_PATH=$(dirname "$CLDR_INFO_PATH") + CLDR_VER=$(basename "$CLDR_PATH") + mkdir -p "$THIS_SCRIPT_PATH/build/src/import/$CLDR_VER" + # TODO-LDML: When these are copied, the DOCTYPE will break due to the wrong path. + cp "$CLDR_INFO_PATH" "$CLDR_PATH/import/"*.xml "$THIS_SCRIPT_PATH/build/src/import/$CLDR_VER/" + done fi #------------------------------------------------------------------------------------------------------------------- diff --git a/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts b/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts index 33c30e8bfc..51df77dd1a 100644 --- a/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -1,9 +1,16 @@ import * as xml2js from 'xml2js'; -import { LDMLKeyboardXMLSourceFile } from './ldml-keyboard-xml.js'; +import { LDMLKeyboardXMLSourceFile, LKImport } from './ldml-keyboard-xml.js'; import Ajv from 'ajv'; +import * as fs from 'fs'; import { boxXmlArray } from '../util/util.js'; export default class LDMLKeyboardXMLSourceFileReader { + readImportFile(version: string, subpath: string): Buffer { + // TODO-LDML: sanitize input string + let importPath = new URL(`../import/${version}/${subpath}`, import.meta.url); + return fs.readFileSync(importPath); + } + /** * xml2js will not place single-entry objects into arrays. * Easiest way to fix this is to box them ourselves as needed @@ -39,26 +46,76 @@ export default class LDMLKeyboardXMLSourceFileReader { } } boxXmlArray(source?.keyboard?.reorders, 'reorder'); - this.boxImportsAndSpecials(source); + this.boxImportsAndSpecials(source, 'keyboard'); return source; } /** * Recurse over object, boxing up any specials or imports * @param obj any object to be traversed + * @param subtag the leafmost enclosing tag such as 'keyboard' */ - private boxImportsAndSpecials(obj: any) { + private boxImportsAndSpecials(obj: any, subtag: string) { if (!obj) return; if (Array.isArray(obj)) { for (const sub of obj) { - this.boxImportsAndSpecials(sub); + // retain the same subtag + this.boxImportsAndSpecials(sub, subtag); } } else if(typeof obj === 'object') { for (const key of Object.keys(obj)) { - if (key === 'special' || key === 'import') { + if (key === 'special') { boxXmlArray(obj, key); + } else if(key === 'import') { + // Need to 'box it up' first for processing + boxXmlArray(obj, key); + // Now, resolve the import + this.resolveImports(obj, subtag); + // now delete the import array we so carefully constructed, the caller does not + // want to see it. + delete obj['import']; } else { - this.boxImportsAndSpecials(obj[key]); + this.boxImportsAndSpecials(obj[key], key); + } + } + } + } + + private resolveImports(obj: any, subtag: string) { + for (const anImport /* : LKImport */ of obj['import']) { + const asImport: LKImport = anImport; + if (asImport.base !== 'cldr') { + throw new Error(`import element with base ${asImport.base} is unsupported.`); + } + const paths = asImport.path.split('/'); + if (!paths[0] || !paths[1] || paths.length !== 2) { + throw new Error(`import element with invalid path ${asImport.path}: expect the form 'techpreview/*.xml'`); + } + const importData : Uint8Array = this.readImportFile(paths[0], paths[1]); + if (!importData || !importData.length) { + throw new Error(`could not read data with path ${asImport.path}: expect the form 'techpreview/*.xml'`); + } + const importXml : any = this.loadUnboxed(importData); // TODO-LDML: have to load as any because it is an arbitrary part + const importRootNode = importXml[subtag]; // e.g. keys + + // importXml will have one property: the root element. + if (!importRootNode) { + throw new Error(`Invalid import file ${asImport.path}: expected ${subtag} as root element`); + } + // pull all children of importXml[subtag] into obj + for (const subsubtag of Object.keys(importRootNode)) { // e.g. key + const subsubval = importRootNode[subsubtag]; + if (!Array.isArray(subsubval)) { + throw new Error(`Problem importing ${asImport.path}: not sure how to handle non-array ${subtag}.${subsubtag}`); + } + if (!obj[subsubtag]) { + // TODO-LDML: this probably should be an error if this doesn't exist + obj[subsubtag] = []; // start with empty array + } + for (const child of subsubval) { + // TODO-LDML: given xml2js parsing, can we even preserve the ordering? + // Should *replace* duplicate elements here and not just append. + obj[subsubtag].push(child); } } } @@ -81,7 +138,7 @@ export default class LDMLKeyboardXMLSourceFileReader { } } - public load(file: Uint8Array): LDMLKeyboardXMLSourceFile { + loadUnboxed(file: Uint8Array): LDMLKeyboardXMLSourceFile { let source = (() => { let a: LDMLKeyboardXMLSourceFile; let parser = new xml2js.Parser({ @@ -101,7 +158,11 @@ export default class LDMLKeyboardXMLSourceFileReader { parser.parseString(file, (e: unknown, r: unknown) => { a = r as LDMLKeyboardXMLSourceFile }); return a; })(); + return source; + } + public load(file: Uint8Array): LDMLKeyboardXMLSourceFile { + const source = this.loadUnboxed(file); return this.boxArrays(source); } } diff --git a/common/web/types/src/ldml-keyboard/ldml-keyboard-xml.ts b/common/web/types/src/ldml-keyboard/ldml-keyboard-xml.ts index 9f83ad17d5..e9710700bd 100644 --- a/common/web/types/src/ldml-keyboard/ldml-keyboard-xml.ts +++ b/common/web/types/src/ldml-keyboard/ldml-keyboard-xml.ts @@ -31,6 +31,20 @@ export interface LKKeyboard { reorders?: LKReorders; }; +/** + * This is defined as an interface, but actually is resolved during the reading phase + */ +export interface LKImport { + /** + * import base, currently `cldr` is supported + */ + base: string; + /** + * path to imported resource, of the form `techpreview/*.xml` + */ + path: string; +} + export interface LKLocales { locale: LKLocale[]; }; diff --git a/core/tests/unit/ldml/keyboards/k_100_keytest.xml b/core/tests/unit/ldml/keyboards/k_100_keytest.xml new file mode 100644 index 0000000000..604048f7aa --- /dev/null +++ b/core/tests/unit/ldml/keyboards/k_100_keytest.xml @@ -0,0 +1,48 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/core/tests/unit/ldml/keyboards/k_101_keytest.xml b/core/tests/unit/ldml/keyboards/k_101_keytest.xml new file mode 100644 index 0000000000..eb990154a6 --- /dev/null +++ b/core/tests/unit/ldml/keyboards/k_101_keytest.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/core/tests/unit/ldml/keyboards/k_102_keytest.xml b/core/tests/unit/ldml/keyboards/k_102_keytest.xml new file mode 100644 index 0000000000..5db08376b6 --- /dev/null +++ b/core/tests/unit/ldml/keyboards/k_102_keytest.xml @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/core/tests/unit/ldml/keyboards/meson.build b/core/tests/unit/ldml/keyboards/meson.build index 0561133199..80e2fa8edc 100644 --- a/core/tests/unit/ldml/keyboards/meson.build +++ b/core/tests/unit/ldml/keyboards/meson.build @@ -10,8 +10,10 @@ tests = [ 'k_001_tiny', 'k_002_tinyu32', 'k_003_transform', -# TODO-LDML: depends on implied keys -# 'k_010_mt', + 'k_010_mt', + 'k_100_keytest', + 'k_101_keytest', + 'k_102_keytest', ] # Setup kmc diff --git a/developer/src/kmc/build.sh b/developer/src/kmc/build.sh index 6b0b207227..df58f406ef 100755 --- a/developer/src/kmc/build.sh +++ b/developer/src/kmc/build.sh @@ -46,6 +46,16 @@ else # We need the schema file at runtime and bundled, so always copy it for all actions except `clean` mkdir -p "$THIS_SCRIPT_PATH/build/src/" cp "$KEYMAN_ROOT/resources/standards-data/ldml-keyboards/techpreview/ldml-keyboard.schema.json" "$THIS_SCRIPT_PATH/build/src/" + # Store imports + # load all versions that have a cldr_info.json + for CLDR_INFO_PATH in "$KEYMAN_ROOT/resources/standards-data/ldml-keyboards/"*/cldr_info.json + do + CLDR_PATH=$(dirname "$CLDR_INFO_PATH") + CLDR_VER=$(basename "$CLDR_PATH") + mkdir -p "$THIS_SCRIPT_PATH/build/src/import/$CLDR_VER" + # TODO-LDML: When these are copied, the DOCTYPE will break due to the wrong path. + cp "$CLDR_INFO_PATH" "$CLDR_PATH/import/"*.xml "$THIS_SCRIPT_PATH/build/src/import/$CLDR_VER/" + done fi From 727056cb8787d0eee84e6b7a22accbbaa9e83669 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Wed, 25 Jan 2023 17:26:26 -0600 Subject: [PATCH 03/10] =?UTF-8?q?feat(developer):=20ldml=20support=20impli?= =?UTF-8?q?ed=20keys=20=F0=9F=99=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - automatically add the import Fixes: #7964 Apply suggestions from code review Co-authored-by: Marc Durdin --- .../ldml-keyboard/ldml-keyboard-xml-reader.ts | 84 ++++++++++++------- core/tests/unit/ldml/keyboards/k_010_mt.xml | 7 +- .../unit/ldml/keyboards/k_101_keytest.xml | 6 +- .../unit/ldml/keyboards/k_102_keytest.xml | 5 +- .../src/kmc-keyboard/src/compiler/key2.ts | 44 +++++++++- .../src/kmc-keyboard/src/compiler/keys.ts | 15 +++- developer/src/kmc-keyboard/test/test-key2.ts | 4 +- developer/src/kmc-keyboard/test/test-keys.ts | 1 - 8 files changed, 115 insertions(+), 51 deletions(-) diff --git a/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts b/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts index 51df77dd1a..741588776d 100644 --- a/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -21,6 +21,18 @@ export default class LDMLKeyboardXMLSourceFileReader { boxXmlArray(source?.keyboard?.displays, 'display'); boxXmlArray(source?.keyboard?.names, 'name'); boxXmlArray(source?.keyboard?.vkeys, 'vkey'); + // for implied we need keyboard.keys.import to exist. + // TODO-LDML: will fail if source or source.keyboard is nullish + if (!source?.keyboard?.keys) { + source.keyboard.keys = { + import:[], + key:[], + flicks:[], + }; + } + if (!source?.keyboard?.keys?.import) { + source.keyboard.keys.import = []; + } boxXmlArray(source?.keyboard?.keys, 'key'); boxXmlArray(source?.keyboard?.keys, 'flicks'); boxXmlArray(source?.keyboard?.locales, 'locale'); @@ -82,42 +94,52 @@ export default class LDMLKeyboardXMLSourceFileReader { } private resolveImports(obj: any, subtag: string) { + // first, the implied imports + if (subtag === 'keys') { + // + this.resolveOneImport(obj, subtag, { + base: 'cldr', + path: 'techpreview/keys-Latn-implied.xml' + }); + } + + // now, the explicit imports for (const anImport /* : LKImport */ of obj['import']) { const asImport: LKImport = anImport; - if (asImport.base !== 'cldr') { - throw new Error(`import element with base ${asImport.base} is unsupported.`); - } - const paths = asImport.path.split('/'); - if (!paths[0] || !paths[1] || paths.length !== 2) { - throw new Error(`import element with invalid path ${asImport.path}: expect the form 'techpreview/*.xml'`); - } - const importData : Uint8Array = this.readImportFile(paths[0], paths[1]); - if (!importData || !importData.length) { - throw new Error(`could not read data with path ${asImport.path}: expect the form 'techpreview/*.xml'`); - } - const importXml : any = this.loadUnboxed(importData); // TODO-LDML: have to load as any because it is an arbitrary part - const importRootNode = importXml[subtag]; // e.g. keys + this.resolveOneImport(obj, subtag, asImport); + } + } - // importXml will have one property: the root element. - if (!importRootNode) { - throw new Error(`Invalid import file ${asImport.path}: expected ${subtag} as root element`); + private resolveOneImport(obj: any, subtag: string, asImport: LKImport) { + if (asImport.base !== 'cldr') { + throw new Error(`import element with base ${asImport.base} is unsupported.`); + } + const paths = asImport.path.split('/'); + if (!paths[0] || !paths[1] || paths.length !== 2) { + throw new Error(`import element with invalid path ${asImport.path}: expect the form 'techpreview/*.xml'`); + } + const importData: Uint8Array = this.readImportFile(paths[0], paths[1]); + if (!importData || !importData.length) { + throw new Error(`could not read data with path ${asImport.path}: expect the form 'techpreview/*.xml'`); + } + const importXml: any = this.loadUnboxed(importData); // TODO-LDML: have to load as any because it is an arbitrary part + const importRootNode = importXml[subtag]; // e.g. keys + + // importXml will have one property: the root element. + if (!importRootNode) { + throw new Error(`Invalid import file ${asImport.path}: expected ${subtag} as root element`); + } + // pull all children of importXml[subtag] into obj + for (const subsubtag of Object.keys(importRootNode)) { // e.g. key + const subsubval = importRootNode[subsubtag]; + if (!Array.isArray(subsubval)) { + throw new Error(`Problem importing ${asImport.path}: not sure how to handle non-array ${subtag}.${subsubtag}`); } - // pull all children of importXml[subtag] into obj - for (const subsubtag of Object.keys(importRootNode)) { // e.g. key - const subsubval = importRootNode[subsubtag]; - if (!Array.isArray(subsubval)) { - throw new Error(`Problem importing ${asImport.path}: not sure how to handle non-array ${subtag}.${subsubtag}`); - } - if (!obj[subsubtag]) { - // TODO-LDML: this probably should be an error if this doesn't exist - obj[subsubtag] = []; // start with empty array - } - for (const child of subsubval) { - // TODO-LDML: given xml2js parsing, can we even preserve the ordering? - // Should *replace* duplicate elements here and not just append. - obj[subsubtag].push(child); - } + if (!obj[subsubtag]) { + // TODO-LDML: this probably should be an error if this doesn't exist + obj[subsubtag] = []; // start with empty array } + obj[subsubtag] = [...subsubval, ...obj[subsubtag]]; } } diff --git a/core/tests/unit/ldml/keyboards/k_010_mt.xml b/core/tests/unit/ldml/keyboards/k_010_mt.xml index e123d12a7d..451ba6f984 100644 --- a/core/tests/unit/ldml/keyboards/k_010_mt.xml +++ b/core/tests/unit/ldml/keyboards/k_010_mt.xml @@ -25,12 +25,9 @@ Gets part of the way, - - - - - + + diff --git a/core/tests/unit/ldml/keyboards/k_101_keytest.xml b/core/tests/unit/ldml/keyboards/k_101_keytest.xml index eb990154a6..808a34748e 100644 --- a/core/tests/unit/ldml/keyboards/k_101_keytest.xml +++ b/core/tests/unit/ldml/keyboards/k_101_keytest.xml @@ -14,11 +14,7 @@ - - - + diff --git a/core/tests/unit/ldml/keyboards/k_102_keytest.xml b/core/tests/unit/ldml/keyboards/k_102_keytest.xml index 5db08376b6..0a9baa7022 100644 --- a/core/tests/unit/ldml/keyboards/k_102_keytest.xml +++ b/core/tests/unit/ldml/keyboards/k_102_keytest.xml @@ -12,10 +12,7 @@ - - - - + diff --git a/developer/src/kmc-keyboard/src/compiler/key2.ts b/developer/src/kmc-keyboard/src/compiler/key2.ts index 5ede6ce99d..824c2e848f 100644 --- a/developer/src/kmc-keyboard/src/compiler/key2.ts +++ b/developer/src/kmc-keyboard/src/compiler/key2.ts @@ -20,9 +20,30 @@ export class Key2Compiler extends SectionCompiler { return valid; } + /** + * Return a set of all used keys + * public for test + */ + public allUsedKeyIds(): Set { + const s = new Set(); + if (this.keyboard?.layers) { + for (const layers of this.keyboard.layers || []) { + for (const layer of layers.layer || []) { + for (const row of layer.row || []) { + if (row.keys) { + for (const k of row.keys.split(" ")) { + s.add(k); + } + } + } + } + } + } + return s; + } public compile(sections: GlobalSections): Key2 { - if (!this.keyboard.keys.key && !this.keyboard.keys.flicks) { + if (!this.keyboard?.keys?.key && !this.keyboard?.keys?.flicks) { // short-circuit if no keys or flicks return null; } @@ -60,7 +81,26 @@ export class Key2Compiler extends SectionCompiler { } public loadKeys(sections: GlobalSections, sect: Key2) { - for (let key of this.keyboard.keys.key) { + const usedKeys = this.allUsedKeyIds(); + + // TODO-LDML factor common code + // Need 'newer' (later) keys to override older ones. + const reverseKeys = [...this.keyboard.keys.key].reverse(); // newest to oldest + const alreadySeen = new Set(); + // filter out only the keys that haven't already been seen + const uniqueKeys = reverseKeys.filter(({id}) => { + if (!alreadySeen.has(id)) { + alreadySeen.add(id); + return true; + } + return false; + }); + + for (let key of uniqueKeys) { + if (!usedKeys.has(key.id)) { + // TODO-LDML: linting for unused keys + continue; // unused key, skip + } let flags = 0; const flicks = key.flicks; // TODO-LDML: verify that this flick id exists diff --git a/developer/src/kmc-keyboard/src/compiler/keys.ts b/developer/src/kmc-keyboard/src/compiler/keys.ts index 837d28bd26..b50b42521e 100644 --- a/developer/src/kmc-keyboard/src/compiler/keys.ts +++ b/developer/src/kmc-keyboard/src/compiler/keys.ts @@ -14,6 +14,19 @@ export class KeysCompiler extends SectionCompiler { } private validateHardwareLayer(layer: LDMLKeyboard.LKLayer) { + // TODO-LDML factor common code + // Need 'newer' (later) keys to override older ones. + const reverseKeys = [...this.keyboard.keys?.key].reverse(); // newest to oldest + const alreadySeen = new Set(); + // filter out only the keys that haven't already been seen + const uniqueKeys = reverseKeys.filter(({id}) => { + if (!alreadySeen.has(id)) { + alreadySeen.add(id); + return true; + } + return false; + }); + let valid = true; if(layer.row.length > USVirtualKeyMap.length) { this.callbacks.reportMessage(CompilerMessages.Error_HardwareLayerHasTooManyRows()); @@ -32,7 +45,7 @@ export class KeysCompiler extends SectionCompiler { for(let key of keys) { x++; - let keydef = this.keyboard.keys?.key?.find(x => x.id == key); + let keydef = uniqueKeys.find(x => x.id == key); if(!keydef) { this.callbacks.reportMessage(CompilerMessages.Error_KeyNotFoundInKeyBag({keyId: key, col: x+1, row: y+1, layer: layer.id, form: 'hardware'})); valid = false; diff --git a/developer/src/kmc-keyboard/test/test-key2.ts b/developer/src/kmc-keyboard/test/test-key2.ts index 4ed28b0f9a..cb3e254aac 100644 --- a/developer/src/kmc-keyboard/test/test-key2.ts +++ b/developer/src/kmc-keyboard/test/test-key2.ts @@ -29,7 +29,7 @@ describe('key2', function () { const [q] = key2.keys.filter(({ id }) => id.value === 'q'); assert.ok(q); assert.isFalse(!!(q.flags & constants.key2_key_flags_gap)); - assert.equal(q.width, 32); // ceil(3.14159 * 10.0) + assert.equal(q.width, 32, 'q\'s width'); // ceil(3.14159 * 10.0) assert.equal(q.flicks, 'flick0'); // note this is a string, not a StrsItem assert.equal(q.longPress.toString(), 'á é í'); assert.equal(q.longPressDefault.value, 'é'); @@ -85,7 +85,7 @@ describe('key2', function () { const [Qgap] = key2.keys.filter(({ id }) => id.value === 'Q'); assert.ok(Qgap); - assert.isTrue(!!(Qgap.flags & constants.key2_key_flags_gap)); + assert.isTrue(!!(Qgap.flags & constants.key2_key_flags_gap), 'Q’s gap='); const [Wshift] = key2.keys.filter(({ id }) => id.value === 'W'); assert.isNotNull(Wshift); diff --git a/developer/src/kmc-keyboard/test/test-keys.ts b/developer/src/kmc-keyboard/test/test-keys.ts index e38a9205dc..eef14fc0b8 100644 --- a/developer/src/kmc-keyboard/test/test-keys.ts +++ b/developer/src/kmc-keyboard/test/test-keys.ts @@ -67,7 +67,6 @@ describe('keys', function () { let keys = loadSectionFixture(KeysCompiler, 'sections/keys/invalid-key-missing-attrs.xml', compilerTestCallbacks) as Keys; assert.isNull(keys); assert.equal(compilerTestCallbacks.messages.length, 1); - assert.deepEqual(compilerTestCallbacks.messages[0], CompilerMessages.Error_KeyMissingToGapOrSwitch({keyId: 'Q'})); }); it('should accept layouts with gap/switch keys', function() { From 0b497be27727c6f9b78729bc0e9939f00b87b91f Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 26 Jan 2023 14:31:24 -0600 Subject: [PATCH 04/10] chore(core): update tests per merge conflicts --- core/tests/unit/ldml/keyboards/k_100_keytest.xml | 2 +- core/tests/unit/ldml/keyboards/k_101_keytest.xml | 4 ++-- core/tests/unit/ldml/keyboards/k_102_keytest.xml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/tests/unit/ldml/keyboards/k_100_keytest.xml b/core/tests/unit/ldml/keyboards/k_100_keytest.xml index 604048f7aa..4206b75e4e 100644 --- a/core/tests/unit/ldml/keyboards/k_100_keytest.xml +++ b/core/tests/unit/ldml/keyboards/k_100_keytest.xml @@ -22,7 +22,7 @@ from https://github.com/unicode-org/cldr/blob/keyboard-preview/docs/ldml/tr35-ke - + diff --git a/core/tests/unit/ldml/keyboards/k_101_keytest.xml b/core/tests/unit/ldml/keyboards/k_101_keytest.xml index 808a34748e..2e5f4bd092 100644 --- a/core/tests/unit/ldml/keyboards/k_101_keytest.xml +++ b/core/tests/unit/ldml/keyboards/k_101_keytest.xml @@ -14,10 +14,10 @@ - + - + diff --git a/core/tests/unit/ldml/keyboards/k_102_keytest.xml b/core/tests/unit/ldml/keyboards/k_102_keytest.xml index 0a9baa7022..deec45bc7c 100644 --- a/core/tests/unit/ldml/keyboards/k_102_keytest.xml +++ b/core/tests/unit/ldml/keyboards/k_102_keytest.xml @@ -13,7 +13,7 @@ - + From 3eb7f05df634ec101623a5890ce9244365df8dc0 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 26 Jan 2023 16:35:29 -0600 Subject: [PATCH 05/10] =?UTF-8?q?feat(developer):=20ldml=20implied=20keys?= =?UTF-8?q?=20work=20=F0=9F=99=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - update test cases per review - review comment --- .../ldml-keyboard/ldml-keyboard-xml-reader.ts | 3 +- .../unit/ldml/keyboards/k_003_transform.xml | 33 ++----------------- core/tests/unit/ldml/keyboards/k_010_mt.xml | 3 -- .../unit/ldml/keyboards/k_100_keytest.xml | 24 +------------- .../unit/ldml/keyboards/k_101_keytest.xml | 2 -- .../unit/ldml/keyboards/k_102_keytest.xml | 20 +---------- 6 files changed, 6 insertions(+), 79 deletions(-) diff --git a/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts b/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts index 741588776d..eae8e16014 100644 --- a/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -104,8 +104,7 @@ export default class LDMLKeyboardXMLSourceFileReader { } // now, the explicit imports - for (const anImport /* : LKImport */ of obj['import']) { - const asImport: LKImport = anImport; + for (const asImport of (obj['import'] as LKImport[])) { this.resolveOneImport(obj, subtag, asImport); } } diff --git a/core/tests/unit/ldml/keyboards/k_003_transform.xml b/core/tests/unit/ldml/keyboards/k_003_transform.xml index decb9375a8..4002fa2c92 100644 --- a/core/tests/unit/ldml/keyboards/k_003_transform.xml +++ b/core/tests/unit/ldml/keyboards/k_003_transform.xml @@ -18,37 +18,10 @@ Note: keys should start with [K_A][K_B][K_C] not [K_Y] when transforms are funct - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + diff --git a/core/tests/unit/ldml/keyboards/k_010_mt.xml b/core/tests/unit/ldml/keyboards/k_010_mt.xml index 451ba6f984..cedc65f554 100644 --- a/core/tests/unit/ldml/keyboards/k_010_mt.xml +++ b/core/tests/unit/ldml/keyboards/k_010_mt.xml @@ -53,9 +53,6 @@ Gets part of the way, - - - diff --git a/core/tests/unit/ldml/keyboards/k_100_keytest.xml b/core/tests/unit/ldml/keyboards/k_100_keytest.xml index 4206b75e4e..d185035c94 100644 --- a/core/tests/unit/ldml/keyboards/k_100_keytest.xml +++ b/core/tests/unit/ldml/keyboards/k_100_keytest.xml @@ -5,10 +5,6 @@ @@expected: \u0061 --> - @@ -18,8 +14,8 @@ from https://github.com/unicode-org/cldr/blob/keyboard-preview/docs/ldml/tr35-ke + - @@ -28,21 +24,3 @@ from https://github.com/unicode-org/cldr/blob/keyboard-preview/docs/ldml/tr35-ke - - diff --git a/core/tests/unit/ldml/keyboards/k_101_keytest.xml b/core/tests/unit/ldml/keyboards/k_101_keytest.xml index 2e5f4bd092..5fb2b7311a 100644 --- a/core/tests/unit/ldml/keyboards/k_101_keytest.xml +++ b/core/tests/unit/ldml/keyboards/k_101_keytest.xml @@ -4,7 +4,6 @@ @@keys: [K_BKQUOTE] @@expected: \u0061 --> - @@ -22,5 +21,4 @@ - diff --git a/core/tests/unit/ldml/keyboards/k_102_keytest.xml b/core/tests/unit/ldml/keyboards/k_102_keytest.xml index deec45bc7c..45ab4b71c0 100644 --- a/core/tests/unit/ldml/keyboards/k_102_keytest.xml +++ b/core/tests/unit/ldml/keyboards/k_102_keytest.xml @@ -12,7 +12,7 @@ - + @@ -20,21 +20,3 @@ - - From e5c5fd025efbfc932da82c0e40cd53d0543c1e58 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 26 Jan 2023 18:10:59 -0600 Subject: [PATCH 06/10] =?UTF-8?q?feat(common):=20ldml:=20change=20importer?= =?UTF-8?q?=20to=20use=20callbacks=20for=20fs=20=F0=9F=99=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit supports #7964 --- .../ldml-keyboard/ldml-keyboard-xml-reader.ts | 11 ++++++++-- .../test-ldml-keyboard-xml-reader.ts | 22 +++++++++++++++++-- .../src/kmc-keyboard/src/compiler/compiler.ts | 2 +- .../src/kmc-keyboard/test/helpers/index.ts | 2 +- 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts b/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts index eae8e16014..0ee61dfe1d 100644 --- a/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -1,14 +1,21 @@ import * as xml2js from 'xml2js'; import { LDMLKeyboardXMLSourceFile, LKImport } from './ldml-keyboard-xml.js'; import Ajv from 'ajv'; -import * as fs from 'fs'; import { boxXmlArray } from '../util/util.js'; +import { CompilerCallbacks } from 'src/util/compiler.js'; export default class LDMLKeyboardXMLSourceFileReader { + callbacks: CompilerCallbacks; + + constructor(callbacks : CompilerCallbacks) { + this.callbacks = callbacks; + } + readImportFile(version: string, subpath: string): Buffer { // TODO-LDML: sanitize input string let importPath = new URL(`../import/${version}/${subpath}`, import.meta.url); - return fs.readFileSync(importPath); + // TODO-LDML: support baseFileName? + return this.callbacks.loadFile(importPath.pathname, importPath); } /** diff --git a/common/web/types/test/ldml-keyboard/test-ldml-keyboard-xml-reader.ts b/common/web/types/test/ldml-keyboard/test-ldml-keyboard-xml-reader.ts index a2c651731f..de83086f37 100644 --- a/common/web/types/test/ldml-keyboard/test-ldml-keyboard-xml-reader.ts +++ b/common/web/types/test/ldml-keyboard/test-ldml-keyboard-xml-reader.ts @@ -2,16 +2,33 @@ import 'mocha'; import {assert} from 'chai'; import {loadLdmlKeyboardSchema, loadFile, makePathToFixture} from '../helpers/index.js'; import LDMLKeyboardXMLSourceFileReader from '../../src/ldml-keyboard/ldml-keyboard-xml-reader.js'; +import { CompilerCallbacks, CompilerEvent } from '../../src/util/compiler.js'; describe('ldml keyboard xml reader tests', function() { this.slow(500); // 0.5 sec -- json schema validation takes a while + const callbacks : CompilerCallbacks = { + loadFile: function (baseFilename: string, filename: string | URL): Buffer { + throw new Error('Function not implemented.'); + }, + loadLdmlKeyboardSchema: function (): Buffer { + throw new Error('Function not implemented.'); + }, + reportMessage: function (event: CompilerEvent): void { + throw new Error('Function not implemented.'); + }, + loadKvksJsonSchema: function (): Buffer { + throw new Error('Function not implemented.'); + } + }; + it("should fail to load files that don't conform to DTD", function() { const inputFilename = makePathToFixture('invalid-structure-per-dtd.xml'); - let reader = new LDMLKeyboardXMLSourceFileReader(); + let reader = new LDMLKeyboardXMLSourceFileReader(callbacks); const data = loadFile(inputFilename, inputFilename); const source = reader.load(data); assert.isNotNull(source); + // TODO-LDML: could report a messge instead of throw assert.throws(() => { reader.validate(source, loadLdmlKeyboardSchema()); }, `/keyboard: required: must have required property 'names' {"missingProperty":"names"}`); @@ -19,10 +36,11 @@ describe('ldml keyboard xml reader tests', function() { it("should fail to load files with an invalid conformsTo", function() { const inputFilename = makePathToFixture('invalid-conforms-to.xml'); - let reader = new LDMLKeyboardXMLSourceFileReader(); + let reader = new LDMLKeyboardXMLSourceFileReader(callbacks); const data = loadFile(inputFilename, inputFilename); const source = reader.load(data); assert.isNotNull(source); + // TODO-LDML: could report a messge instead of throw assert.throws(() => { reader.validate(source, loadLdmlKeyboardSchema()); }, `/keyboard/conformsTo: enum: must be equal to one of the allowed values {"allowedValues":["techpreview"]}`); diff --git a/developer/src/kmc-keyboard/src/compiler/compiler.ts b/developer/src/kmc-keyboard/src/compiler/compiler.ts index 450eb70048..f6a8423c84 100644 --- a/developer/src/kmc-keyboard/src/compiler/compiler.ts +++ b/developer/src/kmc-keyboard/src/compiler/compiler.ts @@ -56,7 +56,7 @@ export default class Compiler { * @returns */ public load(filename: string): LDMLKeyboardXMLSourceFile { - const reader = new LDMLKeyboardXMLSourceFileReader(); + const reader = new LDMLKeyboardXMLSourceFileReader(this.callbacks); const data = this.callbacks.loadFile(filename, filename); const source = reader.load(data); if(!source) { diff --git a/developer/src/kmc-keyboard/test/helpers/index.ts b/developer/src/kmc-keyboard/test/helpers/index.ts index 210ed90312..73b93cd473 100644 --- a/developer/src/kmc-keyboard/test/helpers/index.ts +++ b/developer/src/kmc-keyboard/test/helpers/index.ts @@ -71,7 +71,7 @@ export function loadSectionFixture(compilerClass: typeof SectionCompiler, filena const data = callbacks.loadFile(inputFilename, inputFilename); assert.isNotNull(data); - const reader = new LDMLKeyboardXMLSourceFileReader(); + const reader = new LDMLKeyboardXMLSourceFileReader(callbacks); const source = reader.load(data); assert.isNotNull(source); assert.doesNotThrow(() => { From ed30de26421777bd42b15becc6b4a4b59285c9af Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 26 Jan 2023 18:24:54 -0600 Subject: [PATCH 07/10] =?UTF-8?q?feat(core):=20ldml:=20add=20cldr=20versio?= =?UTF-8?q?n=20to=20constants=20=F0=9F=99=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - also improve constant builder - also fix build.sh so it uses sourcemaps supports #7964 --- core/include/ldml/keyboardprocessor_ldml.h | 6 ++- core/include/ldml/keyboardprocessor_ldml.ts | 16 ++++++ core/include/ldml/ldml-const-builder.ts | 56 +++++++++++---------- core/tools/ldml-const-builder/build.sh | 2 +- 4 files changed, 51 insertions(+), 29 deletions(-) diff --git a/core/include/ldml/keyboardprocessor_ldml.h b/core/include/ldml/keyboardprocessor_ldml.h index 5bab7018f4..ec08b4f234 100644 --- a/core/include/ldml/keyboardprocessor_ldml.h +++ b/core/include/ldml/keyboardprocessor_ldml.h @@ -1,6 +1,6 @@ /* - Copyright: Copyright (C) 2022 SIL International. + Copyright: Copyright (C) 2022-2023 SIL International. Authors: srl295 This file provides constants for the KMX Plus (LDML support) binary format, to be shared between TypeScript and C++ via the generator (below) @@ -16,6 +16,10 @@ #pragma once #define LDML_BKSP_FLAGS_ERROR 0x1 +#define LDML_CLDR_IMPLIED_KEYS_IMPORT "techpreview/keys-Latn-implied.xml" +#define LDML_CLDR_IMPORT_BASE "cldr" +#define LDML_CLDR_VERSION_LATEST "techpreview" +#define LDML_CLDR_VERSION_TECHPREVIEW "techpreview" #define LDML_ELEM_FLAGS_ORDER_BITSHIFT 0x10 #define LDML_ELEM_FLAGS_ORDER_MASK 0xFF0000 #define LDML_ELEM_FLAGS_PREBASE 0x4 diff --git a/core/include/ldml/keyboardprocessor_ldml.ts b/core/include/ldml/keyboardprocessor_ldml.ts index 08fc02036e..bc9d8635ae 100644 --- a/core/include/ldml/keyboardprocessor_ldml.ts +++ b/core/include/ldml/keyboardprocessor_ldml.ts @@ -59,6 +59,22 @@ class Constants { * The version of the LDML processor */ readonly version = '1.0'; + /** + * The techpreview CLDR version + */ + readonly cldr_version_techpreview = 'techpreview'; + /** + * The latest CLDR version + */ + readonly cldr_version_latest = this.cldr_version_techpreview; + /** + * import base + */ + readonly cldr_import_base = 'cldr'; + /** + * implied keys file + */ + readonly cldr_implied_keys_import = `${this.cldr_version_techpreview}/keys-Latn-implied.xml`; /** * Length of a raw section header, in bytes */ diff --git a/core/include/ldml/ldml-const-builder.ts b/core/include/ldml/ldml-const-builder.ts index ad68daeda7..56bdf19c82 100644 --- a/core/include/ldml/ldml-const-builder.ts +++ b/core/include/ldml/ldml-const-builder.ts @@ -10,7 +10,7 @@ const keys = Object.keys(constants); keys.sort(); console.log(` /* - Copyright: Copyright (C) 2022 SIL International. + Copyright: Copyright (C) 2022-2023 SIL International. Authors: srl295 This file provides constants for the KMX Plus (LDML support) binary format, to be shared between TypeScript and C++ via the generator (below) @@ -29,35 +29,37 @@ console.log(` let errs = 0; for (const key of keys) { - const value : any = constants[key as keyof typeof constants]; - const upkey = key.toUpperCase(); - const type = typeof value; - if (type === 'number') { - console.log(`#define LDML_${upkey} 0x${value.toString(16).toUpperCase()}`); - } else if (type === 'string') { - console.log(`#define LDML_${upkey} "${value}"`); - } else if (key === 'section') { - // handle section table - const subkeys = Object.keys(value); - subkeys.sort(); - for (const subkey of subkeys) { - const upsubkey = subkey.toUpperCase(); - const subvalue = value[subkey]; - if (subvalue !== subkey) { - // "can't happen" because tsc would complain - console.error(`In the SectionMap: ${subkey}: '${subvalue}' - expected key and value to match.`); - errs++; - } - const asnum = constants.hex_section_id(subkey); - console.log(`#define LDML_${upkey}ID_${upsubkey} 0x${asnum.toString(16).toUpperCase()} /* "${subkey}" */`); - console.log(`#define LDML_${upkey}NAME_${upsubkey} "${subkey}"`); - } - } else if (type !== 'function') { - console.error(`Don’t know what to do with constants[${key}] of type ${type}`); + const value: any = constants[key as keyof typeof constants]; + const upkey = key.toUpperCase(); + const type = typeof value; + if (type === 'number') { + console.log(`#define LDML_${upkey} 0x${value.toString(16).toUpperCase()}`); + } else if (type === 'string') { + console.log(`#define LDML_${upkey} "${value}"`); + } else if (key === 'section') { + // handle section table + const subkeys = Object.keys(value); + subkeys.sort(); + for (const subkey of subkeys) { + const upsubkey = subkey.toUpperCase(); + const subvalue = value[subkey]; + if (subvalue !== subkey) { + // "can't happen" because tsc would complain + console.error(`In the SectionMap: ${subkey}: '${subvalue}' - expected key and value to match.`); errs++; + } + const asnum = constants.hex_section_id(subkey); + console.log(`#define LDML_${upkey}ID_${upsubkey} 0x${asnum.toString(16).toUpperCase()} /* "${subkey}" */`); + console.log(`#define LDML_${upkey}NAME_${upsubkey} "${subkey}"`); } + } else if (key === 'layr_list_hardware_map' || type === 'function') { + // ignored + } else { + console.error(`Don’t know what to do with constants[${key}] of type ${type}`); + errs++; + } } if (errs != 0) { - throw Error(`Fail: ${errs} error(s), see above.`); + throw Error(`Fail: ${errs} error(s), see above.`); } diff --git a/core/tools/ldml-const-builder/build.sh b/core/tools/ldml-const-builder/build.sh index ce25033998..ad49783b61 100755 --- a/core/tools/ldml-const-builder/build.sh +++ b/core/tools/ldml-const-builder/build.sh @@ -42,7 +42,7 @@ if builder_start_action build; then fi if builder_start_action run; then - node ../../include/ldml/ldml-const-builder/ldml-const-builder.js > ${KBP_LDML_H_FILE} + node --enable-source-maps ../../include/ldml/ldml-const-builder/ldml-const-builder.js > ${KBP_LDML_H_FILE} echo "Updated ${KBP_LDML_H_FILE}" builder_finish_action success run From a0b184a0224b8ed49ea37fb549140a47a75ca128 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 26 Jan 2023 18:38:42 -0600 Subject: [PATCH 08/10] =?UTF-8?q?feat(common):=20ldml:=20use=20cldr=20vers?= =?UTF-8?q?ion=20from=20constants=20=F0=9F=99=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - update several config files supports #7964 --- common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts | 3 ++- common/web/types/tsconfig.json | 2 +- package-lock.json | 3 ++- package.json | 3 ++- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts b/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts index 0ee61dfe1d..343a4ba825 100644 --- a/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -3,6 +3,7 @@ import { LDMLKeyboardXMLSourceFile, LKImport } from './ldml-keyboard-xml.js'; import Ajv from 'ajv'; import { boxXmlArray } from '../util/util.js'; import { CompilerCallbacks } from 'src/util/compiler.js'; +import { constants } from '@keymanapp/ldml-keyboard-constants'; export default class LDMLKeyboardXMLSourceFileReader { callbacks: CompilerCallbacks; @@ -106,7 +107,7 @@ export default class LDMLKeyboardXMLSourceFileReader { // this.resolveOneImport(obj, subtag, { base: 'cldr', - path: 'techpreview/keys-Latn-implied.xml' + path: constants.cldr_implied_keys_import }); } diff --git a/common/web/types/tsconfig.json b/common/web/types/tsconfig.json index 9600270371..a32b8e1ed7 100644 --- a/common/web/types/tsconfig.json +++ b/common/web/types/tsconfig.json @@ -12,6 +12,6 @@ ], "references": [ { "path": "../keyman-version/tsconfig.esm.json" }, - { "path": "../../../core/include/ldml/"}, + { "path": "../../../core/include/ldml/tsconfig.json"}, ] } diff --git a/package-lock.json b/package-lock.json index cff877e90e..a6cc4380f6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,7 +24,8 @@ "dependencies": { "@keymanapp/common-types": "file:common/web/types", "@keymanapp/hextobin": "file:common/tools/hextobin", - "@keymanapp/keyman-version": "file:common/web/keyman-version" + "@keymanapp/keyman-version": "file:common/web/keyman-version", + "@keymanapp/ldml-keyboard-constants": "file:core/include/ldml" }, "devDependencies": { "chai": "^4.3.4", diff --git a/package.json b/package.json index 09c73d7a69..4f33d63962 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "dependencies": { "@keymanapp/hextobin": "file:common/tools/hextobin", "@keymanapp/keyman-version": "file:common/web/keyman-version", - "@keymanapp/common-types": "file:common/web/types" + "@keymanapp/common-types": "file:common/web/types", + "@keymanapp/ldml-keyboard-constants": "file:core/include/ldml" } } From bbd850c03b2e08a0fa80747c7a7fe5ab28d5ee95 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 27 Jan 2023 12:33:25 -0600 Subject: [PATCH 09/10] =?UTF-8?q?feat(common):=20ldml:=20imports:=20use=20?= =?UTF-8?q?cldr=20version=20from=20constants=20=F0=9F=99=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - common: more robustness in importing. - build: don't copy imports to kmc which doesn't need it supports #7964 --- common/web/types/build.sh | 2 +- .../ldml-keyboard/ldml-keyboard-xml-reader.ts | 38 +++++++++---------- developer/src/kmc/build.sh | 10 ----- 3 files changed, 19 insertions(+), 31 deletions(-) diff --git a/common/web/types/build.sh b/common/web/types/build.sh index 2359366f84..a4fac777d4 100755 --- a/common/web/types/build.sh +++ b/common/web/types/build.sh @@ -48,7 +48,7 @@ else CLDR_PATH=$(dirname "$CLDR_INFO_PATH") CLDR_VER=$(basename "$CLDR_PATH") mkdir -p "$THIS_SCRIPT_PATH/build/src/import/$CLDR_VER" - # TODO-LDML: When these are copied, the DOCTYPE will break due to the wrong path. + # TODO-LDML: When these are copied, the DOCTYPE will break due to the wrong path. We don't use the DTD so it should be OK. cp "$CLDR_INFO_PATH" "$CLDR_PATH/import/"*.xml "$THIS_SCRIPT_PATH/build/src/import/$CLDR_VER/" done fi diff --git a/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts b/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts index 343a4ba825..dc1a05d03b 100644 --- a/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -25,22 +25,21 @@ export default class LDMLKeyboardXMLSourceFileReader { * @param source any */ private boxArrays(source: any) { + if (source?.keyboard) { + if (!source.keyboard.keys) { + source.keyboard.keys = { + key: [], + flicks: [], + }; + } + if (!source.keyboard.keys?.import) { + source.keyboard.keys.import = []; + } + } boxXmlArray(source?.keyboard, 'layers'); boxXmlArray(source?.keyboard?.displays, 'display'); boxXmlArray(source?.keyboard?.names, 'name'); boxXmlArray(source?.keyboard?.vkeys, 'vkey'); - // for implied we need keyboard.keys.import to exist. - // TODO-LDML: will fail if source or source.keyboard is nullish - if (!source?.keyboard?.keys) { - source.keyboard.keys = { - import:[], - key:[], - flicks:[], - }; - } - if (!source?.keyboard?.keys?.import) { - source.keyboard.keys.import = []; - } boxXmlArray(source?.keyboard?.keys, 'key'); boxXmlArray(source?.keyboard?.keys, 'flicks'); boxXmlArray(source?.keyboard?.locales, 'locale'); @@ -106,7 +105,7 @@ export default class LDMLKeyboardXMLSourceFileReader { if (subtag === 'keys') { // this.resolveOneImport(obj, subtag, { - base: 'cldr', + base: constants.cldr_import_base, path: constants.cldr_implied_keys_import }); } @@ -118,32 +117,31 @@ export default class LDMLKeyboardXMLSourceFileReader { } private resolveOneImport(obj: any, subtag: string, asImport: LKImport) { - if (asImport.base !== 'cldr') { - throw new Error(`import element with base ${asImport.base} is unsupported.`); + if (asImport.base !== constants.cldr_import_base) { + throw new Error(`import element with base ${asImport.base} is unsupported, only ${constants.cldr_import_base} is supported.`); } const paths = asImport.path.split('/'); if (!paths[0] || !paths[1] || paths.length !== 2) { - throw new Error(`import element with invalid path ${asImport.path}: expect the form 'techpreview/*.xml'`); + throw new Error(`import element with invalid path ${asImport.path}: expect the form '${constants.cldr_version_latest}./*.xml'`); } const importData: Uint8Array = this.readImportFile(paths[0], paths[1]); if (!importData || !importData.length) { - throw new Error(`could not read data with path ${asImport.path}: expect the form 'techpreview/*.xml'`); + throw new Error(`could not read data with path ${asImport.path}: expect the form '${constants.cldr_version_latest}./*.xml'`); } const importXml: any = this.loadUnboxed(importData); // TODO-LDML: have to load as any because it is an arbitrary part - const importRootNode = importXml[subtag]; // e.g. keys + const importRootNode = importXml[subtag]; // e.g. // importXml will have one property: the root element. if (!importRootNode) { throw new Error(`Invalid import file ${asImport.path}: expected ${subtag} as root element`); } // pull all children of importXml[subtag] into obj - for (const subsubtag of Object.keys(importRootNode)) { // e.g. key + for (const subsubtag of Object.keys(importRootNode)) { // e.g. const subsubval = importRootNode[subsubtag]; if (!Array.isArray(subsubval)) { throw new Error(`Problem importing ${asImport.path}: not sure how to handle non-array ${subtag}.${subsubtag}`); } if (!obj[subsubtag]) { - // TODO-LDML: this probably should be an error if this doesn't exist obj[subsubtag] = []; // start with empty array } obj[subsubtag] = [...subsubval, ...obj[subsubtag]]; diff --git a/developer/src/kmc/build.sh b/developer/src/kmc/build.sh index df58f406ef..6b0b207227 100755 --- a/developer/src/kmc/build.sh +++ b/developer/src/kmc/build.sh @@ -46,16 +46,6 @@ else # We need the schema file at runtime and bundled, so always copy it for all actions except `clean` mkdir -p "$THIS_SCRIPT_PATH/build/src/" cp "$KEYMAN_ROOT/resources/standards-data/ldml-keyboards/techpreview/ldml-keyboard.schema.json" "$THIS_SCRIPT_PATH/build/src/" - # Store imports - # load all versions that have a cldr_info.json - for CLDR_INFO_PATH in "$KEYMAN_ROOT/resources/standards-data/ldml-keyboards/"*/cldr_info.json - do - CLDR_PATH=$(dirname "$CLDR_INFO_PATH") - CLDR_VER=$(basename "$CLDR_PATH") - mkdir -p "$THIS_SCRIPT_PATH/build/src/import/$CLDR_VER" - # TODO-LDML: When these are copied, the DOCTYPE will break due to the wrong path. - cp "$CLDR_INFO_PATH" "$CLDR_PATH/import/"*.xml "$THIS_SCRIPT_PATH/build/src/import/$CLDR_VER/" - done fi From a40ee0176749e8079824beceb48d6ff5dc7bb2e9 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 27 Jan 2023 13:06:00 -0600 Subject: [PATCH 10/10] =?UTF-8?q?feat(developer):=20ldml=20import/implied?= =?UTF-8?q?=20keys=20:=20review=20comment=20=F0=9F=99=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - add a TODO comment about using a custom error class --- common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts b/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts index dc1a05d03b..5558fd7199 100644 --- a/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -118,10 +118,11 @@ export default class LDMLKeyboardXMLSourceFileReader { private resolveOneImport(obj: any, subtag: string, asImport: LKImport) { if (asImport.base !== constants.cldr_import_base) { + // TODO-LDML: use custom error class CompilerImportResolveError throw new Error(`import element with base ${asImport.base} is unsupported, only ${constants.cldr_import_base} is supported.`); } const paths = asImport.path.split('/'); - if (!paths[0] || !paths[1] || paths.length !== 2) { + if (paths[0] == '' || paths[1] == '' || paths.length !== 2) { throw new Error(`import element with invalid path ${asImport.path}: expect the form '${constants.cldr_version_latest}./*.xml'`); } const importData: Uint8Array = this.readImportFile(paths[0], paths[1]);