Merge pull request #7402 from keymanapp/fix/developer/epic-ldml/7401-kmc-broken-kbds

feat(developer): implement import and implied 🙀

common/web/types/build.sh

@@ -41,6 +41,16 @@ else
   cp "$KEYMAN_ROOT/resources/standards-data/ldml-keyboards/techpreview/ldml-keyboard.schema.json" "$THIS_SCRIPT_PATH/build/src/"
   cp "$KEYMAN_ROOT/common/schemas/kvks/kvks.schema.json" "$THIS_SCRIPT_PATH/build/src/"
   cp "$KEYMAN_ROOT/common/schemas/keyman-touch-layout/keyman-touch-layout.clean.spec.json" "$THIS_SCRIPT_PATH/build/src/"
+  # Store CLDR imports
+  # load all versions that have a cldr_info.json
+  for CLDR_INFO_PATH in "$KEYMAN_ROOT/resources/standards-data/ldml-keyboards/"*/cldr_info.json
+  do
+    CLDR_PATH=$(dirname "$CLDR_INFO_PATH")
+    CLDR_VER=$(basename "$CLDR_PATH")
+    mkdir -p "$THIS_SCRIPT_PATH/build/src/import/$CLDR_VER"
+    # TODO-LDML: When these are copied, the DOCTYPE will break due to the wrong path. We don't use the DTD so it should be OK.
+    cp "$CLDR_INFO_PATH" "$CLDR_PATH/import/"*.xml "$THIS_SCRIPT_PATH/build/src/import/$CLDR_VER/"
+  done
 fi
 
 #-------------------------------------------------------------------------------------------------------------------

common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts

@@ -1,15 +1,41 @@
 import * as xml2js from 'xml2js';
-import { LDMLKeyboardXMLSourceFile } from './ldml-keyboard-xml.js';
+import { LDMLKeyboardXMLSourceFile, LKImport } from './ldml-keyboard-xml.js';
 import Ajv from 'ajv';
 import { boxXmlArray } from '../util/util.js';
+import { CompilerCallbacks } from 'src/util/compiler.js';
+import { constants } from '@keymanapp/ldml-keyboard-constants';
 
 export default class LDMLKeyboardXMLSourceFileReader {
+  callbacks: CompilerCallbacks;
+
+  constructor(callbacks : CompilerCallbacks) {
+    this.callbacks = callbacks;
+  }
+
+  readImportFile(version: string, subpath: string): Buffer {
+    // TODO-LDML: sanitize input string
+    let importPath = new URL(`../import/${version}/${subpath}`, import.meta.url);
+    // TODO-LDML: support baseFileName?
+    return this.callbacks.loadFile(importPath.pathname, importPath);
+  }
+
   /**
    * xml2js will not place single-entry objects into arrays.
    * Easiest way to fix this is to box them ourselves as needed
    * @param source any
    */
   private boxArrays(source: any) {
+    if (source?.keyboard) {
+      if (!source.keyboard.keys) {
+        source.keyboard.keys = {
+          key: [],
+          flicks: [],
+        };
+      }
+      if (!source.keyboard.keys?.import) {
+        source.keyboard.keys.import = [];
+      }
+    }
     boxXmlArray(source?.keyboard, 'layers');
     boxXmlArray(source?.keyboard?.displays, 'display');
     boxXmlArray(source?.keyboard?.names, 'name');
@@ -39,31 +65,90 @@ export default class LDMLKeyboardXMLSourceFileReader {
       }
     }
     boxXmlArray(source?.keyboard?.reorders, 'reorder');
-    this.boxImportsAndSpecials(source);
+    this.boxImportsAndSpecials(source, 'keyboard');
     return source;
   }
 
   /**
    * Recurse over object, boxing up any specials or imports
    * @param obj any object to be traversed
+   * @param subtag the leafmost enclosing tag such as 'keyboard'
    */
-  private boxImportsAndSpecials(obj: any) {
+  private boxImportsAndSpecials(obj: any, subtag: string) {
     if (!obj) return;
     if (Array.isArray(obj)) {
       for (const sub of obj) {
-        this.boxImportsAndSpecials(sub);
+        // retain the same subtag
+        this.boxImportsAndSpecials(sub, subtag);
       }
     } else if(typeof obj === 'object') {
       for (const key of Object.keys(obj)) {
-        if (key === 'special' || key === 'import') {
+        if (key === 'special') {
           boxXmlArray(obj, key);
+        } else if(key === 'import') {
+          // Need to 'box it up' first for processing
+          boxXmlArray(obj, key);
+          // Now, resolve the import
+          this.resolveImports(obj, subtag);
+          // now delete the import array we so carefully constructed, the caller does not
+          // want to see it.
+          delete obj['import'];
         } else {
-          this.boxImportsAndSpecials(obj[key]);
+          this.boxImportsAndSpecials(obj[key], key);
         }
       }
     }
   }
 
+  private resolveImports(obj: any, subtag: string) {
+    // first, the implied imports
+    if (subtag === 'keys') {
+      // <import base="cldr" path="techpreview/keys-Latn-implied.xml"/>
+      this.resolveOneImport(obj, subtag, {
+        base: constants.cldr_import_base,
+        path: constants.cldr_implied_keys_import
+      });
+    }
+
+    // now, the explicit imports
+    for (const asImport of (obj['import'] as LKImport[])) {
+      this.resolveOneImport(obj, subtag, asImport);
+    }
+  }
+
+  private resolveOneImport(obj: any, subtag: string, asImport: LKImport) {
+    if (asImport.base !== constants.cldr_import_base) {
+      // TODO-LDML: use custom error class CompilerImportResolveError
+      throw new Error(`import element with base ${asImport.base} is unsupported, only ${constants.cldr_import_base} is supported.`);
+    }
+    const paths = asImport.path.split('/');
+    if (paths[0] == '' || paths[1] == '' || paths.length !== 2) {
+      throw new Error(`import element with invalid path ${asImport.path}: expect the form '${constants.cldr_version_latest}./*.xml'`);
+    }
+    const importData: Uint8Array = this.readImportFile(paths[0], paths[1]);
+    if (!importData || !importData.length) {
+      throw new Error(`could not read data with path ${asImport.path}: expect the form '${constants.cldr_version_latest}./*.xml'`);
+    }
+    const importXml: any = this.loadUnboxed(importData); // TODO-LDML: have to load as any because it is an arbitrary part
+    const importRootNode = importXml[subtag]; // e.g. <keys/>
+
+    // importXml will have one property: the root element.
+    if (!importRootNode) {
+      throw new Error(`Invalid import file ${asImport.path}: expected ${subtag} as root element`);
+    }
+    // pull all children of importXml[subtag] into obj
+    for (const subsubtag of Object.keys(importRootNode)) { // e.g. <key/>
+      const subsubval = importRootNode[subsubtag];
+      if (!Array.isArray(subsubval)) {
+        throw new Error(`Problem importing ${asImport.path}: not sure how to handle non-array ${subtag}.${subsubtag}`);
+      }
+      if (!obj[subsubtag]) {
+        obj[subsubtag] = []; // start with empty array
+      }
+      obj[subsubtag] = [...subsubval, ...obj[subsubtag]];
+    }
+  }
+
   public validate(source: LDMLKeyboardXMLSourceFile, schemaSource: Buffer): void {
     const schema = JSON.parse(schemaSource.toString('utf8'));
     const ajv = new Ajv();
@@ -81,7 +166,7 @@ export default class LDMLKeyboardXMLSourceFileReader {
     }
   }
 
-  public load(file: Uint8Array): LDMLKeyboardXMLSourceFile {
+  loadUnboxed(file: Uint8Array): LDMLKeyboardXMLSourceFile {
     let source = (() => {
       let a: LDMLKeyboardXMLSourceFile;
       let parser = new xml2js.Parser({
@@ -101,7 +186,11 @@ export default class LDMLKeyboardXMLSourceFileReader {
       parser.parseString(file, (e: unknown, r: unknown) => { a = r as LDMLKeyboardXMLSourceFile });
       return a;
     })();
+    return source;
+  }
 
+  public load(file: Uint8Array): LDMLKeyboardXMLSourceFile {
+    const source = this.loadUnboxed(file);
     return this.boxArrays(source);
   }
 }

common/web/types/src/ldml-keyboard/ldml-keyboard-xml.ts

@@ -31,6 +31,20 @@ export interface LKKeyboard {
   reorders?: LKReorders;
 };
 
+/**
+ * This is defined as an interface, but actually is resolved during the reading phase
+ */
+export interface LKImport {
+  /**
+   * import base, currently `cldr` is supported
+   */
+  base: string;
+  /**
+   * path to imported resource, of the form `techpreview/*.xml`
+   */
+  path: string;
+}
+
 export interface LKLocales {
   locale: LKLocale[];
 };

common/web/types/test/ldml-keyboard/test-ldml-keyboard-xml-reader.ts

@@ -2,16 +2,33 @@ import 'mocha';
 import {assert} from 'chai';
 import {loadLdmlKeyboardSchema, loadFile, makePathToFixture} from '../helpers/index.js';
 import LDMLKeyboardXMLSourceFileReader from '../../src/ldml-keyboard/ldml-keyboard-xml-reader.js';
+import { CompilerCallbacks, CompilerEvent } from '../../src/util/compiler.js';
 
 describe('ldml keyboard xml reader tests', function() {
   this.slow(500); // 0.5 sec -- json schema validation takes a while
 
+  const callbacks : CompilerCallbacks = {
+    loadFile: function (baseFilename: string, filename: string | URL): Buffer {
+      throw new Error('Function not implemented.');
+    },
+    loadLdmlKeyboardSchema: function (): Buffer {
+      throw new Error('Function not implemented.');
+    },
+    reportMessage: function (event: CompilerEvent): void {
+      throw new Error('Function not implemented.');
+    },
+    loadKvksJsonSchema: function (): Buffer {
+      throw new Error('Function not implemented.');
+    }
+  };
+
   it("should fail to load files that don't conform to DTD", function() {
     const inputFilename = makePathToFixture('invalid-structure-per-dtd.xml');
-    let reader = new LDMLKeyboardXMLSourceFileReader();
+    let reader = new LDMLKeyboardXMLSourceFileReader(callbacks);
     const data = loadFile(inputFilename, inputFilename);
     const source = reader.load(data);
     assert.isNotNull(source);
+    // TODO-LDML: could report a messge instead of throw
     assert.throws(() => {
       reader.validate(source, loadLdmlKeyboardSchema());
     }, `/keyboard: required: must have required property 'names' {"missingProperty":"names"}`);
@@ -19,10 +36,11 @@ describe('ldml keyboard xml reader tests', function() {
 
   it("should fail to load files with an invalid conformsTo", function() {
     const inputFilename = makePathToFixture('invalid-conforms-to.xml');
-    let reader = new LDMLKeyboardXMLSourceFileReader();
+    let reader = new LDMLKeyboardXMLSourceFileReader(callbacks);
     const data = loadFile(inputFilename, inputFilename);
     const source = reader.load(data);
     assert.isNotNull(source);
+    // TODO-LDML: could report a messge instead of throw
     assert.throws(() => {
       reader.validate(source, loadLdmlKeyboardSchema());
     }, `/keyboard/conformsTo: enum: must be equal to one of the allowed values {"allowedValues":["techpreview"]}`);

common/web/types/tsconfig.json

@@ -12,6 +12,6 @@
   ],
   "references": [
     { "path": "../keyman-version/tsconfig.esm.json" },
-    { "path": "../../../core/include/ldml/"},
+    { "path": "../../../core/include/ldml/tsconfig.json"},
   ]
 }

core/include/ldml/keyboardprocessor_ldml.h

@@ -1,6 +1,6 @@
 
 /*
-  Copyright:        Copyright (C) 2022 SIL International.
+  Copyright:        Copyright (C) 2022-2023 SIL International.
   Authors:          srl295
   This file provides constants for the KMX Plus (LDML support) binary format,
   to be shared between TypeScript and C++ via the generator (below)
@@ -16,6 +16,10 @@
 #pragma once
 
 #define LDML_BKSP_FLAGS_ERROR 0x1
+#define LDML_CLDR_IMPLIED_KEYS_IMPORT "techpreview/keys-Latn-implied.xml"
+#define LDML_CLDR_IMPORT_BASE "cldr"
+#define LDML_CLDR_VERSION_LATEST "techpreview"
+#define LDML_CLDR_VERSION_TECHPREVIEW "techpreview"
 #define LDML_ELEM_FLAGS_ORDER_BITSHIFT 0x10
 #define LDML_ELEM_FLAGS_ORDER_MASK 0xFF0000
 #define LDML_ELEM_FLAGS_PREBASE 0x4

core/include/ldml/keyboardprocessor_ldml.ts

@@ -59,6 +59,22 @@ class Constants {
    * The version of the LDML processor
    */
   readonly version = '1.0';
+  /**
+   * The techpreview CLDR version
+   */
+  readonly cldr_version_techpreview = 'techpreview';
+  /**
+   * The latest CLDR version
+   */
+  readonly cldr_version_latest = this.cldr_version_techpreview;
+  /**
+   * import base
+   */
+  readonly cldr_import_base = 'cldr';
+  /**
+   * implied keys file
+   */
+  readonly cldr_implied_keys_import = `${this.cldr_version_techpreview}/keys-Latn-implied.xml`;
   /**
    * Length of a raw section header, in bytes
    */

core/include/ldml/ldml-const-builder.ts

@@ -10,7 +10,7 @@ const keys = Object.keys(constants);
 keys.sort();
 console.log(`
 /*
-  Copyright:        Copyright (C) 2022 SIL International.
+  Copyright:        Copyright (C) 2022-2023 SIL International.
   Authors:          srl295
   This file provides constants for the KMX Plus (LDML support) binary format,
   to be shared between TypeScript and C++ via the generator (below)
@@ -29,35 +29,37 @@ console.log(`
 let errs = 0;
 
 for (const key of keys) {
-    const value : any = constants[key as keyof typeof constants];
-    const upkey = key.toUpperCase();
-    const type = typeof value;
-    if (type === 'number') {
-        console.log(`#define LDML_${upkey} 0x${value.toString(16).toUpperCase()}`);
-    } else if (type === 'string') {
-        console.log(`#define LDML_${upkey} "${value}"`);
-    } else if (key === 'section') {
-        // handle section table
-        const subkeys = Object.keys(value);
-        subkeys.sort();
-        for (const subkey of subkeys) {
-            const upsubkey = subkey.toUpperCase();
-            const subvalue = value[subkey];
-            if (subvalue !== subkey) {
-                // "can't happen" because tsc would complain
-                console.error(`In the SectionMap:  ${subkey}: '${subvalue}' - expected key and value to match.`);
-                errs++;
-            }
-            const asnum = constants.hex_section_id(subkey);
-            console.log(`#define LDML_${upkey}ID_${upsubkey} 0x${asnum.toString(16).toUpperCase()} /* "${subkey}" */`);
-            console.log(`#define LDML_${upkey}NAME_${upsubkey}             "${subkey}"`);
-        }
-    } else if (type !== 'function') {
-        console.error(`Don’t know what to do with constants[${key}] of type ${type}`);
+  const value: any = constants[key as keyof typeof constants];
+  const upkey = key.toUpperCase();
+  const type = typeof value;
+  if (type === 'number') {
+    console.log(`#define LDML_${upkey} 0x${value.toString(16).toUpperCase()}`);
+  } else if (type === 'string') {
+    console.log(`#define LDML_${upkey} "${value}"`);
+  } else if (key === 'section') {
+    // handle section table
+    const subkeys = Object.keys(value);
+    subkeys.sort();
+    for (const subkey of subkeys) {
+      const upsubkey = subkey.toUpperCase();
+      const subvalue = value[subkey];
+      if (subvalue !== subkey) {
+        // "can't happen" because tsc would complain
+        console.error(`In the SectionMap:  ${subkey}: '${subvalue}' - expected key and value to match.`);
         errs++;
+      }
+      const asnum = constants.hex_section_id(subkey);
+      console.log(`#define LDML_${upkey}ID_${upsubkey} 0x${asnum.toString(16).toUpperCase()} /* "${subkey}" */`);
+      console.log(`#define LDML_${upkey}NAME_${upsubkey}             "${subkey}"`);
     }
+  } else if (key === 'layr_list_hardware_map' || type === 'function') {
+    // ignored
+  } else {
+    console.error(`Don’t know what to do with constants[${key}] of type ${type}`);
+    errs++;
+  }
 }
 
 if (errs != 0) {
-    throw Error(`Fail: ${errs} error(s), see above.`);
+  throw Error(`Fail: ${errs} error(s), see above.`);
 }

core/tests/unit/ldml/keyboards/k_003_transform.xml

@@ -18,37 +18,10 @@ Note: keys should start with [K_A][K_B][K_C] not [K_Y] when transforms are funct
   </names>
 
   <keys>
-    <!-- <key id="gap" gap="true" /> -->
-    <key id="space" to=" " width="5" />
-
-    <!-- bug: these keys should not need to be specified -->
-    <key id="a" to="a" />
-    <key id="b" to="b" />
-    <key id="c" to="c" />
-    <key id="d" to="d" />
-    <key id="e" to="e" />
-    <key id="f" to="f" />
-    <key id="g" to="g" />
-    <key id="h" to="h" />
-    <key id="i" to="i" />
-    <key id="j" to="j" />
-    <key id="k" to="k" />
-    <key id="l" to="l" />
-    <key id="m" to="m" />
-    <key id="n" to="n" />
-    <key id="o" to="o" />
-    <key id="p" to="p" />
-    <key id="q" to="q" />
-    <key id="r" to="r" />
-    <key id="s" to="s" />
-    <key id="t" to="t" />
-    <key id="u" to="u" />
-    <key id="v" to="v" />
-    <key id="w" to="w" />
-    <key id="x" to="x" />
-    <key id="y" to="y" />
-    <key id="z" to="z" />
+    <key id="backslash" to="\u{005c}"/>
     <key id="bquote" to="`"/>
+    <key id="equals" to="="/>
+    <key id="hyphen" to="-"/>
   </keys>
 
   <layers form="hardware" hardware="us">

core/tests/unit/ldml/keyboards/k_010_mt.xml

@@ -25,12 +25,9 @@ Gets part of the way,
 
     <keys>
         <!-- imports -->
-        <!-- TODO-LDML: Added next: see #7964 -->
-        <import base="cldr" path="techpreview/keys-Latn-implied.xml"/>
         <import base="cldr" path="techpreview/keys-Zyyy-punctuation.xml"/>
-        <!-- TODO-LDML: Added next: see #7964 -->
-        <key id="space" to=" " width="5" />
-        <!-- TODO-LDML: added next -->
+
+        <!-- TODO-LDML: added next (but it is in a pending CLDR PR) #2652 -->
         <key id="asterisk" to="*" />
         <!-- accent grave -->
         <key id="a-grave" to="à" />
@@ -56,9 +53,6 @@ Gets part of the way,
 
         <!-- Cedilla -->
         <key id="c-cedilla" to="ç" />
-
-        <!-- gaps -->
-        <key id="gap" gap="true" width="1" />
     </keys>
 
 

core/tests/unit/ldml/keyboards/k_100_keytest.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+@@keys: [K_BKQUOTE]
+@@expected: \u0061
+-->
+
+<!DOCTYPE keyboard SYSTEM "../../../../../resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboard.dtd">
+<keyboard locale="en" conformsTo="techpreview">
+  <info author="srl295" indicator="🙀" layout="qwerty" normalization="NFC" />
+
+  <names>
+    <name value="TestKbd" />
+  </names>
+
+  <keys>
+    <!-- tests that gap="true" (with no to=) is OK -->
+    <key id="gap" gap="true" />
+  </keys>
+
+  <layers form="hardware" hardware="us">
+    <layer id="base">
+      <row keys="a gap"/>
+    </layer>
+  </layers>
+</keyboard>

core/tests/unit/ldml/keyboards/k_101_keytest.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+@@keys: [K_BKQUOTE]
+@@expected: \u0061
+-->
+<!DOCTYPE keyboard SYSTEM "../../../../../resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboard.dtd">
+<keyboard locale="en" conformsTo="techpreview">
+  <info author="srl295" indicator="🙀" layout="qwerty" normalization="NFC" />
+
+  <names>
+    <name value="TestKbd" />
+  </names>
+
+  <keys>
+    <!-- no keys listed, keys-Latn-implied.xml will be implicitly imported -->
+  </keys>
+
+  <layers form="hardware" hardware="us">
+    <layer id="base">
+      <row keys="a" /> <!-- number row -->
+    </layer>
+  </layers>
+</keyboard>

core/tests/unit/ldml/keyboards/k_102_keytest.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+@@keys: [K_BKQUOTE]
+@@expected: \u0061
+-->
+
+<!DOCTYPE keyboard SYSTEM "../../../../../resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboard.dtd">
+<keyboard locale="en" conformsTo="techpreview">
+  <info author="srl295" indicator="🙀" layout="qwerty" normalization="NFC" />
+
+  <names>
+    <name value="TestKbd" />
+  </names>
+  <!-- only implied keys - the <keys/> element is omitted. -->
+  <layers form="hardware" hardware="us">
+    <layer id="base">
+      <row keys="a" /> <!-- number row -->
+    </layer>
+  </layers>
+
+</keyboard>

core/tests/unit/ldml/keyboards/meson.build

@@ -10,8 +10,10 @@ tests = [
   'k_001_tiny',
   'k_002_tinyu32',
   'k_003_transform',
-# TODO-LDML: depends on implied keys
-#  'k_010_mt',
+  'k_010_mt',
+  'k_100_keytest',
+  'k_101_keytest',
+  'k_102_keytest',
 ]
 
 # Setup kmc

core/tools/ldml-const-builder/build.sh

@@ -42,7 +42,7 @@ if builder_start_action build; then
 fi
 
 if builder_start_action run; then
-  node ../../include/ldml/ldml-const-builder/ldml-const-builder.js > ${KBP_LDML_H_FILE}
+  node --enable-source-maps ../../include/ldml/ldml-const-builder/ldml-const-builder.js > ${KBP_LDML_H_FILE}
   echo "Updated ${KBP_LDML_H_FILE}"
 
   builder_finish_action success run

developer/src/kmc-keyboard/src/compiler/compiler.ts

@@ -56,7 +56,7 @@ export default class Compiler {
    * @returns
    */
   public load(filename: string): LDMLKeyboardXMLSourceFile {
-    const reader = new LDMLKeyboardXMLSourceFileReader();
+    const reader = new LDMLKeyboardXMLSourceFileReader(this.callbacks);
     const data = this.callbacks.loadFile(filename, filename);
     const source = reader.load(data);
     if(!source) {

developer/src/kmc-keyboard/src/compiler/key2.ts

@@ -20,9 +20,30 @@ export class Key2Compiler extends SectionCompiler {
     return valid;
   }
 
+  /**
+   * Return a set of all used keys
+   * public for test
+   */
+  public allUsedKeyIds(): Set<string> {
+    const s = new Set<string>();
+    if (this.keyboard?.layers) {
+      for (const layers of this.keyboard.layers || []) {
+        for (const layer of layers.layer || []) {
+          for (const row of layer.row || []) {
+            if (row.keys) {
+              for (const k of row.keys.split(" ")) {
+                s.add(k);
+              }
+            }
+          }
+        }
+      }
+    }
+    return s;
+  }
 
   public compile(sections: GlobalSections): Key2 {
-    if (!this.keyboard.keys.key && !this.keyboard.keys.flicks) {
+    if (!this.keyboard?.keys?.key && !this.keyboard?.keys?.flicks) {
       // short-circuit if no keys or flicks
       return null;
     }
@@ -60,7 +81,26 @@ export class Key2Compiler extends SectionCompiler {
   }
 
   public loadKeys(sections: GlobalSections, sect: Key2) {
-    for (let key of this.keyboard.keys.key) {
+    const usedKeys = this.allUsedKeyIds();
+
+    // TODO-LDML factor common code
+    // Need 'newer' (later) keys to override older ones.
+    const reverseKeys = [...this.keyboard.keys.key].reverse(); // newest to oldest
+    const alreadySeen = new Set<string>();
+    // filter out only the keys that haven't already been seen
+    const uniqueKeys = reverseKeys.filter(({id}) => {
+      if (!alreadySeen.has(id)) {
+        alreadySeen.add(id);
+        return true;
+      }
+      return false;
+    });
+
+    for (let key of uniqueKeys) {
+      if (!usedKeys.has(key.id)) {
+        // TODO-LDML: linting for unused keys
+        continue; // unused key, skip
+      }
       let flags = 0;
       const flicks = key.flicks;
       // TODO-LDML: verify that this flick id exists

developer/src/kmc-keyboard/src/compiler/keys.ts

@@ -14,6 +14,19 @@ export class KeysCompiler extends SectionCompiler {
   }
 
   private validateHardwareLayer(layer: LDMLKeyboard.LKLayer) {
+    // TODO-LDML factor common code
+    // Need 'newer' (later) keys to override older ones.
+    const reverseKeys = [...this.keyboard.keys?.key].reverse(); // newest to oldest
+    const alreadySeen = new Set<string>();
+    // filter out only the keys that haven't already been seen
+    const uniqueKeys = reverseKeys.filter(({id}) => {
+      if (!alreadySeen.has(id)) {
+        alreadySeen.add(id);
+        return true;
+      }
+      return false;
+    });
+
     let valid = true;
     if(layer.row.length > USVirtualKeyMap.length) {
       this.callbacks.reportMessage(CompilerMessages.Error_HardwareLayerHasTooManyRows());
@@ -32,7 +45,7 @@ export class KeysCompiler extends SectionCompiler {
       for(let key of keys) {
         x++;
 
-        let keydef = this.keyboard.keys?.key?.find(x => x.id == key);
+        let keydef = uniqueKeys.find(x => x.id == key);
         if(!keydef) {
           this.callbacks.reportMessage(CompilerMessages.Error_KeyNotFoundInKeyBag({keyId: key, col: x+1, row: y+1, layer: layer.id, form: 'hardware'}));
           valid = false;

developer/src/kmc-keyboard/test/helpers/index.ts

@@ -71,7 +71,7 @@ export function loadSectionFixture(compilerClass: typeof SectionCompiler, filena
   const data = callbacks.loadFile(inputFilename, inputFilename);
   assert.isNotNull(data);
 
-  const reader = new LDMLKeyboardXMLSourceFileReader();
+  const reader = new LDMLKeyboardXMLSourceFileReader(callbacks);
   const source = reader.load(data);
   assert.isNotNull(source);
   assert.doesNotThrow(() => {

developer/src/kmc-keyboard/test/test-key2.ts

@@ -29,7 +29,7 @@ describe('key2', function () {
     const [q] = key2.keys.filter(({ id }) => id.value === 'q');
     assert.ok(q);
     assert.isFalse(!!(q.flags & constants.key2_key_flags_gap));
-    assert.equal(q.width, 32); // ceil(3.14159 * 10.0)
+    assert.equal(q.width, 32, 'q\'s width'); // ceil(3.14159 * 10.0)
     assert.equal(q.flicks, 'flick0'); // note this is a string, not a StrsItem
     assert.equal(q.longPress.toString(), 'á é í');
     assert.equal(q.longPressDefault.value, 'é');
@@ -85,7 +85,7 @@ describe('key2', function () {
 
     const [Qgap] = key2.keys.filter(({ id }) => id.value === 'Q');
     assert.ok(Qgap);
-    assert.isTrue(!!(Qgap.flags & constants.key2_key_flags_gap));
+    assert.isTrue(!!(Qgap.flags & constants.key2_key_flags_gap), 'Q’s gap=');
 
     const [Wshift] = key2.keys.filter(({ id }) => id.value === 'W');
     assert.isNotNull(Wshift);

developer/src/kmc-keyboard/test/test-keys.ts

@@ -67,7 +67,6 @@ describe('keys', function () {
     let keys = loadSectionFixture(KeysCompiler, 'sections/keys/invalid-key-missing-attrs.xml', compilerTestCallbacks) as Keys;
     assert.isNull(keys);
     assert.equal(compilerTestCallbacks.messages.length, 1);
-
     assert.deepEqual(compilerTestCallbacks.messages[0], CompilerMessages.Error_KeyMissingToGapOrSwitch({keyId: 'Q'}));
   });
   it('should accept layouts with gap/switch keys', function() {

package-lock.json

@@ -24,7 +24,8 @@
       "dependencies": {
         "@keymanapp/common-types": "file:common/web/types",
         "@keymanapp/hextobin": "file:common/tools/hextobin",
-        "@keymanapp/keyman-version": "file:common/web/keyman-version"
+        "@keymanapp/keyman-version": "file:common/web/keyman-version",
+        "@keymanapp/ldml-keyboard-constants": "file:core/include/ldml"
       },
       "devDependencies": {
         "chai": "^4.3.4",

package.json

@@ -28,6 +28,7 @@
   "dependencies": {
     "@keymanapp/hextobin": "file:common/tools/hextobin",
     "@keymanapp/keyman-version": "file:common/web/keyman-version",
-    "@keymanapp/common-types": "file:common/web/types"
+    "@keymanapp/common-types": "file:common/web/types",
+    "@keymanapp/ldml-keyboard-constants": "file:core/include/ldml"
   }
 }

resources/standards-data/ldml-keyboards/techpreview/3.0/mt.xml

@@ -14,7 +14,7 @@
 
     <keys>
         <!-- imports -->
-        <import base="cldr" path="techpreview/key-Zyyy-punctuation.xml"/>
+        <import base="cldr" path="techpreview/keys-Zyyy-punctuation.xml"/>
 
         <!-- accent grave -->
         <key id="a-grave" to="à" />
@@ -33,22 +33,19 @@
         <key id="C-tikka" to="Ċ" />
         <key id="g-tikka" to="ġ" />
         <key id="G-tikka" to="Ġ" />
-        <key id="h-maqtua" to="ħ" />
-        <key id="H-maqtua" to="Ħ" />
+        <key id="h-maqtugha" to="ħ" /> <!-- maqtugħa, i.e. cut -->
+        <key id="H-maqtugha" to="Ħ" /> <!-- maqtugħa, i.e. cut -->
         <key id="z-tikka" to="ż" />
         <key id="Z-tikka" to="Ż" />
 
         <!-- Cedilla -->
         <key id="c-cedilla" to="ç" />
-
-        <!-- gaps -->
-        <key id="gap" gap="true" width="1" />
     </keys>
 
     <layers form="hardware" hardware="iso">
         <layer modifier="none">
-            <row keys="c-tikka 1 2 3 4 5 6 7 8 9 0 minus equals" />
-            <row keys="q w e r t y u i o p g-tikka h-maqtua" />
+            <row keys="c-tikka 1 2 3 4 5 6 7 8 9 0 hyphen equal" />
+            <row keys="q w e r t y u i o p g-tikka h-maqtugha" />
             <row keys="a s d f g h j k l semi-colon hash" />
             <row keys="z-tikka z x c v b n m comma period slash" />
             <row keys="space" />
@@ -56,7 +53,7 @@
 
         <layer modifier="shift">
             <row keys="C-tikka bang double-quote euro dollar percent caret amp open-paren close-paren underscore plus" />
-            <row keys="Q W E R T Y U I O P G-tikka H-maqtua" />
+            <row keys="Q W E R T Y U I O P G-tikka H-maqtugha" />
             <row keys="A S D F G H J K L colon at tilde" />
             <row keys="Z-tikka Z X C V B N M open-angle close-angle question" />
             <row keys="space" />

resources/standards-data/ldml-keyboards/techpreview/cldr_info.json

@@ -1,5 +1,5 @@
 {
-  "sha": "e906a2e9a628e7339e4fbc68f8007747d5f5374d",
-  "description": "release-42-301-ge906a2e9a6",
-  "date": "Sat, 29 Oct 2022 21:03:08 +0000"
+  "sha": "d0bdd8868a502afb6c1c1a3e2bf7c3404620f23b",
+  "description": "release-42-m1-67-gd0bdd8868a",
+  "date": "Wed, 25 Jan 2023 21:03:15 +0000"
 }

resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboard.dtd

@@ -1,5 +1,5 @@
 <!--
-Copyright © 1991-2022 Unicode, Inc.
+Copyright © 1991-2023 Unicode, Inc.
 For terms of use, see http://www.unicode.org/copyright.html
 SPDX-License-Identifier: Unicode-DFS-2016
 CLDR data files are interpreted according to the LDML specification (http://unicode.org/reports/tr35/)
@@ -119,6 +119,8 @@ Please see CLDR-15034 for the latest information. -->
     <!--@VALUE-->
 <!ATTLIST key multiTap CDATA #IMPLIED >
     <!--@VALUE-->
+<!ATTLIST key stretch (true) #IMPLIED >
+    <!--@VALUE-->
 <!ATTLIST key switch NMTOKEN #IMPLIED >
     <!--@MATCH:regex/[A-Za-z0-9][A-Za-z0-9-]*-->
     <!--@VALUE-->

resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboard.xsd

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  Copyright © 1991-2022 Unicode, Inc.
+  Copyright © 1991-2023 Unicode, Inc.
   For terms of use, see http://www.unicode.org/copyright.html
   SPDX-License-Identifier: Unicode-DFS-2016
   CLDR data files are interpreted according to the LDML specification (http://unicode.org/reports/tr35/)
@@ -226,6 +226,13 @@
       <xs:attribute name="longPress"/>
       <xs:attribute name="longPressDefault"/>
       <xs:attribute name="multiTap"/>
+      <xs:attribute name="stretch">
+        <xs:simpleType>
+          <xs:restriction base="xs:token">
+            <xs:enumeration value="true"/>
+          </xs:restriction>
+        </xs:simpleType>
+      </xs:attribute>
       <xs:attribute name="switch" type="xs:NMTOKEN"/>
       <xs:attribute name="transform">
         <xs:simpleType>
@@ -247,6 +254,7 @@
   <!-- @MATCH:any -->
   <!-- @VALUE -->
   <!-- @VALUE -->
+  <!-- @VALUE -->
   <!-- @MATCH:regex/[A-Za-z0-9][A-Za-z0-9-]* -->
   <!-- @VALUE -->
   <!-- @VALUE -->

resources/standards-data/ldml-keyboards/techpreview/import/keys-Latn-implied.xml

@@ -0,0 +1,78 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Copyright © 1991-2023 Unicode, Inc.
+For terms of use, see http://www.unicode.org/copyright.html
+SPDX-License-Identifier: Unicode-DFS-2016
+CLDR data files are interpreted according to the LDML specification (http://unicode.org/reports/tr35/)
+-->
+<!-- GENERATED by KeyboardTool. This is the implied import for implied keys. -->
+<!DOCTYPE keys SYSTEM "../dtd/ldmlKeyboard.dtd">
+<keys>
+
+    <key id="gap" gap="true" width="1"/>
+    <key id="space" to=" " stretch="true" width="1"/>
+
+    <key id="0" to="0"/>
+    <key id="1" to="1"/>
+    <key id="2" to="2"/>
+    <key id="3" to="3"/>
+    <key id="4" to="4"/>
+    <key id="5" to="5"/>
+    <key id="6" to="6"/>
+    <key id="7" to="7"/>
+    <key id="8" to="8"/>
+    <key id="9" to="9"/>
+    <key id="A" to="A"/>
+    <key id="B" to="B"/>
+    <key id="C" to="C"/>
+    <key id="D" to="D"/>
+    <key id="E" to="E"/>
+    <key id="F" to="F"/>
+    <key id="G" to="G"/>
+    <key id="H" to="H"/>
+    <key id="I" to="I"/>
+    <key id="J" to="J"/>
+    <key id="K" to="K"/>
+    <key id="L" to="L"/>
+    <key id="M" to="M"/>
+    <key id="N" to="N"/>
+    <key id="O" to="O"/>
+    <key id="P" to="P"/>
+    <key id="Q" to="Q"/>
+    <key id="R" to="R"/>
+    <key id="S" to="S"/>
+    <key id="T" to="T"/>
+    <key id="U" to="U"/>
+    <key id="V" to="V"/>
+    <key id="W" to="W"/>
+    <key id="X" to="X"/>
+    <key id="Y" to="Y"/>
+    <key id="Z" to="Z"/>
+    <key id="a" to="a"/>
+    <key id="b" to="b"/>
+    <key id="c" to="c"/>
+    <key id="d" to="d"/>
+    <key id="e" to="e"/>
+    <key id="f" to="f"/>
+    <key id="g" to="g"/>
+    <key id="h" to="h"/>
+    <key id="i" to="i"/>
+    <key id="j" to="j"/>
+    <key id="k" to="k"/>
+    <key id="l" to="l"/>
+    <key id="m" to="m"/>
+    <key id="n" to="n"/>
+    <key id="o" to="o"/>
+    <key id="p" to="p"/>
+    <key id="q" to="q"/>
+    <key id="r" to="r"/>
+    <key id="s" to="s"/>
+    <key id="t" to="t"/>
+    <key id="u" to="u"/>
+    <key id="v" to="v"/>
+    <key id="w" to="w"/>
+    <key id="x" to="x"/>
+    <key id="y" to="y"/>
+    <key id="z" to="z"/>
+</keys>
+

resources/standards-data/ldml-keyboards/techpreview/import/keys-Zyyy-punctuation.xml

@@ -2,36 +2,41 @@
 <!DOCTYPE keys SYSTEM "../dtd/ldmlKeyboard.dtd">
 <keys>
     <!-- General Symbols -->
+    <key id="amp" to="\u{0026}" />
     <key id="apos" to="'" />
-    <key id="bang" to="!" />
-    <key id="close-paren" to=")" />
-    <key id="double-quote" to="\u{0022}" />
-    <key id="equal" to="=" />
-    <key id="hash" to="#" />
-    <key id="hyphen" to="-" />
-    <key id="open-paren" to="(" />
-    <key id="umlaut" to="¨" />
-    <key id="underscore" to="_" />
     <key id="at" to="@" />
     <key id="backslash" to="\u{005C}" />
+    <key id="bang" to="!" />
     <key id="caret" to="^" />
     <key id="close-angle" to=">" />
     <key id="close-curly" to="}" />
+    <key id="close-paren" to=")" />
     <key id="close-square" to="]" />
     <key id="colon" to=":" />
+    <key id="comma" to="," />
+    <key id="double-quote" to="\u{0022}" />
+    <key id="equal" to="=" />
     <key id="grave" to="`" />
+    <key id="hash" to="#" />
+    <key id="hyphen" to="-" />
     <key id="not" to="¬" />
     <key id="open-angle" to="\u{003C}" />
     <key id="open-curly" to="{" />
+    <key id="open-paren" to="(" />
     <key id="open-square" to="[" />
+    <key id="percent" to="%" />
+    <key id="period" to="." />
     <key id="pipe" to="|" />
     <key id="plus" to="+" />
     <key id="question" to="?" />
     <key id="semi-colon" to=";" />
+    <key id="slash" to="/" />
     <key id="tilde" to="~" />
-    <key id="amp" to="\u{0026}" />
+    <key id="umlaut" to="¨" />
+    <key id="underscore" to="_" />
 
     <!-- Currency -->
     <key id="euro" to="€" />
     <key id="pound" to="£" />
+    <key id="dollar" to="$" />
 </keys>

resources/standards-data/ldml-keyboards/techpreview/ldml-keyboard.schema.json

@@ -159,6 +159,12 @@
         "multiTap": {
           "type": "string"
         },
+        "stretch": {
+          "enum": [
+            "true"
+          ],
+          "type": "string"
+        },
         "switch": {
           "type": "string"
         },