Merge pull request #13382 from keymanapp/chore/merge-master-into-autocorrect

chore: merge master into autocorrect 🚂

.github/workflows/api-verification.yml

@@ -26,7 +26,7 @@ jobs:
 
     steps:
     - name: Restore artifacts
-      uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+      uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
       with:
         path: |
           artifacts

.github/workflows/close-linked-issues-for-merged-prs.yml

@@ -0,0 +1,20 @@
+name: Close linked issues for merged pull requests
+
+on:
+  pull_request:
+    types: [closed]
+    branches:
+      - beta
+      - 'stable-*'
+      - 'epic/*'
+
+jobs:
+  closeIssueOnPrMergeTrigger:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Closes issues related to a merged pull request.
+        uses: ldez/gha-mjolnir@5574ed1f1151e4d2f11e3513cd85920a3a46bb7b # v1.5.0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/deb-packaging.yml

@@ -221,7 +221,7 @@ jobs:
         export DEBIAN_PRIORITY=critical
         export DEBCONF_NOWARNINGS=yes
         sudo apt-get update
-        sudo apt-get install -q -y dput
+        sudo apt-get install -q -y dput rsync
 
     - name: Setup .dput.cf
       run: |
@@ -315,7 +315,7 @@ jobs:
         echo "GIT_USER=${{ github.event.client_payload.user }}" >> artifacts/env
 
     - name: Cache artifacts
-      uses: actions/cache/save@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+      uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
       with:
         path: |
           artifacts

HISTORY.md

@@ -1,5 +1,38 @@
 # Keyman Version History
 
+## 19.0.9 alpha 2025-02-27
+
+* docs: move node.js before Emscripten (#13366)
+* chore: add a workflow to automatically close linked issues (#13368)
+
+## 19.0.8 alpha 2025-02-21
+
+* feat(windows): hack in some fun features for kmdevlink (#13237)
+* feat(developer): use ABNF to validate LDML transform (#13236)
+* fix(linux): remove `--platform amd64` from docker build script (#13318)
+
+## 19.0.7 alpha 2025-02-20
+
+* chore(linux): allow to build and use Debian docker image (#13284)
+
+## 19.0.6 alpha 2025-02-19
+
+* chore: merge beta to master B18S1 (#13239)
+* fix(linux): add missing dependency for uploading to llso (#13280)
+
+## 19.0.5 alpha 2025-02-15
+
+* fix(linux): update location of lcov.deb for Jammy (#13253)
+
+## 19.0.4 alpha 2025-02-14
+
+* feat(developer): serialize KMXPlus into XML (#13174)
+
+## 19.0.3 alpha 2025-02-13
+
+* docs: update keyboard processor build source (#13221)
+* chore(linux): additional code cleanup after Focal removal (#13206)
+
 ## 19.0.2 alpha 2025-02-12
 
 * chore(linux): remove support of Ubuntu 20.04 Focal (#13203)
@@ -9,6 +42,30 @@
 * refactor(windows): rename `TKeymanMutex.MutexOwned` to `TakeOwnership` and add `ReleaseOwnership` (#13168)
 * chore: increment to alpha 19.0 (#13187)
 
+## 18.0.191 beta 2025-02-12
+
+* feat(windows): handle a hard windows reset occurring while downloading updated keyman files (#13128)
+* docs(developer): update what's new for 18.0 (#13198)
+* chore(mac): update whats new for Keyman 18 (#13197)
+* chore(ios): update whats new for Keyman 18 (#13199)
+* fix: use tier and version from branch when merging history from another branch (#13170)
+* fix(linux): start system service when switching keyboards (#13172)
+* chore(linux): update actions/cache to non-deprecated version (#13193)
+* chore(linux): update branch that's used for Debian packaging (#13192)
+* chore(linux): remove support of Ubuntu 20.04 Focal (#13202)
+
+## 18.0.190 beta 2025-02-11
+
+* fix(developer): bundle ttfmeta library internally with kmc-keyboard-info (#11631)
+* fix(developer): support Windows and Unicode names in .ttf (#11633)
+* chore(developer): increase timeout for kmc-ldml compiler test (#11635)
+* chore(common): deps: update eslint typescript plugins (#11842)
+* refactor(windows): rename `TKeymanMutex.MutexOwned` to `TakeOwnership` and add `ReleaseOwnership` (#13168)
+
+## 18.0.189 beta 2025-02-11
+
+* chore: move to beta
+
 ## 18.0.188 alpha 2025-02-10
 
 * fix(windows): check the params status flag equals ucrsUpdateReady before attempting to download the keyman setup file (#13154)
@@ -1321,7 +1378,7 @@
 * chore(common): move to 18.0 alpha (#10713)
 * chore: move to 18.0 alpha
 
-## 17.0.335 alpha 2025-02-06
+## 17.0.335 stable 2025-02-06
 
 * fix(android): improve resource-update tool handling of host Activity's closure (#13057)
 * fix(ios): prevent message-handler collision (#13058)

VERSION.md

@@ -1 +1 @@
-19.0.3
+19.0.10

common/web/types/src/kmx/kmx-plus/kmx-plus.ts

@@ -385,15 +385,11 @@ export class UnicodeSetItem extends VarsItem {
 };
 
 export class SetVarItem extends VarsItem {
-  constructor(id: string, value: string[], sections: DependencySections, rawItems: string[]) {
+  constructor(id: string, value: string[], sections: DependencySections) {
     super(id, value.join(' '), sections);
     this.items = sections.elem.allocElementString(sections, value);
-    this.rawItems = rawItems;
   }
-  // element string array
-  items: ElementString;
-  // like items, but with unprocessed marker strings
-  rawItems: string[];
+  items: ElementString;  // element string array
   valid() : boolean {
     return !!this.items;
   }
@@ -409,10 +405,12 @@ export class StringVarItem extends VarsItem {
 // 'tran'
 
 export class TranTransform {
-  from: StrsItem;
-  to: StrsItem;
-  mapFrom: StrsItem; // var name
-  mapTo: StrsItem; // var name
+  from: StrsItem; // "from" computed regex string
+  to: StrsItem; // "to" (replacement) computed regex string
+  mapFrom: StrsItem; // var name for map
+  mapTo: StrsItem; // var name for map
+  _from?: string; // Not part of binary file: for use in the XML serializer. If present, sets the from= attribute for XML.
+  _to?: string; // Not part of binary file: for use in the XML serializer. If present, sets the to= attribute for XML.
 }
 
 export class TranGroup {
@@ -424,6 +422,9 @@ export class TranGroup {
 export class TranReorder {
   elements: ElementString;
   before: ElementString;
+  _before?: string; // Not part of binary file: for use in the XML serializer. If present, sets the before= attribute for XML.
+  _from?: string; // Not part of binary file: for use in the XML serializer. If present, sets the from= attribute for XML.
+  _order?: string; // Not part of binary file: for use in the XML serializer. If present, sets the order= attribute for XML.
 };
 
 export class Tran extends Section {

developer/src/common/web/utils/src/xml-utils.ts

@@ -114,6 +114,13 @@ const GENERATOR_OPTIONS: KeymanXMLOptionsBag = {
     textNodeName: '_',
     suppressEmptyNode: true,
   },
+  keyboard3: {
+    attributeNamePrefix: '$',
+    ignoreAttributes: false,
+    format: true,
+    textNodeName: '_',
+    suppressEmptyNode: true,
+  },
 };
 
 /** wrapper for XML parsing support */

developer/src/kmc-ldml/.gitignore

@@ -0,0 +1,3 @@
+src/util/abnf/**/*.pegjs
+src/util/abnf/**/*.d.ts
+src/util/abnf/**/*.js

developer/src/kmc-ldml/build.sh

@@ -27,24 +27,45 @@ builder_describe "Keyman kmc Keyboard Compiler module" \
   "--dry-run,-n              don't actually publish, just dry run"
 
 builder_describe_outputs \
-  configure     /node_modules \
+  configure     /developer/src/kmc-ldml/src/util/abnf/46/transform-from-required.js \
   build         /developer/src/kmc-ldml/build/src/main.js \
   api           /developer/build/api/kmc-ldml.api.json
 
 builder_parse "$@"
 
 function do_clean() {
-  rm -rf ./build/ ./tsconfig.tsbuildinfo
+  rm -rf ./build/ ./tsconfig.tsbuildinfo ./src/util/abnf/*/*.pegjs ./src/util/abnf/*/*.ts ./src/util/abnf/*/*.js
 }
 
 function do_configure() {
   verify_npm_setup
+  do_build_abnf
 }
 
 function do_build() {
   npm run build
 }
 
+function do_build_abnf() {
+  # we convert over all abnf files found. 
+  local file
+  for file in "$KEYMAN_ROOT/resources/standards-data/ldml-keyboards"/*/abnf/*.abnf; do
+    local cldrver="$(basename $(dirname $(dirname "$file")))"
+    local base="$(basename "$file" .abnf)"
+    local peg="$base.pegjs"
+    local outdir="./src/util/abnf/$cldrver"
+    local outfile="$outdir/$peg"
+    local outjs="$outdir/$base.js"
+    if [ ! -f "$outjs" ]; then
+      mkdir -p "$outdir"
+      printf "${COLOR_GREY}abnf_gen ${COLOR_PURPLE}${cldrver}/${base}.abnf -> ${peg}${COLOR_RESET}\n"
+      "$KEYMAN_ROOT/node_modules/.bin/abnf_gen"  "$file" -o "$outfile"
+      printf "${COLOR_GREY}peggy ${COLOR_PURPLE}${cldrver}/${peg} -> ${base}.js${COLOR_RESET}\n"
+      "$KEYMAN_ROOT/node_modules/.bin/peggy" "$outfile" -o "$outjs" --format es --dts
+    fi
+  done
+}
+
 function do_build_fixtures() {
   # Build basic.kmx and emit its checksum
   mkdir -p ./build/test/fixtures

developer/src/kmc-ldml/package.json

@@ -38,10 +38,12 @@
     "@types/mocha": "^5.2.7",
     "@types/node": "^20.4.1",
     "@types/semver": "^7.3.12",
+    "abnf": "^4.3.1",
     "c8": "^7.12.0",
     "chalk": "^2.4.2",
     "common-tags": "^1.8.2",
     "mocha": "^8.4.0",
+    "peggy": "^4.2.0",
     "typescript": "^5.4.5"
   },
   "mocha": {

developer/src/kmc-ldml/src/compiler/keys.ts

@@ -15,8 +15,9 @@ import { SubstitutionUse, Substitutions } from './substitution-tracker.js';
 /** reserved name for the special gap key. space is not allowed in key ids. */
 const reserved_gap = "gap (reserved)";
 
-
 export class KeysCompiler extends SectionCompiler {
+  /** keys that are of a reserved type */
+  public static RESERVED_KEY = Symbol('Reserved Key');
   static validateSubstitutions(
     keyboard: LDMLKeyboard.LKKeyboard,
     st: Substitutions
@@ -222,6 +223,19 @@ export class KeysCompiler extends SectionCompiler {
   /** count of reserved keys, for tests */
   public static readonly reserved_count = KeysCompiler.reserved_keys.length;
 
+  /** mark as reserved */
+  private static asReserved(k : KeysKeys) : KeysKeys {
+    const o = k as any;
+    o[KeysCompiler.RESERVED_KEY] = true;
+    return k;
+  }
+
+  /** true if a reserved key */
+  public static isReserved(k : KeysKeys) : boolean {
+    const o = k as any;
+    return !!o[KeysCompiler.RESERVED_KEY];
+  }
+
   /** load up all reserved keys */
   getReservedKeys(sections: KMXPlus.DependencySections) : Map<String, KeysKeys> {
     const r = new Map<String, KeysKeys>();
@@ -231,7 +245,7 @@ export class KeysCompiler extends SectionCompiler {
     const no_list = sections.list.allocList([], {}, sections);
 
     // now add the reserved key(s).
-    r.set(reserved_gap, {
+    r.set(reserved_gap, KeysCompiler.asReserved({
       flags: constants.keys_key_flags_gap | constants.keys_key_flags_extend,
       id: sections.strs.allocString(reserved_gap),
       flicks: '',
@@ -241,7 +255,7 @@ export class KeysCompiler extends SectionCompiler {
       switch: no_string,
       to: no_string,
       width: 10.0, // 10 * .1
-    });
+    }));
 
     if (r.size !== KeysCompiler.reserved_count) {
       throw Error(`Internal Error: KeysCompiler.reserved_count=${KeysCompiler.reserved_count} != ${r.size} actual reserved keys.`);
@@ -357,7 +371,7 @@ export class KeysCompiler extends SectionCompiler {
         flags |= constants.keys_key_flags_extend;
       }
       const width = Math.ceil((key.width || 1) * 10.0); // default, width=1
-      sect.keys.push({
+      sect.keys.push(SectionCompiler.copySymbols({
         flags,
         flicks: flickId,
         id,
@@ -367,7 +381,7 @@ export class KeysCompiler extends SectionCompiler {
         switch: keySwitch, // 'switch' is a reserved word
         to,
         width,
-      });
+      }, key));
     }
   }
 

developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts

@@ -258,4 +258,11 @@ export class LdmlCompilerMessages {
     \`<transform from="($[fromSet])" to="$[1:toSet]"/>\`.
   `);
 
+  static ERROR_UnparseableTransformTo = SevErrorTransform | 0x06;
+  static Error_UnparseableTransformTo = (o: {to: string, message: string}) => m(
+    this.ERROR_UnparseableTransformTo,
+    `Invalid transform to="${def(o.to)}": "${def(o.message)}"`,
+  );
+
+
 }

developer/src/kmc-ldml/src/compiler/section-compiler.ts

@@ -55,4 +55,20 @@ export abstract class SectionCompiler {
     ]);
     return defaults;
   }
+
+  /**
+   * Copy symbols from 'from' onto 'onto'
+   * This is used to propagate special symbols such as ImportStatus
+   * and XML
+   * @param onto object to copy onto
+   * @param from source for symbols
+   * @returns the onto object
+   */
+  public static copySymbols<T>(onto: T, from: any) : T {
+    const o = onto as any;
+    for (const sym of Object.getOwnPropertySymbols(from)) {
+      o[sym] = from[sym];
+    }
+    return onto;
+  }
 }

developer/src/kmc-ldml/src/compiler/tran.ts

@@ -17,6 +17,7 @@ import LKTransforms = LDMLKeyboard.LKTransforms;
 import { verifyValidAndUnique } from "../util/util.js";
 import { LdmlCompilerMessages } from "./ldml-compiler-messages.js";
 import { Substitutions, SubstitutionUse } from "./substitution-tracker.js";
+import { transform_from_parse, transform_to_parse } from "../util/abnf/abnf.js";
 
 type TransformCompilerType = 'simple' | 'backspace';
 
@@ -138,11 +139,20 @@ export abstract class TransformCompiler<T extends TransformCompilerType, TranBas
 
   private compileTransform(sections: DependencySections, transform: LKTransform) : TranTransform {
     let result = new TranTransform();
+    // setup for serializing
+    result._from = transform.from;
+    result._to = transform.to;
     let cookedFrom = transform.from;
 
     // check for incorrect \uXXXX escapes. Do this before substituting markers or sets.
+    // We run this first because it's more helpful than the ABNF.
     cookedFrom = this.checkEscapes(cookedFrom); // check for \uXXXX escapes before normalizing
 
+    if (cookedFrom === '') {
+      this.callbacks.reportMessage(LdmlCompilerMessages.Error_TransformFromMatchesNothing({ from: cookedFrom }));
+      return null;
+    }
+
     cookedFrom = sections.vars.substituteStrings(cookedFrom, sections, true);
     const mapFrom = LdmlKeyboardTypes.VariableParser.CAPTURE_SET_REFERENCE.exec(cookedFrom);
     const mapTo = LdmlKeyboardTypes.VariableParser.MAPPED_SET_REFERENCE.exec(transform.to || '');
@@ -183,6 +193,24 @@ export abstract class TransformCompiler<T extends TransformCompilerType, TranBas
       return null;
     }
 
+    // run the parser here next, on the original from
+    try {
+      if (cookedFrom != null) {
+        transform_from_parse(transform.from);
+      }
+    } catch (e) {
+      this.callbacks.reportMessage(LdmlCompilerMessages.Error_UnparseableTransformFrom({ from: cookedFrom, message: e.toString() }));
+      return null;
+    }
+    
+    // run the parser here next, on the original to
+    try {
+      transform_to_parse(transform.to || '');
+    } catch (e) {
+      this.callbacks.reportMessage(LdmlCompilerMessages.Error_UnparseableTransformTo({ to: transform.to || '', message: e.toString() }));
+      return null;
+    }
+  
     // cookedFrom is cooked above, since there's some special treatment
     result.from = sections.strs.allocString(cookedFrom, {
       unescape: false,
@@ -262,6 +290,10 @@ export abstract class TransformCompiler<T extends TransformCompilerType, TranBas
 
   private compileReorder(sections: DependencySections, reorder: LKReorder): TranReorder {
     let result = new TranReorder();
+    // for serializing
+    result._before = reorder.before;
+    result._from = reorder.from;
+    result._order = reorder.order;
     if (reorder.from && this.checkEscapes(reorder.from) === null) {
       return null; // error'ed
     }

developer/src/kmc-ldml/src/compiler/vars.ts

@@ -285,7 +285,7 @@ export class VarsCompiler extends SectionCompiler {
     // this is not 'forMatch', all variables are to be assumed as string literals, not regex
     // content.
     const cookedItems: string[] = rawItems.map(v => result.substituteMarkerString(v, false));
-    result.sets.push(new SetVarItem(id, cookedItems, sections, rawItems));
+    result.sets.push(new SetVarItem(id, cookedItems, sections));
   }
   addUnicodeSet(result: Vars, e: LDMLKeyboard.LKUSet, sections: DependencySections): void {
     const { id } = e;

developer/src/kmc-ldml/src/util/abnf/abnf.ts

@@ -0,0 +1,6 @@
+/**
+ * Re-export the generated parsers. At present, we hard code CLDR version.
+ */
+
+export { parse as transform_to_parse } from './46/transform-to-required.js';
+export { parse as transform_from_parse } from './46/transform-from-required.js';

developer/src/kmc-ldml/src/util/serialize.ts

@@ -0,0 +1,274 @@
+/*
+ * Keyman is copyright (C) SIL Global. MIT License.
+ *
+ * This module contains routines for serializing from a KMXPlus file into XML.
+ */
+
+import { KMXPlus } from "@keymanapp/common-types";
+import { KeymanXMLWriter, LDMLKeyboard } from "@keymanapp/developer-utils";
+import { constants } from "@keymanapp/ldml-keyboard-constants";
+import { KeysCompiler } from "../compiler/keys.js";
+
+/**
+ * Serialize a KMXPlusFile back to XML.
+ * This is implemented for the LDML editor to be able to mutate LDML (XML) content by:
+ *  1. reading the original XML
+ *  2. compiling to KMXPlusFile
+ *  3. modifying the KMXPlusFile
+ *  4. serializing back to XML
+ *
+ * There are limitations:
+ *  - TODO-LDML-EDITOR: Transforms would not be serialized properly, due to
+ *    regex munging around markers and such.
+ *
+ *    To work around this, fields with underscores such as _from and _to are added to
+ *    the KMXPlusFile classes. These provide hints as to what the output XML should be,
+ *    and are populated by the tran compiler.
+ *
+ *  - TODO-LDML-EDITOR: Comments, whitespace, etc. are not preserved by this
+ *    approach. Updates to the XML parsing will support this, see #10622.
+ *
+ * @param kmx input KMXPlusFile
+ * @returns XML String
+ */
+export function kmxToXml(kmx: KMXPlus.KMXPlusFile): string {
+  const writer = new KeymanXMLWriter("keyboard3");
+  const { kmxplus } = kmx;
+  const {
+    //    sect,
+    bksp,
+    disp,
+    //    elem,
+    keys,
+    layr,
+    //    list,
+    loca,
+    meta,
+    //    strs,
+    tran,
+    //    uset,
+    vars,
+  } = kmxplus;
+  const data = {
+    keyboard3: {
+      ...getRootAttributes(),
+      ...getLocales(),
+      version: getVersion(),
+      info: getInfo(),
+      ...getDisplays(),
+      ...getKeys(),
+      ...getFlicks(),
+      ...getLayers(),
+      ...getVariables(),
+      ...getTransforms(),
+    }
+  };
+
+  return writer.write(data);
+
+  function getRootAttributes() {
+    const conform = meta.conform.value;
+    return {
+      '$xmlns': `https://schemas.unicode.org/cldr/${conform}/keyboard3`,
+      '$locale': kmx.kmxplus.loca.locales[0].value,
+      '$conformsTo': conform,
+    };
+  }
+
+  function getLocales() {
+    if (loca?.locales?.length < 2) {
+      return {}; // no additional locales
+    } else {
+      return {
+        locales:
+          loca.locales.map(({ value }) => ({ '$id': value })),
+      }
+    }
+  }
+
+  function getInfo() {
+    return {
+      '$author': meta.author.value,
+      '$name': meta.name.value,
+      '$layout': meta.layout.value,
+      '$indicator': meta.indicator.value,
+    };
+  }
+
+  function getVersion() {
+    return { '$number': kmx.kmxplus.meta.version.value };
+  }
+
+  function getDisplays() {
+    const displays = {
+      display: disp?.disps.map(disp => getDisplay(disp)) || [],
+      ...getDisplaySettings(),
+    };
+    if (displays?.display?.length || displays?.displayOptions) {
+      return { displays }
+    } else {
+      return {};
+    }
+  }
+
+  function stringToAttr(attr: string, s?: KMXPlus.StrsItem, override?: string) {
+    if (override) return asAttr(attr, override);
+    if (!s || !s?.value?.length) return {};
+    return Object.fromEntries([[`\$${attr}`, s.value]]);
+  }
+
+  function asAttr(attr: string, s?: any) {
+    if (s === undefined) return {};
+    return Object.fromEntries([[`\$${attr}`, s]]);
+  }
+
+  function numberToAttr(attr: string, s?: number) {
+    if (s === undefined) return {};
+    return Object.fromEntries([[`\$${attr}`, s.toString()]]);
+  }
+
+  function getDisplay(disp: KMXPlus.DispItem) {
+    return {
+      ...stringToAttr('output', disp?.to),
+      ...stringToAttr('keyId', disp?.id),
+      ...stringToAttr('display', disp?.display),
+    };
+  }
+
+  function getDisplaySettings() {
+    if (!disp?.baseCharacter?.value) return {};
+    return {
+      displayOptions: {
+        '$baseCharacter': disp?.baseCharacter?.value,
+      }
+    };
+  }
+
+  function getKeys() {
+    if (!keys?.keys?.length) {
+      return {};
+    }
+    return {
+      keys: {
+        key: keys.keys
+          // skip reserved keys (gap)
+          .filter((key: KMXPlus.KeysKeys) =>
+            !KeysCompiler.isReserved(key) &&
+            !LDMLKeyboard.ImportStatus.isImpliedImport(key))
+          .map((key: KMXPlus.KeysKeys) => ({
+            ...stringToAttr('id', key.id),
+            ...stringToAttr('output', key.to),
+            ...asAttr('longPressKeyIds', key?.longPress?.join(' ') || undefined),
+          })),
+      },
+    };
+  }
+
+  function getFlicks() {
+    // skip the null flicks
+    if (keys?.flicks?.length < 2) {
+      return {};
+    }
+    return {
+      flicks: {
+        // keys.key..
+      }
+    };
+  }
+
+  function getLayers() {
+    if (!layr?.lists?.length) {
+      return {};
+    }
+    return {
+      layers: layr.lists.map(({ hardware, minDeviceWidth, layers }) => ({
+        ...stringToAttr('formId', hardware),
+        ...numberToAttr('minDeviceWidth', minDeviceWidth),
+        layer: layers.map(({ id, mod, rows }) => ({
+          ...stringToAttr('id', id),
+          ...asAttr('modifiers', modToString(mod)),
+          row: rows.map(({ keys }) => ({
+            ...asAttr('keys', keys.map(({ value }) => value).join(' ')),
+          })),
+        })),
+      })),
+    };
+  }
+
+  function getVariables() {
+    if (!vars?.strings.length && !vars?.sets.length && !vars?.usets.length) {
+      return {};
+    }
+    function varToObj(v: KMXPlus.VarsItem): any {
+      const { id, value } = v;
+      return {
+        ...stringToAttr('id', id),
+        ...stringToAttr('value', value),
+      };
+    }
+    function varsToArray(vars: KMXPlus.VarsItem[]): any[] {
+      return vars.map(varToObj);
+    }
+    const { strings, sets, usets } = vars;
+
+    return {
+      variables: {
+        string: varsToArray(strings),
+        set: varsToArray(sets),
+        uset: varsToArray(usets),
+      },
+    };
+  }
+
+  function getTransforms() {
+    return {
+      transforms: [
+      ...getTransformType("simple", tran),
+      ...getTransformType("backspace", bksp),
+      ],
+    };
+  }
+
+  /** NB: Bksp is a child class of Tran */
+  function getTransformType(type: string, t: KMXPlus.Tran) {
+    if (!t?.groups?.length) {
+      return [];
+    }
+    const { groups } = t;
+    return [{
+        ...asAttr('type', type),
+        transformGroup: groups.map((group) => {
+          if (group.type === constants.tran_group_type_transform) {
+            return {
+              transform: group.transforms.map(({from, to, _from, _to}) => ({
+                ...stringToAttr('from', from, _from),
+                ...stringToAttr('to', to, _to),
+              })),
+            };
+          } else if(group.type === constants.tran_group_type_reorder) {
+            return {
+              reorder: group.reorders.map(({before, elements, _before, _from, _order}) => ({
+                ...asAttr('before', _before || before.toString()),
+                ...asAttr('from', _from || elements.toString()),
+                ...asAttr('order', _order),
+              })),
+            };
+          } else {
+            throw Error(`Invalid tran.group.type ${group.type}`);
+          }
+        }),
+    }];
+  }
+}
+
+
+/** convert a keys_mod value to a space-separated string list */
+function modToString(mod: number) {
+  // first try exact match
+  const matches: string[] = [];
+  for (const [name, value] of constants.keys_mod_map.entries()) {
+    if (mod === value) return name; // exact match
+    if (mod & value) matches.push(name);
+  }
+  return matches.sort().join(' ');
+}

developer/src/kmc-ldml/test/compiler-e2e.tests.ts

@@ -3,7 +3,10 @@ import {assert} from 'chai';
 import hextobin from '@keymanapp/hextobin';
 import { KMXBuilder } from '@keymanapp/developer-utils';
 import {checkMessages, compileKeyboard, compilerTestCallbacks, compilerTestOptions, makePathToFixture} from './helpers/index.js';
+import { compareXml } from './helpers/compareXml.js';
 import { LdmlKeyboardCompiler } from '../src/compiler/compiler.js';
+import { kmxToXml } from '../src/util/serialize.js';
+import { writeFileSync } from 'node:fs';
 
 /** Overall compiler tests */
 describe('compiler-tests', function() {
@@ -35,8 +38,34 @@ describe('compiler-tests', function() {
     let expected = await hextobin(binaryFilename, undefined, {silent:true});
 
     assert.deepEqual<Uint8Array>(code, expected);
+
+    // now output it again as XML
+    const outputFilename = makePathToFixture('basic-serialized.xml');
+    const asXml = kmxToXml(kmx);
+    writeFileSync(outputFilename, asXml, 'utf-8');
+
   });
 
+  it('should-serialize-kmx', async function() {
+    this.timeout(4000);
+    // Let's build basic.xml
+    // It should match basic.kmx (built from basic.txt)
+
+    const inputFilename = makePathToFixture('basic.xml');
+
+    // Compile the keyboard
+    const kmx = await compileKeyboard(inputFilename, {...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false});
+    assert.isNotNull(kmx);
+
+    // now output it as XML
+    const outputFilename = makePathToFixture('basic-serialized.xml');
+    const asXml = kmxToXml(kmx);
+    writeFileSync(outputFilename, asXml, 'utf-8');
+
+    compareXml(outputFilename, inputFilename);
+  });
+
+
   it('should handle non existent files', async () => {
     const filename = 'DOES_NOT_EXIST.xml';
     const k = new LdmlKeyboardCompiler();

developer/src/kmc-ldml/test/fixtures/basic-serialized.xml

@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="utf-8"?>
+<keyboard3 xmlns="https://schemas.unicode.org/cldr/45/keyboard3" locale="mt" conformsTo="45">
+  <version number="1.0.0"/>
+  <info author="srl295" name="TestKbd" layout="qwerty" indicator="🙀"/>
+  <displays>
+    <display output="a" display="^"/>
+    <display keyId="e" display="^e"/>
+    <displayOptions baseCharacter="e"/>
+  </displays>
+  <keys>
+    <key id="hmaqtugha" output="ħ" longPressKeyIds="a e"/>
+    <key id="that" output="ថា"/>
+  </keys>
+  <layers formId="us" minDeviceWidth="123">
+    <layer modifiers="none">
+      <row keys="hmaqtugha that"/>
+    </layer>
+  </layers>
+  <variables>
+    <string id="a" value="\m{a}"/>
+    <string id="vst" value="abc"/>
+    <set id="vse" value="a b c"/>
+    <uset id="vus" value="[abc]"/>
+  </variables>
+  <transforms type="simple">
+    <transformGroup>
+      <transform from="^a" to="â"/>
+      <transform from="a" to="\m{a}"/>
+    </transformGroup>
+    <transformGroup>
+      <transform from="\m{a}"/>
+    </transformGroup>
+    <transformGroup>
+      <reorder before="\u{1A6B}" from="\u{1A60}[\u{1A75}-\u{1A79}]\u{1A45}" order="10 55 10"/>
+    </transformGroup>
+  </transforms>
+  <transforms type="backspace">
+    <transformGroup>
+      <transform from="^e"/>
+    </transformGroup>
+  </transforms>
+</keyboard3>

developer/src/kmc-ldml/test/fixtures/sections/tran/fail-bad-tran-1.xml

@@ -7,6 +7,7 @@
 
   <transforms type="simple">
     <transformGroup>
+      <!-- fails: capture group is not terminated -->
       <transform from="AB(now if only I would terminate this group.." to="x"/>
     </transformGroup>
   </transforms>

developer/src/kmc-ldml/test/fixtures/sections/tran/fail-bad-tran-3.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<keyboard3 xmlns="https://schemas.unicode.org/cldr/45/keyboard3" locale="mt" conformsTo="45">
+    <info name="tran-minimal" />
+
+    <keys/>
+
+    <transforms type="simple">
+        <transformGroup>
+            <transform from="(abc){1,}" /> <!-- fails: unbounded quantifier -->
+        </transformGroup>
+    </transforms>
+</keyboard3>

developer/src/kmc-ldml/test/fixtures/sections/tran/fail-bad-tran-4.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<keyboard3 xmlns="https://schemas.unicode.org/cldr/45/keyboard3" locale="mt" conformsTo="45">
+    <info name="tran-minimal" />
+
+    <keys/>
+
+    <transforms type="simple">
+        <transformGroup>
+            <transform from="ABC" to="\u{123}\u" />   <!-- illegal escape -->
+        </transformGroup>
+    </transforms>
+</keyboard3>

developer/src/kmc-ldml/test/fixtures/sections/tran/fail-bad-tran-5.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<keyboard3 xmlns="https://schemas.unicode.org/cldr/45/keyboard3" locale="mt" conformsTo="45">
+    <info name="tran-minimal" />
+
+    <keys/>
+
+    <transforms type="simple">
+        <transformGroup>
+            <transform from="\z" /> <!-- illegal char class--> 
+        </transformGroup>
+    </transforms>
+</keyboard3>

developer/src/kmc-ldml/test/fixtures/sections/tran/fail-bad-tran-6.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<keyboard3 xmlns="https://schemas.unicode.org/cldr/45/keyboard3" locale="mt" conformsTo="45">
+    <info name="tran-minimal" />
+
+    <keys/>
+
+    <transforms type="simple">
+        <transformGroup>
+            <transform from="z{99,99}" /> <!-- illegal quantifier--> 
+        </transformGroup>
+    </transforms>
+</keyboard3>

developer/src/kmc-ldml/test/helpers/compareXml.ts

@@ -0,0 +1,25 @@
+import {assert} from 'chai';
+import {readFileSync} from 'node:fs';
+import { KeymanXMLReader } from "@keymanapp/developer-utils";
+
+/**
+ *
+ * @param actual path to actual XML
+ * @param expect path to expected XML
+ * @param mutator optional function that will be applied to the parsed object
+ */
+export function compareXml(actual : string, expect: string, mutator?: (input: any) => any) {
+    if (!mutator) {
+        // no-op
+        mutator = (x: any) => x;
+    }
+    const reader = new KeymanXMLReader('keyboard3');
+
+    const actualStr = readFileSync(actual, 'utf-8');
+    const expectStr = readFileSync(expect, 'utf-8');
+
+    const actualParsed = mutator(reader.parse(actualStr));
+    const expectParsed = mutator(reader.parse(expectStr));
+
+    assert.deepEqual(actualParsed, expectParsed);
+}

developer/src/kmc-ldml/test/helpers/index.ts

@@ -275,12 +275,6 @@ export function testCompilationCases(compiler: SectionCompilerNew, cases : Compi
         return;
       }
       let section = await loadSectionFixture(compiler, testcase.subpath, callbacks, testcase.dependencies || dependencies);
-      if (expectFailure) {
-        assert.isNull(section, 'expected compilation result failure (null)');
-      } else {
-        assert.isNotNull(section, `failed with ${compilerEventFormat(callbacks.messages)}`);
-      }
-
       const testcaseErrors = matchCompilerEventsOrBoolean(callbacks.messages, testcase.errors);
       const testcaseWarnings = matchCompilerEvents(callbacks.messages, testcase.warnings);
       // if we expected errors or warnings, show them
@@ -296,6 +290,12 @@ export function testCompilationCases(compiler: SectionCompilerNew, cases : Compi
         // no warnings, so expect zero messages
         assert.sameDeepMembers(callbacks.messages, [], 'expected zero messages but got ' + callbacks.messages);
       }
+      
+      if (expectFailure) {
+        assert.isNull(section, 'expected compilation result failure (null)');
+      } else {
+        assert.isNotNull(section, `failed with ${compilerEventFormat(callbacks.messages)}`);
+      }
 
       // run the user-supplied callback if any
       if (testcase.callback) {

developer/src/kmc-ldml/test/tran.tests.ts

@@ -287,8 +287,8 @@ describe('tran', function () {
       subpath: `sections/tran/fail-bad-tran-1.xml`,
       errors: [
         { code: LdmlCompilerMessages.ERROR_UnparseableTransformFrom,
-          matchMessage: /Invalid regular expression.*Unterminated group/,
-        }
+          matchMessage: /.*Unterminated group.*/,
+        },
       ],
     },
     {
@@ -298,6 +298,15 @@ describe('tran', function () {
         LdmlCompilerMessages.Error_InvalidQuadEscape({ cp: 295 }),
       ],
     },
+    {
+      subpath: `sections/tran/fail-bad-tran-3.xml`,
+      errors: [
+        {
+          code: LdmlCompilerMessages.ERROR_UnparseableTransformFrom,
+          matchMessage: /.*Syntax.*0-9.*/,
+        }
+      ],
+    },
     {
       subpath: `sections/tran/fail-missing-var-1.xml`,
       errors: [
@@ -334,9 +343,35 @@ describe('tran', function () {
         LdmlCompilerMessages.Error_MissingStringVariable({ id: "missingstr" }),
       ],
     },
-    // cases that share the same error code
-    ...[1, 2, 3].map(n => ({
-      subpath: `sections/tran/fail-IllegalTransformDollarsign-${n}.xml`,
+    // cases that are now caught by the abnf
+    ...[
+      'fail-bad-tran-5',
+    ].map(s => ({
+      subpath: `sections/tran/${s}.xml`,
+      errors: [
+        {
+          code: LdmlCompilerMessages.ERROR_UnparseableTransformFrom,
+          matchMessage: /.*/,
+        }
+      ],
+    })),
+    ...[
+      'fail-bad-tran-4',
+    ].map(s => ({
+      subpath: `sections/tran/${s}.xml`,
+      errors: [
+        {
+          code: LdmlCompilerMessages.ERROR_UnparseableTransformTo,
+          matchMessage: /.*/,
+        }
+      ],
+    })),
+    ...[
+      'fail-IllegalTransformDollarsign-1',
+      'fail-IllegalTransformDollarsign-2',
+      'fail-IllegalTransformDollarsign-3',
+    ].map(s => ({
+      subpath: `sections/tran/${s}.xml`,
       errors: [
         {
           code: LdmlCompilerMessages.ERROR_IllegalTransformDollarsign,
@@ -344,8 +379,11 @@ describe('tran', function () {
         }
       ],
     })),
-    ...[1, 2].map(n => ({
-      subpath: `sections/tran/fail-IllegalTransformAsterisk-${n}.xml`,
+    ...[
+      'fail-IllegalTransformAsterisk-1',
+      'fail-IllegalTransformAsterisk-2',
+    ].map(s => ({
+      subpath: `sections/tran/${s}.xml`,
       errors: [
         {
           code: LdmlCompilerMessages.ERROR_IllegalTransformAsterisk,
@@ -353,8 +391,11 @@ describe('tran', function () {
         }
       ],
     })),
-    ...[1, 2].map(n => ({
-      subpath: `sections/tran/fail-IllegalTransformPlus-${n}.xml`,
+    ...[
+      'fail-IllegalTransformPlus-1',
+      'fail-IllegalTransformPlus-2',
+    ].map(n => ({
+      subpath: `sections/tran/${n}.xml`,
       errors: [
         {
           code: LdmlCompilerMessages.ERROR_IllegalTransformPlus,

developer/src/kmc-ldml/test/tsconfig.json

@@ -10,7 +10,7 @@
     },
   "include": [
       "**/*.tests.ts",
-      "./helpers/index.ts"
+      "./helpers/*.ts",
   ],
   "references": [
       { "path": "../../../../common/web/keyman-version" },

developer/src/kmc-ldml/tsconfig.json

@@ -7,7 +7,8 @@
     "baseUrl": ".",
   },
   "include": [
-    "src/**/*.ts"
+    "src/**/*.ts",
+    "src/util/abnf/**/*.js",
   ],
   "references": [
     { "path": "../../../common/web/keyman-version" },

docs/build/windows.md

@@ -162,8 +162,19 @@ SETX KEYMAN_ROOT "c:\Projects\keyman\keyman"
 * KeymanWeb
 
 **Requirements**:
-* Emscripten
 * node.js
+* Emscripten
+
+#### node.js
+
+Our recommended way to install node.js is to use
+[nvm-windows](https://github.com/coreybutler/nvm-windows). This makes it
+easy to switch between versions of node.js.
+
+```bat
+nvm install 20.16.0
+nvm use 20.16.0
+```
 
 #### Emscripten
 
@@ -216,17 +227,6 @@ installed on your computer:
 SETX KEYMAN_USE_EMSDK 1
 ```
 
-#### node.js
-
-Our recommended way to install node.js is to use
-[nvm-windows](https://github.com/coreybutler/nvm-windows). This makes it
-easy to switch between versions of node.js.
-
-```bat
-nvm install 20.16.0
-nvm use 20.16.0
-```
-
 **Optional environment variables**:
 
 To let the Keyman build scripts control the version of node.js installed

linux/debian/control

@@ -43,8 +43,8 @@ Build-Depends:
  xserver-xephyr,
  xvfb,
 Standards-Version: 4.7.0
-Vcs-Git: https://github.com/keymanapp/keyman.git -b stable-17.0 [linux/debian]
-Vcs-Browser: https://github.com/keymanapp/keyman/tree/stable-17.0/linux/debian
+Vcs-Git: https://github.com/keymanapp/keyman.git -b beta [linux/debian]
+Vcs-Browser: https://github.com/keymanapp/keyman/tree/beta/linux/debian
 Homepage: https://www.keyman.com
 Rules-Requires-Root: binary-targets
 

linux/ibus-keyman/src/KeymanSystemServiceClient.cpp

@@ -25,6 +25,7 @@ public:
   void SetCapsLockIndicator(guint32 capsLock);
   gint32 GetCapsLockIndicator();
   void CallOrderedOutputSentinel();
+  void Ping();
 };
 
 KeymanSystemServiceClient::KeymanSystemServiceClient() {
@@ -111,6 +112,23 @@ KeymanSystemServiceClient::CallOrderedOutputSentinel() {
   }
 }
 
+void KeymanSystemServiceClient::Ping() {
+  if (!bus) {
+    // we already reported the error in the c'tor, so just return
+    return;
+  }
+
+  sd_bus_error *error = NULL;
+  int result = sd_bus_call_method(bus, KEYMAN_BUS_NAME, KEYMAN_OBJECT_PATH,
+    KEYMAN_INTERFACE_NAME, "Ping", error, &msg, "");
+  if (result < 0) {
+    g_error("%s: Failed to call method Ping: %s. %s. %s.",
+      __FUNCTION__, strerror(-result), error ? error->name : "-", error ? error->message : "-");
+    sd_bus_error_free(error);
+    return;
+  }
+}
+
 void
 set_capslock_indicator(guint32 capsLock) {
   KeymanSystemServiceClient client;
@@ -128,3 +146,10 @@ call_ordered_output_sentinel() {
   KeymanSystemServiceClient client;
   client.CallOrderedOutputSentinel();
 }
+
+void
+ping_keyman_system_service() {
+  g_message("%s: Pinging keyman-system-service", __FUNCTION__);
+  KeymanSystemServiceClient client;
+  client.Ping();
+}

linux/ibus-keyman/src/KeymanSystemServiceClient.h

@@ -10,6 +10,7 @@ extern "C" {
 void set_capslock_indicator(guint32 capsLockState);
 gint32 get_capslock_indicator();
 void call_ordered_output_sentinel();
+void ping_keyman_system_service();
 
 #ifdef __cplusplus
 }

linux/ibus-keyman/src/engine.c

@@ -553,6 +553,8 @@ ibus_keyman_engine_constructor(
       return NULL;
     }
 
+    ping_keyman_system_service();
+
     set_context_if_needed(engine);
 
     return (GObject *) keyman;

linux/keyman-system-service/src/KeymanSystemService.cpp

@@ -76,11 +76,22 @@ on_call_ordered_output_sentinel(
   return sd_bus_reply_method_return(msg, "");
 }
 
+static int32_t
+on_ping(
+  sd_bus_message *msg,
+  void *user_data,
+  sd_bus_error *ret_error
+) {
+  *ret_error = SD_BUS_ERROR_NULL;
+  return sd_bus_reply_method_return(msg, "s", "pong");
+}
+
 static const sd_bus_vtable system_service_vtable[] = {
   SD_BUS_VTABLE_START(0),
     SD_BUS_METHOD("SetCapsLockIndicator", "b", "", on_set_caps_lock_indicator, SD_BUS_VTABLE_UNPRIVILEGED),
     SD_BUS_METHOD("GetCapsLockIndicator", "", "b", on_get_caps_lock_indicator, SD_BUS_VTABLE_UNPRIVILEGED),
     SD_BUS_METHOD("CallOrderedOutputSentinel", "", "", on_call_ordered_output_sentinel, SD_BUS_VTABLE_UNPRIVILEGED),
+    SD_BUS_METHOD("Ping", "", "s", on_ping, SD_BUS_VTABLE_UNPRIVILEGED),
   SD_BUS_VTABLE_END
 };
 

linux/keyman-system-service/src/com.keyman.SystemService1.System.xml

@@ -34,5 +34,13 @@
     <method name="CallOrderedOutputSentinel">
     </method>
 
+    <!--
+      Ping:
+
+      Verifies that the keyman-system-service is started and available
+    -->
+    <method name="Ping">
+    </method>
+
   </interface>
 </node>

package-lock.json

@@ -1290,10 +1290,12 @@
         "@types/mocha": "^5.2.7",
         "@types/node": "^20.4.1",
         "@types/semver": "^7.3.12",
+        "abnf": "^4.3.1",
         "c8": "^7.12.0",
         "chalk": "^2.4.2",
         "common-tags": "^1.8.2",
         "mocha": "^8.4.0",
+        "peggy": "^4.2.0",
         "typescript": "^5.4.5"
       }
     },
@@ -3658,6 +3660,30 @@
         "@octokit/openapi-types": "^11.2.0"
       }
     },
+    "node_modules/@peggyjs/from-mem": {
+      "version": "1.3.5",
+      "resolved": "https://registry.npmjs.org/@peggyjs/from-mem/-/from-mem-1.3.5.tgz",
+      "integrity": "sha512-oRyzXE7nirAn+5yYjCdWQHg3EG2XXcYRoYNOK8Quqnmm+9FyK/2YWVunwudlYl++M3xY+gIAdf0vAYS+p0nKfQ==",
+      "dev": true,
+      "dependencies": {
+        "semver": "7.6.3"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@peggyjs/from-mem/node_modules/semver": {
+      "version": "7.6.3",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz",
+      "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==",
+      "dev": true,
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
     "node_modules/@pkgjs/parseargs": {
       "version": "0.11.0",
       "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz",
@@ -5765,6 +5791,35 @@
         "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
       }
     },
+    "node_modules/abnf": {
+      "version": "4.3.1",
+      "resolved": "https://registry.npmjs.org/abnf/-/abnf-4.3.1.tgz",
+      "integrity": "sha512-j4A8wWqKqkcSjx5xFESo9GtW2EUvlUZutcWB1knhxSP9kaXJ/YwL0g6dvMhHRjCPCNsIWwNGoKMHzPwemSpCvw==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "commander": "^13.0.0",
+        "peggy": "^4.2.0"
+      },
+      "bin": {
+        "abnf_ast": "bin/abnf_ast.js",
+        "abnf_check": "bin/abnf_check.js",
+        "abnf_gen": "bin/abnf_gen.js",
+        "abnf_test": "bin/abnf_test.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/abnf/node_modules/commander": {
+      "version": "13.1.0",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-13.1.0.tgz",
+      "integrity": "sha512-/rFeCpNJQbhSZjGVwO9RFV3xPqbnERS8MmIQzCtD/zl6gpJuV/bMLuN92oG3F7d8oDEHHRrujSXNUr8fpjntKw==",
+      "dev": true,
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/accepts": {
       "version": "1.3.8",
       "license": "MIT",
@@ -12376,6 +12431,33 @@
         "through": "~2.3"
       }
     },
+    "node_modules/peggy": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/peggy/-/peggy-4.2.0.tgz",
+      "integrity": "sha512-ZjzyJYY8NqW8JOZr2PbS/J0UH/hnfGALxSDsBUVQg5Y/I+ZaPuGeBJ7EclUX2RvWjhlsi4pnuL1C/K/3u+cDeg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@peggyjs/from-mem": "1.3.5",
+        "commander": "^12.1.0",
+        "source-map-generator": "0.8.0"
+      },
+      "bin": {
+        "peggy": "bin/peggy.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/peggy/node_modules/commander": {
+      "version": "12.1.0",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz",
+      "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==",
+      "dev": true,
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/pend": {
       "version": "1.2.0",
       "license": "MIT"
@@ -13533,6 +13615,15 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/source-map-generator": {
+      "version": "0.8.0",
+      "resolved": "https://registry.npmjs.org/source-map-generator/-/source-map-generator-0.8.0.tgz",
+      "integrity": "sha512-psgxdGMwl5MZM9S3FWee4EgsEaIjahYV5AzGnwUvPhWeITz/j6rKpysQHlQ4USdxvINlb8lKfWGIXwfkrgtqkA==",
+      "dev": true,
+      "engines": {
+        "node": ">= 10"
+      }
+    },
     "node_modules/source-map-support": {
       "version": "0.5.21",
       "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz",

resources/build/version/src/fixupHistory.ts

@@ -6,6 +6,7 @@ import { GitHub } from '@actions/github';
 import { readFileSync, writeFileSync } from 'fs';
 import { gt } from 'semver';
 import { reportHistory } from './reportHistory.js';
+import { spawnChild } from './util/spawnAwait.js';
 
 interface PRInformation {
   title: string;
@@ -16,16 +17,28 @@ interface PRInformation {
 // splitPullsIntoHistory
 // ------------------------------------------------------------------------------------
 
-const splicePullsIntoHistory = async (pulls: PRInformation[]): Promise<{count: number, pulls: number[]}> => {
+const splicePullsIntoHistory = async (pulls: PRInformation[], base?: string): Promise<{count: number, pulls: number[]}> => {
 
   let currentPulls: number[] = [];
 
   //
-  // Get current version and history from VERSION.md and TIER.md
+  // Get current version and history from VERSION.md and TIER.md. This may not
+  // yet be committed, so read the details from the worktree.
   //
 
-  const version = readFileSync('./VERSION.md', 'utf8').trim();
-  const tier = readFileSync('./TIER.md', 'utf8').trim();
+  let version = readFileSync('./VERSION.md', 'utf8').trim();
+  let tier = readFileSync('./TIER.md', 'utf8').trim();
+
+  if(base) {
+    // If we are merging history from another branch, we need to use the data
+    // from that branch. In this case, we must assume that the version and tier
+    // data are definitely already committed.
+    const currentBase = (await spawnChild('git', ['branch', '--show-current'])).trim();
+    if(currentBase != base) {
+      version = (await spawnChild('git', ['show', base+':VERSION.md'])).trim();
+      tier = (await spawnChild('git', ['show', base+':TIER.md'])).trim();
+    }
+  }
   //logInfo(`VERSION="${version}"`);
 
   //
@@ -208,7 +221,7 @@ export const fixupHistory = async (
   // Splice these into HISTORY.md
   //
 
-  const historyResult = await splicePullsIntoHistory(pulls);
+  const historyResult = await splicePullsIntoHistory(pulls, base);
 
   //
   // Write a comment to GitHub for each of the pulls

resources/docker-images/base/Dockerfile

@@ -1,7 +1,8 @@
 # Keyman is copyright (C) SIL Global. MIT License.
 
-ARG UBUNTU_VERSION=latest
-FROM ubuntu:${UBUNTU_VERSION}
+ARG DISTRO=ubuntu
+ARG DISTRO_VERSION=latest
+FROM ${DISTRO}:${DISTRO_VERSION}
 
 LABEL org.opencontainers.image.authors="SIL Global."
 LABEL org.opencontainers.image.url="https://github.com/keymanapp/keyman.git"
@@ -21,8 +22,10 @@ ENV DEBCONF_NOWARNINGS=yes
 # Update to the latest
 RUN apt-get -q -y update &&  \
   apt-get -q -y install ca-certificates curl gnupg meson software-properties-common sudo && \
-  add-apt-repository ppa:keymanapp/keyman && \
-  add-apt-repository ppa:keymanapp/keyman-alpha
+  if [[ "$(lsb_release -is)" == "Ubuntu" ]]; then \
+    add-apt-repository ppa:keymanapp/keyman && \
+    add-apt-repository ppa:keymanapp/keyman-alpha ; \
+  fi
 RUN apt-get -q -y update &&  \
   apt-get -q -y upgrade
 

resources/docker-images/build.sh

@@ -18,7 +18,8 @@ builder_describe \
   ":core" \
   ":linux" \
   ":web" \
-  "--ubuntu-version=UBUNTU_VERSION  The Ubuntu version (default: ${KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER})" \
+  "--distro=DISTRO                  The distribution to use for the base image (debian or ubuntu, default: ubuntu)" \
+  "--distro-version=DISTRO_VERSION  The Ubuntu/Debian version (default: ${KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER})" \
   "--no-cache                       Force rebuild of docker images" \
   "build                            Build docker images" \
   "test                             Test the docker images by running configure,build,test for all or the specified platforms"
@@ -31,11 +32,7 @@ _add_build_args() {
   local name=$3
   local value
 
-  if [[ -n "${!var:-}" ]]; then
-    value="${!var}"
-  else
-    value="${!default_var:-}"
-  fi
+  value="${!var:=${!default_var:-}}"
 
   build_args+=(--build-arg="${var}=${value}")
 
@@ -49,11 +46,14 @@ _add_build_args() {
 _convert_parameters_to_build_args() {
   build_args=()
   build_version=
-  local required_node_version
+  local required_node_version keyman_default_distro
   # shellcheck disable=SC2034
   required_node_version="$(_print_expected_node_version)"
+  # shellcheck disable=SC2034
+  keyman_default_distro="ubuntu"
 
-  _add_build_args UBUNTU_VERSION               KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER  ""
+  _add_build_args DISTRO                       keyman_default_distro                    ""
+  _add_build_args DISTRO_VERSION               KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER  ""
   _add_build_args JAVA_VERSION                 KEYMAN_VERSION_JAVA                      java
   _add_build_args REQUIRED_NODE_VERSION        required_node_version                    node
   _add_build_args REQUIRED_EMSCRIPTEN_VERSION  KEYMAN_MIN_VERSION_EMSCRIPTEN            emsdk
@@ -63,8 +63,16 @@ _convert_parameters_to_build_args() {
   fi
 }
 
+_check_for_default_values() {
+  if [[ -z "${DISTRO_VERSION:-}" ]] && [[ -z "${JAVA_VERSION:-}" ]]; then
+    is_default_values=true
+  else
+    is_default_values=false
+  fi
+}
+
 _is_default_values() {
-  [[ -z "${UBUNTU_VERSION:-}" ]] && [[ -z "${JAVA_VERSION:-}" ]]
+  ${is_default_values}
 }
 
 build_action() {
@@ -72,10 +80,9 @@ build_action() {
 
   builder_echo debug "Building image for ${platform}"
 
-  _convert_parameters_to_build_args
-
   if [[ "${platform}" == "base" ]]; then
-    docker pull --platform "amd64" "ubuntu:${UBUNTU_VERSION:-${KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER}}"
+    # shellcheck disable=SC2154 # set by _convert_parameters_to_build_args
+    docker pull "${DISTRO}:${DISTRO_VERSION}"
   elif [[ "${platform}" == "linux" ]]; then
     cp "${KEYMAN_ROOT}/linux/debian/control" "${platform}"
   fi
@@ -87,12 +94,12 @@ build_action() {
   # shellcheck disable=SC2164
   cd "${platform}"
   # shellcheck disable=SC2248,SC2086
-  docker build ${OPTION_NO_CACHE:-} --platform amd64 -t "keymanapp/keyman-${platform}-ci:${build_version}" "${build_args[@]}" .
+  docker build ${OPTION_NO_CACHE:-} -t "keymanapp/keyman-${platform}-ci:${build_version}" "${build_args[@]}" .
   # If the user didn't specify particular versions we will additionaly create an image
   # with the tag 'default'.
   if _is_default_values; then
     builder_echo debug "Setting default tag for ${platform}"
-    docker build --platform amd64 -t "keymanapp/keyman-${platform}-ci:default" "${build_args[@]}" .
+    docker build -t "keymanapp/keyman-${platform}-ci:default" "${build_args[@]}" .
   fi
   # shellcheck disable=SC2164,SC2103
   cd -
@@ -103,9 +110,13 @@ test_action() {
   local platform=$1
 
   builder_echo debug "Testing image for ${platform}"
-  ./run.sh "${platform}" -- ./build.sh configure,build,test:"${platform}"
+  ./run.sh --distro "${DISTRO}" --distro-version "${DISTRO_VERSION}" \
+    "${platform}" -- ./build.sh configure,build,test:"${platform}"
 }
 
+_check_for_default_values
+_convert_parameters_to_build_args
+
 if builder_has_action build; then
   build_action base
   BASE_VERSION="${build_version}"

resources/docker-images/linux/Dockerfile

@@ -23,7 +23,7 @@ RUN apt-get update && \
 # version 2.0-1 from Mantic.
 RUN LCOV_VERSION=$(dpkg -s lcov | grep Version | cut -d' ' -f2) && \
   if dpkg --compare-versions "${LCOV_VERSION}" lt 2.0; then \
-    curl -sS -o /tmp/lcov.deb --location http://mirrors.kernel.org/ubuntu/pool/universe/l/lcov/lcov_2.0-1_all.deb && \
+    curl -sS -o /tmp/lcov.deb --location https://old-releases.ubuntu.com/ubuntu/pool/universe/l/lcov/lcov_2.0-1ubuntu0.2_all.deb && \
     apt-get -qy install /tmp/lcov.deb && \
     rm /tmp/lcov.deb ; \
   fi

resources/docker-images/run.sh

@@ -7,6 +7,7 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")"
 ## END STANDARD BUILD SCRIPT INCLUDE
 
 . "${KEYMAN_ROOT}/resources/build/minimum-versions.inc.sh"
+. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh"
 
 ################################ Main script ################################
 
@@ -16,44 +17,53 @@ builder_describe \
   "core" \
   "linux" \
   "web" \
-  "--ubuntu-version=UBUNTU_VERSION  The Ubuntu version (default: ${KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER})"
+  "--distro=DISTRO                  The distribution (debian or ubuntu, default: ubuntu)" \
+  "--distro-version=DISTRO_VERSION  The Ubuntu/Debian version (default: ${KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER})"
 
 builder_parse "$@"
 
 run_android() {
   docker run -it --rm -v "${KEYMAN_ROOT}":/home/build/build \
-    -v "${KEYMAN_ROOT}/core/build/docker-core":/home/build/build/core/build \
-    keymanapp/keyman-android-ci:default \
+    -v "${KEYMAN_ROOT}/core/build/docker-core/${build_dir}":/home/build/build/core/build \
+    "keymanapp/keyman-android-ci:${image_version}" \
     "${builder_extra_params[@]}"
 }
 
 run_core() {
   docker run -it --rm -v "${KEYMAN_ROOT}":/home/build/build \
-    -v "${KEYMAN_ROOT}/core/build/docker-core":/home/build/build/core/build \
-    keymanapp/keyman-core-ci:default \
+    -v "${KEYMAN_ROOT}/core/build/docker-core/${build_dir}":/home/build/build/core/build \
+    "keymanapp/keyman-core-ci:${image_version}" \
     "${builder_extra_params[@]}"
 }
 
 run_linux() {
-  mkdir -p "${KEYMAN_ROOT}/linux/build/docker-linux"
-  mkdir -p "${KEYMAN_ROOT}/linux/keyman-system-service/build/docker-linux"
+  mkdir -p "${KEYMAN_ROOT}/linux/build/docker-linux/${build_dir}"
+  mkdir -p "${KEYMAN_ROOT}/linux/keyman-system-service/build/docker-linux/${build_dir}"
   docker run -it --privileged --rm -v "${KEYMAN_ROOT}":/home/build/build \
-    -v "${KEYMAN_ROOT}/core/build/docker-core":/home/build/build/core/build \
-    -v "${KEYMAN_ROOT}/linux/build/docker-linux":/home/build/build/linux/build \
-    -v "${KEYMAN_ROOT}/linux/keyman-system-service/build/docker-linux":/home/build/build/linux/keyman-system-service/build \
+    -v "${KEYMAN_ROOT}/core/build/docker-core/${build_dir}":/home/build/build/core/build \
+    -v "${KEYMAN_ROOT}/linux/build/docker-linux/${build_dir}":/home/build/build/linux/build \
+    -v "${KEYMAN_ROOT}/linux/keyman-system-service/build/docker-linux/${build_dir}":/home/build/build/linux/keyman-system-service/build \
     -e DESTDIR=/tmp \
-    keymanapp/keyman-linux-ci:default \
+    "keymanapp/keyman-linux-ci:${image_version}" \
     "${builder_extra_params[@]}"
 }
 
 run_web() {
   docker run -it --privileged --rm -v "${KEYMAN_ROOT}":/home/build/build \
-    -v "${KEYMAN_ROOT}/core/build/docker-core":/home/build/build/core/build \
-    keymanapp/keyman-web-ci:default \
+    -v "${KEYMAN_ROOT}/core/build/docker-core/${build_dir}":/home/build/build/core/build \
+    "keymanapp/keyman-web-ci:${image_version}" \
     "${builder_extra_params[@]}"
 }
 
-mkdir -p "${KEYMAN_ROOT}/core/build/docker-core"
+if [[ -z "${DISTRO_VERSION:-}" ]]; then
+  image_version=default
+  build_dir=default
+else
+  image_version="${DISTRO:-}-${DISTRO_VERSION}-java${KEYMAN_VERSION_JAVA}-node$(_print_expected_node_version)-emsdk${KEYMAN_MIN_VERSION_EMSCRIPTEN}"
+  build_dir="${DISTRO:-}-${DISTRO_VERSION}"
+fi
+
+mkdir -p "${KEYMAN_ROOT}/core/build/docker-core/${build_dir}"
 
 builder_run_action android  run_android
 builder_run_action core     run_core

resources/standards-data/ldml-keyboards/46/abnf/transform-from-required.abnf

@@ -0,0 +1,143 @@
+; Copyright (c) 2025 Unicode, Inc.
+; For terms of use, see http://www.unicode.org/copyright.html
+; SPDX-License-Identifier: Unicode-3.0
+; CLDR data files are interpreted according to the LDML specification (http://unicode.org/reports/tr35/)
+
+; This is an ABNF grammar for the CLDR Keyboard spec transform match syntax.
+; Note that there are sample matching/failing data files in tools/scripts/keyboard-abnf-tests/
+
+; An entire <transform from="..." /> string.
+; Note that the empty string is not a match.
+; Also note that a string may match this ABNF but be invalid according to the spec - which see.
+
+from-match        = start-context atoms / atoms
+
+; special marker anchoring to the start of context
+start-context  = "^"
+
+; sequence of items for input match. note that empty is not allowed, must be at least one atom.
+atoms             = atom *(disjunction atom / atom)
+
+; for use with or
+disjunction = "|"
+
+; a 'quark' is the matching part of an atom, and then a quantifier
+atom = quark quantifier / quark
+
+; quark can be a grouping or non grouping
+quark  = non-group / group
+
+non-group = simple-matcher / escaped-codepoints / variable
+
+variable = string-variable / set-variable
+
+string-variable = "${" var-id "}"
+
+set-variable = "$[" var-id "]"
+
+; variable ID
+var-id = 1*32IDCHAR
+
+group = capturing-group / non-capturing-group
+
+quantifier    =  bounded-quantifier / optional-quantifier
+
+escaped-codepoints           = backslash "u" "{" codepoints-hex "}"
+escaped-codepoint           = backslash "u" "{" codepoint-hex "}"
+
+bounded-quantifier = "{" DIGIT "," DIGIT "}"
+optional-quantifier =  "?"
+
+non-capturing-group = "(" "?" ":" atoms ")"
+
+; a capturing group may not contain other capturing groups.
+capturing-group = "(" catoms ")"
+
+; capturing atoms can't include any groups
+catoms = catom *(catom)
+; capturing atoms can't include any groups
+catom = cquark / cquark quantifier
+
+; capturing atoms can't include groups
+cquark = non-group
+
+; multiple hex codepoints
+codepoints-hex = codepoint-hex *(SP codepoint-hex)
+
+; one hex codepoint (1-6 digits)
+codepoint-hex =  1*6LHEXDIG
+
+simple-matcher      = text-char / class / match-any-codepoint / match-marker
+
+match-any-codepoint = "."
+
+match-marker = match-any-marker / match-named-marker
+match-any-marker = "\m{.}"
+match-named-marker = "\m{" marker-id "}"
+; marker id is nmtoken, but may be UAX31 in the future.
+marker-id = NMTOKEN
+
+class = fixed-class / set-class
+
+fixed-class = backslash fixed-class-char
+
+fixed-class-char = "s" / "S" / "t" / "r" / "n" / "f" / "v" / "d" / "w" / "D" / "W"
+
+set-class = "[" set-negator set-members "]"
+set-members = set-member *(set-member)
+set-member = char-range / range-char / match-marker / escaped-codepoint
+char-range = range-edge "-" range-edge
+range-edge = escaped-codepoint / range-char
+set-negator = "^" / ""
+
+; Restrictions on characters in various contexts
+
+; normal text
+text-char         = content-char / ws / escaped-char / "-" / ":"
+; text in a range sequence
+range-char        = content-char / ws / escaped-range-char / "." / "|" / "{" / "}"
+; group for everything BUT syntax chars.
+content-char      = ASCII-PUNCT / ALPHA / DIGIT / NON-ASCII
+
+; Character escapes
+escaped-char = backslash ( escapable-char )
+escapable-char = backslash / "$" / "{" / "|" / "}" / "(" / ")" / "*" / "+" / "." / "/" / "?" / "[" / "]" / "^"
+
+escaped-range-char = backslash escapable-range-char
+escapable-range-char = escapable-char / "-"
+
+backslash    = %x5C ; U+005C REVERSE SOLIDUS "\"
+ws = SP / HTAB / CR / LF / %x3000
+
+IDCHAR = ALPHA / DIGIT / "_"
+; ASCII-CTRLS        = %x01-08       ; omit NULL (%x00), HTAB (%x09) and LF (%x0A)
+;                   / %x0B-0C        ; omit CR (%x0D)
+;                   / %x0E-1F        ; omit SP (%x20)
+ASCII-PUNCT        = %x21-23       ; omit DOLLAR
+                  / %x25-27        ; omit () * +
+                  / %x2C           ; omit . (%x2E) and - (%x2D)
+                  / %x2F           ; skip over digits and :
+                  / %x3B-3E        ; omit ? 3f
+                  / %x5F           ; omit upper A-Z and [\]^
+                  / %x60           ; omit a-z {|}
+                  / %x7E-7F        ; just for completeness
+NON-ASCII =         %x7E-D7FF      ; omit surrogates
+                  / %xE000-10FFFF  ; that's the rest. (TODO: omit other non-characters)
+
+; from STD-68
+DIGIT          =  %x30-39             ; 0-9
+ALPHA          =  %x41-5A / %x61-7A   ; A-Z / a-z
+SP             =  %x20
+HTAB           =  %xF900              ; horizontal tab
+LF             =  %x0A                ; linefeed
+CR             =  %x0D                ; carriage return
+HEXDIG         =  DIGIT / "A" / "B" / "C" / "D" / "E" / "F"
+; like HEXDIG but lowercase also
+LHEXDIG         =  HEXDIG / "a" / "b" / "c" / "d" / "e" / "f"
+
+; from XML
+NAMESTARTCHAR   =   	":" / ALPHA / "_" / %xC0-D6 / %xD8-F6 / %xF8-2FF / %x370-37D / %x37F-1FFF / %x200C-200D / %x2070-218F / %x2C00-2FEF / %x3001-D7FF / %xF900-FDCF / %xFDF0-FFFD
+NAMESTARTCHAR   =/  %x10000-10FFFF    ; SKIP-NODE-ABNF: TODO: <https://github.com/hildjj/node-abnf/issues/25>
+
+NAMECHAR	   =   	NAMESTARTCHAR / "-" / "." / DIGIT / %xB7 / %x0300-036F / %x203F-2040
+NMTOKEN	   =   	1*NAMECHAR

resources/standards-data/ldml-keyboards/46/abnf/transform-to-required.abnf

@@ -0,0 +1,95 @@
+; Copyright (c) 2025 Unicode, Inc.
+; For terms of use, see http://www.unicode.org/copyright.html
+; SPDX-License-Identifier: Unicode-3.0
+; CLDR data files are interpreted according to the LDML specification (http://unicode.org/reports/tr35/)
+
+; This is an ABNF grammar for the CLDR Keyboard spec transform to= (replacement) match syntax.
+; Note that there are sample matching/failing data files in tools/scripts/keyboard-abnf-tests/
+
+; An entire <transform to="..." /> string.
+; An empty string is valid, meaning deletion.
+; Also note that a string may match this ABNF but be invalid according to the spec - which see.
+to-replacement          = atoms
+
+; a sequence of items for the output production
+atoms             = *(atom)
+
+; each atom can be one of several things
+atom = replacement-char / escaped-char / group-reference / escaped-codepoints / named-marker / string-variable / mapped-set
+
+; normal text being output
+replacement-char         = content-char / ws / "-" / ":" / "(" / ")" / "." / "*" / "+" / "?" / "[" / "]" / "^" / "{" / "}" / "|"
+
+; Character escapes
+escaped-char = backslash ( backslash / "$" ) / "$$"
+
+; reference to a capture group
+group-reference = "$" DIGIT
+
+; hex codepoint such as \u{01234}
+escaped-codepoints           = backslash "u" "{" codepoints-hex "}"
+
+; multiple hex codepoints
+codepoints-hex = codepoint-hex *(SP codepoint-hex)
+
+; one hex codepoint (1-6 digits)
+codepoint-hex =  1*6LHEXDIG
+
+; a specific marker ID.
+named-marker = "\m{" marker-id "}"
+
+; marker id is nmtoken, but may be UAX31 in the future.
+marker-id = NMTOKEN
+
+
+; substitution of a string variable
+string-variable = "${" var-id "}"
+
+; variable ID
+var-id = 1*32IDCHAR
+
+; special case for a mapped set variable
+mapped-set = "$[1:" var-id "]"
+
+; group for everything BUT syntax chars.
+content-char      =  ASCII-PUNCT / ALPHA / DIGIT / NON-ASCII
+
+; \
+backslash    = %x5C ; U+005C REVERSE SOLIDUS "\"
+
+; whitespace
+ws = SP / HTAB / CR / LF / %x3000
+
+IDCHAR = ALPHA / DIGIT / "_"
+; below is same as transform-from for maintenance
+; ASCII-CTRLS        = %x01-08       ; omit NULL (%x00), HTAB (%x09) and LF (%x0A)
+;                   / %x0B-0C        ; omit CR (%x0D)
+;                   / %x0E-1F        ; omit SP (%x20)
+ASCII-PUNCT        = %x21-23       ; omit DOLLAR
+                  / %x25-27        ; omit () * +
+                  / %x2C           ; omit . (%x2E) and - (%x2D)
+                  / %x2F           ; skip over digits and :
+                  / %x3B-3E        ; omit ? 3f
+                  / %x5F           ; omit upper A-Z and [\]^
+                  / %x60           ; omit a-z {|}
+                  / %x7E-7F        ; just for completeness
+NON-ASCII =         %x7E-D7FF      ; omit surrogates
+                  / %xE000-10FFFF  ; that's the rest. (TODO: omit other non-characters)
+
+; from STD-68
+DIGIT          =  %x30-39             ; 0-9
+ALPHA          =  %x41-5A / %x61-7A   ; A-Z / a-z
+SP             =  %x20
+HTAB           =  %xF900              ; horizontal tab
+LF             =  %x0A                ; linefeed
+CR             =  %x0D                ; carriage return
+HEXDIG         =  DIGIT / "A" / "B" / "C" / "D" / "E" / "F"
+; like HEXDIG but lowercase also
+LHEXDIG         =  HEXDIG / "a" / "b" / "c" / "d" / "e" / "f"
+
+; from XML
+NAMESTARTCHAR   =   	":" / ALPHA / "_" / %xC0-D6 / %xD8-F6 / %xF8-2FF / %x370-37D / %x37F-1FFF / %x200C-200D / %x2070-218F / %x2C00-2FEF / %x3001-D7FF / %xF900-FDCF / %xFDF0-FFFD
+NAMESTARTCHAR   =/  %x10000-10FFFF    ; SKIP-NODE-ABNF: TODO: <https://github.com/hildjj/node-abnf/issues/25>
+
+NAMECHAR	   =   	NAMESTARTCHAR / "-" / "." / DIGIT / %xB7 / %x0300-036F / %x203F-2040
+NMTOKEN	   =   	1*NAMECHAR

resources/standards-data/ldml-keyboards/fetch-latest-cldr.sh

@@ -49,10 +49,12 @@ DTD_DIR="${KEYBOARDS_DIR}/dtd"
 IMPORT_DIR="${KEYBOARDS_DIR}/import"
 DATA_DIR="${KEYBOARDS_DIR}/3.0"
 TEST_DIR="${KEYBOARDS_DIR}/test"
+ABNF_DIR="${KEYBOARDS_DIR}/abnf"
 
 # a file to check
-CHECK_1="${DTD_DIR}/ldmlKeyboard3.dtd"      # Critical, present in prior CLDR
-CHECK_2="${DTD_DIR}/ldmlKeyboardTest3.dtd"  # Only in Keyboard 3.0+
+CHECK_1="${DTD_DIR}/ldmlKeyboard3.dtd"            # Critical, present in prior CLDR
+CHECK_2="${DTD_DIR}/ldmlKeyboardTest3.dtd"        # Only in Keyboard 3.0+
+CHECK_3="${ABNF_DIR}/transform-from-required.abnf" # Present in v47+
 
 if [[ ! -f "${CHECK_1}" ]];
 then
@@ -64,6 +66,12 @@ then
     builder_die "${CHECK_2} did not exist: is ${CLDR_DIR} a valid CLDR keyboard directory?"
 fi
 
+if [[ ! -f "${CHECK_3}" ]];
+then
+    builder_die "${CHECK_3} did not exist: does ${CLDR_DIR} contain CLDR 47+? Or did ABNF change?"
+fi
+
+
 # collect git info
 GIT_DESCRIBE=$(cd "${CLDR_DIR}" && git describe HEAD || echo unknown)
 GIT_SHA=$(cd "${CLDR_DIR}" && git rev-parse HEAD || echo unknown)
@@ -82,7 +90,7 @@ pwd
 # delete the old files in case some were removed from CLDR
 rm -rf ./import ./3.0 ./dtd ./test
 # copy over everything
-cp -Rv "${IMPORT_DIR}" "${DATA_DIR}" "${DTD_DIR}" "${TEST_DIR}" .
+cp -Rv "${IMPORT_DIR}" "${DATA_DIR}" "${DTD_DIR}" "${TEST_DIR}" "${ABNF_DIR}" .
 
 # delete old files, no reason to keep them
 rm -vf dtd/{ldmlKeyboard,ldmlPlatform}.{xsd,dtd}

windows/src/desktop/kmshell/main/Keyman.System.UpdateStateMachine.pas

@@ -118,6 +118,7 @@ uses
   GlobalProxySettings,
   kmint,
   keymanapi_TLB,
+  KeymanMutex,
   Keyman.System.KeymanSentryClient,
   Keyman.System.DownloadUpdate,
   Keyman.System.RemoteUpdateCheck,
@@ -131,6 +132,7 @@ const
   SPackageUpgradeFilename = 'upgrade_packages.inf';
   kmShellContinue = 0;
   kmShellExit = 1;
+  KeymanDownloadMutexName = 'KeymanDownloading';
 
   { State Class Memebers }
 
@@ -753,18 +755,29 @@ procedure DownloadingState.Enter;
 var
   DownloadResult: Boolean;
   RetryCount: Integer;
+  FMutex: TKeymanMutex;
 begin
   // Enter DownloadingState
   bucStateContext.SetRegistryState(usDownloading);
 
-  RetryCount := 0;
-  DownloadResult := False;
+  FMutex := TKeymanMutex.Create(KeymanDownloadMutexName);
 
-  while (not DownloadResult) and (RetryCount < 3) do
-  begin
-    DownloadResult := DownloadUpdatesBackground;
-    if not DownloadResult then
+  try
+    // Should be impossible but just exit anyway and let the process current
+    // downloading process finish.
+    if not FMutex.TakeOwnership then
+    begin
+      Exit;
+    end;
+
+    RetryCount := 0;
+    repeat
+      DownloadResult := DownloadUpdatesBackground;
       Inc(RetryCount);
+    until DownloadResult or (RetryCount >= 3);
+    FMutex.ReleaseOwnership;
+  finally
+    FreeAndNil(FMutex);
   end;
 
   if (not DownloadResult) then
@@ -805,14 +818,45 @@ begin
 end;
 
 function DownloadingState.HandleKmShell;
+var
+  FMutex: TKeymanMutex;
 begin
-  // Downloading state, in other process, so continue
+  // Whether already downloading in another process or download has failed and
+  // this function has clean up and force a restart, kmshell should continue processing
   Result := kmShellContinue;
+  // Check to ensure a download process is running if not
+  // clean up return to the the idle state and check for updates
+  FMutex := TKeymanMutex.Create(KeymanDownloadMutexName);
+  try
+    if FMutex.TakeOwnership then
+    begin
+      bucStateContext.RemoveCachedFiles;
+      FMutex.ReleaseOwnership; // Mutex must be freed before changing state
+      ChangeState(IdleState);
+      bucStateContext.CurrentState.HandleCheck;
+    end;
+  finally
+    FreeAndNil(FMutex);
+  end;
 end;
 
 procedure DownloadingState.HandleDownload;
+var
+  FMutex: TKeymanMutex;
 begin
-  // Enter Already Downloading
+  // If downloading process is not running clean files and return to idle
+  FMutex := TKeymanMutex.Create(KeymanDownloadMutexName);
+  try
+    if FMutex.TakeOwnership then
+    begin
+      bucStateContext.RemoveCachedFiles;
+      FMutex.ReleaseOwnership;  // Mutex must be freed before changing state
+      ChangeState(IdleState);
+      bucStateContext.CurrentState.HandleCheck;
+    end;
+  finally
+    FreeAndNil(FMutex);
+  end;
 end;
 
 procedure DownloadingState.HandleAbort;

windows/src/support/kmdevlink/UfrmMain.pas

@@ -12,7 +12,9 @@ uses
 
 const
   SStatusSiteURL = 'https://status.keyman.com';
-  SSearchURL = 'https://github.com/keymanapp/%s/issues/%s';
+  SOpenIssueURL = 'https://github.com/keymanapp/%s/issues/%d';
+  SOpenAllIssuesURL = 'https://github.com/keymanapp/%s/issues';
+  SIssueSearchURL = 'https://github.com/keymanapp/%s/issues?q=state%%3Aopen%%20%s';
   SAddIssueURL = 'https://github.com/keymanapp/keyman/issues/new';
 
 type
@@ -81,6 +83,7 @@ var
 implementation
 
 uses
+  System.NetEncoding,
   ErrorControlledRegistry,
   ShellApi,
   UfrmCharacterIdentifier,
@@ -228,8 +231,8 @@ end;
 
 procedure TfrmMain.cmdOpenIssueOrPRClick(Sender: TObject);
 var
-  parts: TArray<string>;
-  repo, number: string;
+  URL: string;
+  iq: TIssueQuery;
 begin
   with TfrmOpenCRMRecord.Create(Self) do
   try
@@ -240,20 +243,21 @@ begin
     Free;
   end;
 
-  parts := CustomerText.Split(['#']);
-  if Length(parts) = 1 then
+  iq := SearchTextToQuery(CustomerText);
+  if iq.searchString <> '' then
   begin
-    repo := 'keyman';
-    number := parts[0];
+    URL := Format(SIssueSearchURL, [iq.repo, TNetEncoding.URL.Encode(iq.searchString)]);
+  end
+  else if iq.issueNumber > 0 then
+  begin
+    URL := Format(SOpenIssueURL, [iq.repo, iq.issueNumber]);
   end
   else
   begin
-    repo := parts[0];
-    if repo = '' then repo := 'keyman' else repo := RepoShortNameToFullName(repo);
-
-    number := parts[1];
+    URL := Format(SOpenAllIssuesURL, [iq.repo]);
   end;
-  if not TUtilExecute.URL(Format(SSearchURL, [repo, number])) then  // I3349
+
+  if not TUtilExecute.URL(URL) then  // I3349
     ShowMessage(SysErrorMessage(GetLastError));
 end;
 

windows/src/support/kmdevlink/UfrmOpenCRMRecord.dfm

@@ -20,9 +20,9 @@ object frmOpenCRMRecord: TfrmOpenCRMRecord
   object TntLabel1: TLabel
     Left = 20
     Top = 20
-    Width = 62
+    Width = 115
     Height = 19
-    Caption = '&Issue/PR'
+    Caption = '&Issue/PR/Search'
     Font.Charset = DEFAULT_CHARSET
     Font.Color = clWindowText
     Font.Height = -16
@@ -45,9 +45,9 @@ object frmOpenCRMRecord: TfrmOpenCRMRecord
     ParentFont = False
   end
   object editSearchFor: TEdit
-    Left = 108
+    Left = 156
     Top = 17
-    Width = 341
+    Width = 293
     Height = 27
     Font.Charset = DEFAULT_CHARSET
     Font.Color = clWindowText
@@ -66,7 +66,7 @@ object frmOpenCRMRecord: TfrmOpenCRMRecord
     Caption = 'OK'
     Default = True
     ModalResult = 1
-    TabOrder = 1
+    TabOrder = 3
   end
   object cmdCancel: TButton
     Left = 240
@@ -76,12 +76,12 @@ object frmOpenCRMRecord: TfrmOpenCRMRecord
     Cancel = True
     Caption = 'Cancel'
     ModalResult = 2
-    TabOrder = 2
+    TabOrder = 4
   end
   object cbRepository: TComboBox
-    Left = 108
+    Left = 156
     Top = 69
-    Width = 341
+    Width = 293
     Height = 27
     Style = csDropDownList
     Font.Charset = DEFAULT_CHARSET
@@ -90,7 +90,16 @@ object frmOpenCRMRecord: TfrmOpenCRMRecord
     Font.Name = 'Tahoma'
     Font.Style = []
     ParentFont = False
-    TabOrder = 3
+    TabOrder = 1
     OnClick = cbRepositoryClick
   end
+  object cmdCopyHTML: TButton
+    Left = 156
+    Top = 110
+    Width = 159
+    Height = 25
+    Caption = 'Copy short form as &HTML link'
+    TabOrder = 2
+    OnClick = cmdCopyHTMLClick
+  end
 end

windows/src/support/kmdevlink/UfrmOpenCRMRecord.pas

@@ -17,9 +17,11 @@ type
     cmdCancel: TButton;
     lblRepo: TLabel;
     cbRepository: TComboBox;
+    cmdCopyHTML: TButton;
     procedure cbRepositoryClick(Sender: TObject);
     procedure editSearchForChange(Sender: TObject);
     procedure FormCreate(Sender: TObject);
+    procedure cmdCopyHTMLClick(Sender: TObject);
   private
     Changing: Boolean;
     function GetSearchText: WideString;
@@ -33,8 +35,21 @@ type
 function RepoFullNameToShortName(name: string): string;
 function RepoShortNameToFullName(name: string): string;
 
+
+type
+  TIssueQuery = record
+    searchString: string;
+    repo: string;
+    issueNumber: Integer;
+  end;
+
+function SearchTextToQuery(s: string): TIssueQuery;
+
 implementation
 
+uses
+  Vcl.Clipbrd;
+
 {$R *.dfm}
 
 const repos: TArray<TArray<string>> = [
@@ -74,28 +89,131 @@ begin
   Result := name;
 end;
 
+function SearchTextToQuery(s: string): TIssueQuery;
+var
+  parts: TArray<String>;
+begin
+  Result.repo := 'keyman';
+  Result.issueNumber := 0;
+  Result.searchString := '';
+
+  s := s.Trim;
+  parts := s.Split(['#']);
+  if Length(parts) = 0 then
+  begin
+    Exit;
+  end
+  else if Length(parts) = 1 then
+  begin
+    Result.issueNumber := StrToIntDef(s, 0);
+    if IntToStr(Result.issueNumber) <> s then
+    begin
+      Result.issueNumber := 0;
+      Result.searchString := s;
+    end;
+  end
+  else
+  begin
+    Result.repo := RepoShortNameToFullName(parts[0]);
+    Result.issueNumber := StrToIntDef(parts[1], 0);
+    if IntToStr(Result.issueNumber) <> parts[1] then
+    begin
+      Result.issueNumber := 0;
+      Result.searchString := parts[1];
+    end;
+  end;
+end;
+
+type
+  TMyClipboard = class(TClipboard);
+
+procedure TfrmOpenCRMRecord.cmdCopyHTMLClick(Sender: TObject);
+var
+  c: TMyClipboard;
+  m: TMemoryStream;
+  ss: TStream;
+  displayRepo, html, s: string;
+  iq: TIssueQuery;
+  CF_HTML: UINT;
+  header: string;
+const
+  // https://learn.microsoft.com/en-us/windows/win32/dataxchg/html-clipboard-format
+  // yeesh what a format
+  header_template =
+    'Version:0.9'#$D#$A+
+    'StartHTML:%0.09d'#$D#$A+
+    'EndHTML:%0.09d'#$D#$A+
+    'StartFragment:%0.09d'#$D#$A+
+    'EndFragment:%0.09d'#$D#$A;
+  start_fragment =  '<!--StartFragment -->';
+  end_fragment = '<!--EndFragment -->';
+  context_start = '<html>'#$D#$A'<body>'#$D#$A;
+  context_end = #$D#$A'</body>'#$D#$A'</html>';
+begin
+  iq := SearchTextToQuery(SearchText);
+  if iq.repo = 'keyman' then
+    displayRepo := ''
+  else
+    displayRepo := iq.repo;
+  html := Format('<a href="https://github.com/keymanapp/%0:s/issues/%1:d">%2:s#%1:d</a>', [
+    iq.repo, iq.issueNumber, displayRepo
+  ]);
+
+  // Warning, this will go sadly badly with non-ascii letters
+  // because I am lazily not using UTF8Strings at this point
+
+  header := Format(header_template, [0,0,0,0]);
+  s := Format(header_template, [
+    header.Length,
+    header.Length + context_start.Length + start_fragment.Length + html.Length + end_fragment.Length + context_end.Length,
+    header.Length + context_start.Length + start_fragment.Length,
+    header.Length + context_start.Length + start_fragment.Length + html.Length
+  ]) + context_start + start_fragment + html + end_fragment + context_end + #0;
+
+  ss := TStringStream.Create(s, TEncoding.UTF8);
+  try
+    CF_HTML := RegisterClipboardFormat('HTML Format');
+    m := TMemoryStream.Create;
+    try
+      m.CopyFrom(ss, 0);
+      c := TMyClipboard(Clipboard); // access protected members yay delphi
+      c.SetBuffer(CF_HTML, m.Memory^, m.Size);
+    finally
+      m.Free;
+    end;
+  finally
+    ss.Free;
+  end;
+end;
+
 procedure TfrmOpenCRMRecord.editSearchForChange(Sender: TObject);
 var
-  s: string;
-  parts: TArray<String>;
-  repo: string;
+  iq: TIssueQuery;
 begin
   if Changing then Exit;
 
   Changing := True;
 
-  s := editSearchFor.Text;
-  parts := s.Split(['#']);
-  if Length(parts) = 1 then
+  iq := SearchTextToQuery(SearchText);
+
+  if iq.searchString <> '' then
   begin
-    cbRepository.ItemIndex := cbRepository.Items.IndexOf('keyman');
+    cmdCopyHTML.Enabled := False;
+    cmdOK.Caption := '&Search';
+  end
+  else if iq.issueNumber > 0 then
+  begin
+    cmdCopyHTML.Enabled := True;
+    cmdOK.Caption := '&Open issue';
   end
   else
   begin
-    repo := RepoShortNameToFullName(parts[0]);
-    cbRepository.ItemIndex := cbRepository.Items.IndexOf(repo);
+    cmdCopyHTML.Enabled := False;
+    cmdOK.Caption := '&All issues';
   end;
 
+  cbRepository.ItemIndex := cbRepository.Items.IndexOf(iq.repo);
+
   Changing := False;
 end;
 
@@ -106,11 +224,12 @@ begin
   for i := 0 to High(repos) do
     cbRepository.Items.Add(repos[i][0]);
   cbRepository.ItemIndex := cbRepository.Items.IndexOf('keyman');
+  editSearchForChange(nil);
 end;
 
 function TfrmOpenCRMRecord.GetSearchText: WideString;
 begin
-  Result := editSearchFor.Text;
+  Result := Trim(editSearchFor.Text);
 end;
 
 procedure TfrmOpenCRMRecord.cbRepositoryClick(Sender: TObject);