From bc378dcac6f1050a7da6407fc474913387b2d2d5 Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Tue, 18 Jul 2023 19:16:23 -0500
Subject: [PATCH 01/83] =?UTF-8?q?feat(core):=20move=20transform=20processi?=
 =?UTF-8?q?ng=20to=20u32=20=20=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- merge code paths…

#7375
---
 core/src/ldml/ldml_processor.cpp         | 20 ++++-----
 core/src/ldml/ldml_transforms.cpp        | 54 +++++++++++-------------
 core/src/ldml/ldml_transforms.hpp        | 24 ++++-------
 core/tests/unit/ldml/test_transforms.cpp | 42 +++++++++---------
 4 files changed, 62 insertions(+), 78 deletions(-)

diff --git a/core/src/ldml/ldml_processor.cpp b/core/src/ldml/ldml_processor.cpp
index 06257c8b11..9e341e62ef 100644
--- a/core/src/ldml/ldml_processor.cpp
+++ b/core/src/ldml/ldml_processor.cpp
@@ -250,11 +250,8 @@ ldml_processor::process_event(
               // not a char, get out
               break;
             }
+            ctxt.emplace_front(1, c->character);
             // extract UTF-32 to 1 or 2 UTF-16 chars in a string
-            km::kbp::kmx::char16_single buf;
-            const int len = km::kbp::kmx::Utf32CharToUtf16(c->character, buf);
-            const std::u16string str(buf.ch, len);
-            ctxt.push_front(str); // prepend to string
           }
         }
 
@@ -276,12 +273,12 @@ ldml_processor::process_event(
         // Process the transforms
         if (!!transforms) {
           // add the newly added char to ctxt
-          ctxt.push_back(str);
+          ctxt.push_back(str32);
+
+          std::u32string outputString;
 
-          std::u16string outputString;
-  
           // TODO-LDML: unroll ctxt into a str. Would be better to have transforms be able to process a vector
-          std::u16string ctxtstr;
+          std::u32string ctxtstr;
           for (size_t i = 0; i < ctxt.size(); i++) {
             ctxtstr.append(ctxt[i]);
           }
@@ -296,10 +293,9 @@ ldml_processor::process_event(
               state->actions().push_backspace(KM_KBP_BT_CHAR, deletedChar);  // Cause prior char to be removed
             }
             // Now, add in the updated text
-            const std::u32string outstr32 = kmx::u16string_to_u32string(outputString);
-            for (size_t i = 0; i < outstr32.length(); i++) {
-              state->context().push_character(outstr32[i]);
-              state->actions().push_character(outstr32[i]);
+            for (size_t i = 0; i < outputString.length(); i++) {
+              state->context().push_character(outputString[i]);
+              state->actions().push_character(outputString[i]);
             }
           }
         }
diff --git a/core/src/ldml/ldml_transforms.cpp b/core/src/ldml/ldml_transforms.cpp
index 9d5275ab41..7d6920fe57 100644
--- a/core/src/ldml/ldml_transforms.cpp
+++ b/core/src/ldml/ldml_transforms.cpp
@@ -9,6 +9,8 @@
 #include "debuglog.h"
 #include <algorithm>
 #include <string>
+#include "kmx/kmx_xstring.h"
+
 
 #ifndef assert
 #define assert(x)  // TODO-LDML
@@ -279,11 +281,11 @@ reorder_group::apply(std::u32string &str) const {
   return applied;
 }
 
-transform_entry::transform_entry(const std::u16string &from, const std::u16string &to) : fFrom(from), fTo(to) {
+transform_entry::transform_entry(const std::u32string &from, const std::u32string &to) : fFrom(from), fTo(to) {
 }
 
 size_t
-transform_entry::match(const std::u16string &input) const {
+transform_entry::match(const std::u32string &input) const {
   if (input.length() < fFrom.length()) {
     return 0;
   }
@@ -295,8 +297,8 @@ transform_entry::match(const std::u16string &input) const {
   return substr.length();
 }
 
-std::u16string
-transform_entry::apply(const std::u16string & /*input*/, size_t /*matchLen*/) const {
+std::u32string
+transform_entry::apply(const std::u32string & /*input*/, size_t /*matchLen*/) const {
   return fTo;
 }
 
@@ -325,7 +327,7 @@ transform_group::transform_group() {
  * return the first transform match in this group
  */
 const transform_entry *
-transform_group::match(const std::u16string &input, size_t &subMatched) const {
+transform_group::match(const std::u32string &input, size_t &subMatched) const {
   for (auto transform = begin(); (subMatched == 0) && (transform < end()); transform++) {
     // TODO-LDML: non regex implementation
     // is the match area too short?
@@ -345,7 +347,7 @@ transform_group::match(const std::u16string &input, size_t &subMatched) const {
  * @return match length: number of chars at end of input string to modify.  0 if no match.
  */
 size_t
-transforms::apply(const std::u16string &input, std::u16string &output) {
+transforms::apply(const std::u32string &input, std::u32string &output) {
   /**
    * Example:
    * Group0:   za -> c, a -> bb
@@ -380,7 +382,7 @@ transforms::apply(const std::u16string &input, std::u16string &output) {
    */
   size_t matched = 0;
   /** modified copy of input */
-  std::u16string updatedInput = input;
+  std::u32string updatedInput = input;
   for (auto group = transform_groups.begin(); group < transform_groups.end(); group++) {
     // for each transform group
     // break out once there's a match
@@ -398,7 +400,7 @@ transforms::apply(const std::u16string &input, std::u16string &output) {
         // now apply the found transform
 
         // update subOutput (string) and subMatched
-        std::u16string subOutput = transform->apply(updatedInput, subMatched);
+        std::u32string subOutput = transform->apply(updatedInput, subMatched);
 
         // remove the matched part of the updatedInput
         updatedInput.resize(updatedInput.length() - subMatched);  // chop of the subMatched part at end
@@ -420,7 +422,15 @@ transforms::apply(const std::u16string &input, std::u16string &output) {
         }
       }
     } else if (group->type == any_group_type::reorder) {
-      // TODO-LDML reorder
+      // TODO-LDML: cheesy solution
+      std::u32string str2 = updatedInput;
+      if (group->reorder.apply(str2)) {
+        // pretend the whole thing matched
+        output.resize(0);
+        output.append(str2);
+        updatedInput.resize(0);
+        updatedInput.append(str2);
+      }
     }
     // else: continue to next group
   }
@@ -442,8 +452,8 @@ transforms::apply(const std::u16string &input, std::u16string &output) {
 
 // simple impl
 bool
-transforms::apply(std::u16string &str) {
-  std::u16string output;
+transforms::apply(std::u32string &str) {
+  std::u32string output;
   size_t matchLength = apply(str, output);
   if (matchLength == 0) {
     return false;
@@ -452,22 +462,6 @@ transforms::apply(std::u16string &str) {
   str.append(output);
   return true;
 }
-
-bool
-transforms::apply(std::u32string &str) {
-  bool rc = false;
-  // TODO-LDML: PoC implementation for now, need to refactor into fcns
-  // ONLY reorder
-  for (auto group = transform_groups.begin(); group < transform_groups.end(); group++) {
-    assert(group->type == reorder);  // TODO-LDML
-    auto rgroup = group->reorder;
-    if (rgroup.apply(str)) {
-      rc = true;
-    }
-  }
-  return rc;
-}
-
 // Loader
 
 transforms *
@@ -515,8 +509,8 @@ transforms::load(
 
       for (KMX_DWORD itemNumber = 0; itemNumber < group->count; itemNumber++) {
         const kmx::COMP_KMXPLUS_TRAN_TRANSFORM *element = tranHelper.getTransform(group->index + itemNumber);
-        const std::u16string fromStr                    = kplus.strs->get(element->from);
-        const std::u16string toStr                      = kplus.strs->get(element->to);
+        const std::u32string fromStr                    = kmx::u16string_to_u32string(kplus.strs->get(element->from));
+        const std::u32string toStr                      = kmx::u16string_to_u32string(kplus.strs->get(element->to));
         std::u16string mapFrom, mapTo;
 
         if (element->mapFrom && element->mapTo) {
@@ -525,7 +519,7 @@ transforms::load(
           mapTo   = kplus.strs->get(element->mapTo);
         }
 
-        newGroup.emplace_back(fromStr, toStr);  // creating a transform_entry
+        newGroup.emplace_back(fromStr, toStr /* ,mapFrom, mapTo */);  // creating a transform_entry
       }
       transforms->addGroup(newGroup);
     } else if (group->type == LDML_TRAN_GROUP_TYPE_REORDER) {
diff --git a/core/src/ldml/ldml_transforms.hpp b/core/src/ldml/ldml_transforms.hpp
index 7904eeffc8..bd278d059a 100644
--- a/core/src/ldml/ldml_transforms.hpp
+++ b/core/src/ldml/ldml_transforms.hpp
@@ -62,30 +62,30 @@ private:
 class transform_entry {
 public:
   transform_entry(
-      const std::u16string &from,
-      const std::u16string &to
+      const std::u32string &from,
+      const std::u32string &to
       /*TODO-LDML: mapFrom, mapTo*/
   );
 
   /**
    * @returns length if it's a match
    */
-  size_t match(const std::u16string &input) const;
+  size_t match(const std::u32string &input) const;
 
   /**
    * @returns output string
    */
-  std::u16string apply(const std::u16string &input, size_t matchLen) const;
+  std::u32string apply(const std::u32string &input, size_t matchLen) const;
 
 private:
-  const std::u16string fFrom;  // TODO-LDML: regex
-  const std::u16string fTo;
+  const std::u32string fFrom;  // TODO-LDML: regex
+  const std::u32string fTo;
 };
 
 /**
  * An ordered list of strings.
  */
-typedef std::deque<std::u16string> string_list;
+typedef std::deque<std::u32string> string_list;
 
 /**
  * a group of <transform> entries - a <transformGroup>
@@ -100,7 +100,7 @@ public:
    * @param subMatched on output, the matched length
    * @returns alias to transform_entry or nullptr
    */
-  const transform_entry *match(const std::u16string &input, size_t &subMatched) const;
+  const transform_entry *match(const std::u32string &input, size_t &subMatched) const;
 };
 
 /** a single char, categorized according to reorder rules*/
@@ -217,13 +217,7 @@ public:
    * @param output if matched, contains the replacement output text
    * @return length in chars of the input (counting from the end) which matched context
    */
-  size_t apply(const std::u16string &input, std::u16string &output);
-
-  /**
-   * For tests
-   * @return true if str was altered
-   */
-  bool apply(std::u16string &str);
+  size_t apply(const std::u32string &input, std::u32string &output);
 
   /**
    * For tests - TODO-LDML only supports reorder
diff --git a/core/tests/unit/ldml/test_transforms.cpp b/core/tests/unit/ldml/test_transforms.cpp
index c72fd6bcf8..a28ca46ef3 100644
--- a/core/tests/unit/ldml/test_transforms.cpp
+++ b/core/tests/unit/ldml/test_transforms.cpp
@@ -40,7 +40,7 @@ test_transforms() {
   std::cout << __FILE__ << ":" << __LINE__ << " - basic " << std::endl;
   {
     // start with one
-    transform_entry te(std::u16string(u"e^"), std::u16string(u"E"));  // keep it simple
+    transform_entry te(std::u32string(U"e^"), std::u32string(U"E"));  // keep it simple
     // OK now make a group do it
     transforms tr;
     transform_group st;
@@ -51,17 +51,17 @@ test_transforms() {
 
     // see if we can match the same
     {
-      std::u16string src(u"barQ^");
+      std::u32string src(U"barQ^");
       bool res = tr.apply(src);
       zassert_equal(res, false);
-      zassert_string_equal(src, std::u16string(u"barQ^"));  // no change
+      zassert_string_equal(src, std::u32string(U"barQ^"));  // no change
     }
 
     {
-      std::u16string src(u"fooe^");
+      std::u32string src(U"fooe^");
       bool res = tr.apply(src);
       zassert_equal(res, true);
-      zassert_string_equal(src, std::u16string(u"fooE"));
+      zassert_string_equal(src, std::u32string(U"fooE"));
     }
   }
 
@@ -73,23 +73,23 @@ test_transforms() {
     // setup
     {
       transform_group st;
-      st.emplace_back(std::u16string(u"za"), std::u16string(u"c"));
-      st.emplace_back(std::u16string(u"a"), std::u16string(u"bb"));
+      st.emplace_back(std::u32string(U"za"), std::u32string(U"c"));
+      st.emplace_back(std::u32string(U"a"), std::u32string(U"bb"));
       tr.addGroup(st);
     }
     {
       transform_group st;
-      st.emplace_back(std::u16string(u"bb"), std::u16string(u"ccc"));
+      st.emplace_back(std::u32string(U"bb"), std::u32string(U"ccc"));
       tr.addGroup(st);
     }
     {
       transform_group st;
-      st.emplace_back(std::u16string(u"cc"), std::u16string(u"d"));
+      st.emplace_back(std::u32string(U"cc"), std::u32string(U"d"));
       tr.addGroup(st);
     }
     {
       transform_group st;
-      st.emplace_back(std::u16string(u"tcd"), std::u16string(u"e"));
+      st.emplace_back(std::u32string(U"tcd"), std::u32string(U"e"));
       tr.addGroup(st);
     }
 
@@ -97,31 +97,31 @@ test_transforms() {
 
     // see if we can match the same
     {
-      std::u16string src(u"ta");
+      std::u32string src(U"ta");
       bool res = tr.apply(src);
       // pipe (|) symbol shows where the 'output' is delineated
       // t|a --> t|bb --> t|ccc --> t|cd --> |e
-      zassert_string_equal(src, std::u16string(u"e"));
+      zassert_string_equal(src, std::u32string(U"e"));
       zassert_equal(res, true);
     }
     {
-      std::u16string src(u"qza");
+      std::u32string src(U"qza");
       bool res = tr.apply(src);
       // pipe (|) symbol shows where the 'output' is delineated
       // q|za -> q|c
-      zassert_string_equal(src, std::u16string(u"qc"));
+      zassert_string_equal(src, std::u32string(U"qc"));
       zassert_equal(res, true);
     }
     {
-      std::u16string src(u"qa");
+      std::u32string src(U"qa");
       bool res = tr.apply(src);
-      zassert_string_equal(src, std::u16string(u"qcd"));
+      zassert_string_equal(src, std::u32string(U"qcd"));
       zassert_equal(res, true);
     }
     {
-      std::u16string src(u"tb");
+      std::u32string src(U"tb");
       bool res = tr.apply(src);
-      zassert_string_equal(src, std::u16string(u"tb"));
+      zassert_string_equal(src, std::u32string(U"tb"));
       zassert_equal(res, false);
     }
   }
@@ -132,13 +132,13 @@ test_transforms() {
     transforms tr;
     {
       transform_group st;
-      st.emplace_back(std::u16string(u"िह"), std::u16string(u"हि"));
+      st.emplace_back(std::u32string(U"िह"), std::u32string(U"हि"));
       tr.addGroup(st);
     }
     {
-      std::u16string src(u"िह");
+      std::u32string src(U"िह");
       bool res = tr.apply(src);
-      zassert_string_equal(src, std::u16string(u"हि"));
+      zassert_string_equal(src, std::u32string(U"हि"));
       zassert_equal(res, true);
     }
   }

From d31118c63df0ec27f249d709e4670ed01aac12ac Mon Sep 17 00:00:00 2001
From: "Joshua A. Horton" <joshua_horton@sil.org>
Date: Wed, 19 Jul 2023 14:21:59 +0700
Subject: [PATCH 02/83] fix(web): allows registering precached keyboards

---
 .../package-cache/src/cloud/queryEngine.ts    | 20 ++++++++++++++++---
 .../src/keyboardRequisitioner.ts              | 12 +++++++++++
 2 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/web/src/engine/package-cache/src/cloud/queryEngine.ts b/web/src/engine/package-cache/src/cloud/queryEngine.ts
index e4651e0975..219dd911df 100644
--- a/web/src/engine/package-cache/src/cloud/queryEngine.ts
+++ b/web/src/engine/package-cache/src/cloud/queryEngine.ts
@@ -1,3 +1,5 @@
+import { EventEmitter } from 'eventemitter3';
+
 import { PathConfiguration } from 'keyman/engine/paths';
 
 import { default as KeyboardStub, ErrorStub, KeyboardAPISpec, mergeAndResolveStubPromises } from '../keyboardStub.js';
@@ -55,7 +57,11 @@ type CloudLanguagesQueryResult = {
 
 export type CloudQueryResult = CloudKeyboardQueryResult | CloudLanguagesQueryResult;
 
-export default class CloudQueryEngine {
+interface EventMap {
+  'unboundregister': (registration: ReturnType<CloudQueryEngine['_registerCore']>) => void
+}
+
+export default class CloudQueryEngine extends EventEmitter<EventMap> {
   private cloudResolutionPromises: Record<number, ManagedPromise<KeyboardStub[] | ManagedPromise<LanguageAPIPropertySpec[]>>> = {};
 
   private _languageListPromise: ManagedPromise<LanguageAPIPropertySpec[]>;
@@ -65,6 +71,8 @@ export default class CloudQueryEngine {
   private pathConfig: PathConfiguration;
 
   constructor(requestEngine: CloudRequesterInterface, pathConfig: PathConfiguration) {
+    super();
+
     this.requestEngine = requestEngine;
     this.pathConfig = pathConfig;
 
@@ -138,10 +146,16 @@ export default class CloudQueryEngine {
       result = new Error(CLOUD_REGISTRATION_ERR + err);
     }
 
-    if(promiseid) {
+    if(!promiseid) {
+      this.emit('unboundregister', result);
+      return;
+    } else {
       const promise: ManagedPromise<KeyboardStub[]> | ManagedPromise<LanguageAPIPropertySpec[]> = this.cloudResolutionPromises[promiseid];
 
-      if(promise) {
+      if(!promise) {
+        this.emit('unboundregister', result);
+        return;
+      } else {
         try {
           if(result instanceof Error) {
             promise.reject(result as Error);
diff --git a/web/src/engine/package-cache/src/keyboardRequisitioner.ts b/web/src/engine/package-cache/src/keyboardRequisitioner.ts
index 6a3f060510..e06703234c 100644
--- a/web/src/engine/package-cache/src/keyboardRequisitioner.ts
+++ b/web/src/engine/package-cache/src/keyboardRequisitioner.ts
@@ -97,6 +97,18 @@ export default class KeyboardRequisitioner {
     this.pathConfig = pathConfig;
     this.cache = new StubAndKeyboardCache(keyboardLoader);
     this.cloudQueryEngine = new CloudQueryEngine(keyboardRequester, this.pathConfig);
+
+    // Handles keymanweb.com's precached keyboard array.  There is no associated promise,
+    // so there's nothing handling the `register` call's results otherwise.
+    this.cloudQueryEngine.on('unboundregister', (registration) => {
+      try {
+        if(Array.isArray(registration)) {
+          registration.forEach((entry) => {
+            this.cache.addStub(entry);
+          });
+        }
+      } finally { }
+    });
   }
 
   addKeyboardArray(x: (string|RawKeyboardMetadata)[]): Promise<(KeyboardStub | ErrorStub)[]> {

From 6581dbbbdb5fb3d83744b0b1b67c768df82711fb Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Wed, 19 Jul 2023 15:36:02 +0700
Subject: [PATCH 03/83] chore(web): Apply suggestions from code review

---
 .../package-cache/src/keyboardRequisitioner.ts     | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/web/src/engine/package-cache/src/keyboardRequisitioner.ts b/web/src/engine/package-cache/src/keyboardRequisitioner.ts
index e06703234c..aeedc27300 100644
--- a/web/src/engine/package-cache/src/keyboardRequisitioner.ts
+++ b/web/src/engine/package-cache/src/keyboardRequisitioner.ts
@@ -101,13 +101,13 @@ export default class KeyboardRequisitioner {
     // Handles keymanweb.com's precached keyboard array.  There is no associated promise,
     // so there's nothing handling the `register` call's results otherwise.
     this.cloudQueryEngine.on('unboundregister', (registration) => {
-      try {
-        if(Array.isArray(registration)) {
-          registration.forEach((entry) => {
-            this.cache.addStub(entry);
-          });
-        }
-      } finally { }
+      // Internal, undocumented use-case of `keyman.register`:  precached keyboard loading
+      // Other uses may trigger errors, especially if there's a type-structure mismatch.
+      // Those errors should not be handled here; let them surface.
+      if(Array.isArray(registration)) {
+        registration.forEach((entry) => {
+          this.cache.addStub(entry);
+        });
     });
   }
 

From 152d36a654ba06a57cd1b7be86fe20383116cdc8 Mon Sep 17 00:00:00 2001
From: Eberhard Beilharz <eb1@sil.org>
Date: Wed, 14 Jun 2023 18:01:14 +0200
Subject: [PATCH 04/83] fix(linux): Fix installation of keyboards with lang tag
 `mul`

We now use the canonical tag without appending the script (unless
necessary).

Fixes #8620.
---
 linux/keyman-config/keyman_config/install_kmp.py |  4 ++--
 linux/keyman-config/tests/test_install_kmp.py    | 13 ++++++++-----
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/linux/keyman-config/keyman_config/install_kmp.py b/linux/keyman-config/keyman_config/install_kmp.py
index 06c41a0cac..95d1e294e0 100755
--- a/linux/keyman-config/keyman_config/install_kmp.py
+++ b/linux/keyman-config/keyman_config/install_kmp.py
@@ -238,9 +238,9 @@ class InstallKmp():
         if not language:
             return language
 
-        language = CanonicalLanguageCodeUtils.findBestTag(language, False, True)
+        language = CanonicalLanguageCodeUtils.findBestTag(language, False, False)
         for supportedLanguage in supportedLanguages:
-            tag = CanonicalLanguageCodeUtils.findBestTag(supportedLanguage['id'], False, True)
+            tag = CanonicalLanguageCodeUtils.findBestTag(supportedLanguage['id'], False, False)
             if tag == language:
                 return tag
         return None
diff --git a/linux/keyman-config/tests/test_install_kmp.py b/linux/keyman-config/tests/test_install_kmp.py
index d1e10bd645..3320c532b5 100644
--- a/linux/keyman-config/tests/test_install_kmp.py
+++ b/linux/keyman-config/tests/test_install_kmp.py
@@ -199,19 +199,22 @@ class InstallKmpTests(unittest.TestCase):
         languages = [
             {'id': 'de'},
             {'id': 'esi-Latn'},
-            {'id': 'dyo'}
+            {'id': 'dyo'},
+            {'id': 'fuh-Arab'}
         ]
 
         for testcase in [
             {'given': 'de', 'expected': 'de'},
-            {'given': 'esi', 'expected': 'esi-Latn'},
-            {'given': 'esi-Latn', 'expected': 'esi-Latn'},
+            {'given': 'esi', 'expected': 'esi'},
+            {'given': 'esi-Latn', 'expected': 'esi'},
             {'given': 'es', 'expected': None},
             {'given': 'en', 'expected': None},
             {'given': None, 'expected': None},
             # #3399
-            {'given': 'dyo-latn', 'expected': 'dyo-Latn'},
-            {'given': 'dyo', 'expected': 'dyo-Latn'},
+            {'given': 'dyo-latn', 'expected': 'dyo'},
+            {'given': 'dyo', 'expected': 'dyo'},
+            {'given': 'fuh-Arab', 'expected': 'fuh-Arab'},
+            {'given': 'fuh', 'expected': None},
         ]:
             with self.subTest(data=testcase):
                 # Execute

From 81f46e1a6d05bdf601b30dc39a013167dd1f5166 Mon Sep 17 00:00:00 2001
From: Eberhard Beilharz <eb1@sil.org>
Date: Thu, 15 Jun 2023 18:27:57 +0200
Subject: [PATCH 05/83] feat(linux): Minimize tags before adding keyboards

This ensures that bcp47 language tags are in a form the OS
understands.
---
 linux/ibus-keyman/build.sh                    |   2 +-
 linux/ibus-keyman/meson.build                 |   1 +
 linux/ibus-keyman/src/bcp47util.c             |  99 +++++++++++++++++
 linux/ibus-keyman/src/bcp47util.h             |   7 ++
 linux/ibus-keyman/src/keymanutil.c            |  72 ++++++++-----
 linux/ibus-keyman/src/meson.build             |   3 +-
 linux/ibus-keyman/src/test/bcp47util_tests.c  | 100 +++++++++++++++++
 linux/ibus-keyman/src/test/meson.build        | 101 ++++++++++++++----
 linux/ibus-keyman/src/test/run-single-test.sh |  48 +++++++++
 linux/ibus-keyman/src/test/run-tests.sh       |   6 +-
 linux/ibus-keyman/src/test/setup-tests.sh     |   6 ++
 linux/ibus-keyman/tests/meson.build           |   2 +-
 .../tests/scripts/test-helper.inc.sh          |  96 +++++++++++++----
 13 files changed, 466 insertions(+), 77 deletions(-)
 create mode 100644 linux/ibus-keyman/src/bcp47util.c
 create mode 100644 linux/ibus-keyman/src/bcp47util.h
 create mode 100644 linux/ibus-keyman/src/test/bcp47util_tests.c
 create mode 100755 linux/ibus-keyman/src/test/run-single-test.sh
 create mode 100755 linux/ibus-keyman/src/test/setup-tests.sh

diff --git a/linux/ibus-keyman/build.sh b/linux/ibus-keyman/build.sh
index 1b8ef97823..58bc057691 100755
--- a/linux/ibus-keyman/build.sh
+++ b/linux/ibus-keyman/build.sh
@@ -61,7 +61,7 @@ fi
 if builder_start_action test; then
   cd "$THIS_SCRIPT_PATH/$MESON_PATH"
   if builder_has_option --no-integration; then
-    meson test --print-errorlogs $builder_verbose keymanutil-tests print-kmpdetails-test print-kmp-test
+    meson test --print-errorlogs $builder_verbose setup-src-test keymanutil-tests print-kmpdetails-test print-kmp-test bcp47-util-tests teardown-src-test
   else
     meson test --print-errorlogs $builder_verbose
   fi
diff --git a/linux/ibus-keyman/meson.build b/linux/ibus-keyman/meson.build
index e4b31df7db..2e402b868e 100644
--- a/linux/ibus-keyman/meson.build
+++ b/linux/ibus-keyman/meson.build
@@ -13,6 +13,7 @@ ibus =           dependency('ibus-1.0',             version: '>= 1.2.0')
 gtk =            dependency('gtk+-3.0',             version: '>= 2.4')
 json_glib =      dependency('json-glib-1.0',        version: '>= 1.0')
 systemd  =       dependency('libsystemd')
+icu =            dependency('icu-i18n')
 
 core_dir = meson.current_source_dir() / '../../core'
 common_dir = meson.current_source_dir() / '../../common'
diff --git a/linux/ibus-keyman/src/bcp47util.c b/linux/ibus-keyman/src/bcp47util.c
new file mode 100644
index 0000000000..0b2013038a
--- /dev/null
+++ b/linux/ibus-keyman/src/bcp47util.c
@@ -0,0 +1,99 @@
+#include <gio/gio.h>
+#include <unicode/locid.h>
+#include <unicode/uloc.h>
+
+/// Minimize the BCP-47 `tag` so that unnecessary parts get ommitted.
+/// The result gets stored in `minimzedTag`.
+///
+/// @param tag           The tag to process
+/// @param minimizedTag  Caller-provided character array for the
+///                      resulting minimized tag.
+/// @param tagCapacity   Array size of `minimizedTag`
+/// @return The length of the minimized tag, or -1 in error case
+int bcp47_minimize(const char* tag, char* minimizedTag, int tagCapacity) {
+  UErrorCode status = U_ZERO_ERROR;
+  if (!tag || strlen(tag) == 0) {
+    strncpy(minimizedTag, "", tagCapacity);
+    return -1;
+  }
+
+  // special treatment for `und-Latn` which is used by sil_ipa keyboard
+  if (strcmp(tag, "und-Latn") == 0) {
+    strncpy(minimizedTag, "und-Latn", tagCapacity);
+    return strlen(minimizedTag);
+  }
+
+  int capacity = 255;
+  char workingTag[capacity];
+
+  // special treatment for tags that start with `und`: replace `und` with `en`.
+  // ICU 70 doesn't properly treat `und`.
+  int isUnd = strncmp(tag, "und", 3) == 0;
+  if (isUnd) {
+    strcpy(workingTag, "en");
+    strncat(&workingTag[2], &tag[3], capacity - 3);
+  } else {
+    strncpy(workingTag, tag, capacity - 1);
+  }
+  workingTag[capacity - 1] = 0;
+
+  char localeId[capacity];
+  uloc_forLanguageTag(workingTag, localeId, capacity, NULL, &status);
+  if (U_FAILURE(status)) {
+    g_error("%s: uloc_forLanguageTag returned %0x", __FUNCTION__, status);
+    return -1;
+  }
+
+  char minimizedLocaleId[capacity];
+  uloc_minimizeSubtags(localeId, minimizedLocaleId, capacity, &status);
+  if (U_FAILURE(status)) {
+    g_error("%s: uloc_minimizeSubtags returned %0x", __FUNCTION__, status);
+    return -1;
+  }
+
+  int taglen = uloc_toLanguageTag(minimizedLocaleId, minimizedTag, tagCapacity, FALSE, &status);
+  if (U_FAILURE(status)) {
+    g_error("%s: uloc_toLanguageTag returned %0x", __FUNCTION__, status);
+    return -1;
+  }
+
+  if (isUnd) {
+    // Replace 'en' with 'und' again
+    strncpy(workingTag, &minimizedTag[2], capacity - 1);
+    workingTag[capacity - 1] = 0;
+    strcpy(minimizedTag, "und");
+    strncat(minimizedTag, workingTag, tagCapacity - 4);
+    minimizedTag[tagCapacity - 1] = 0;
+    taglen = strlen(minimizedTag);
+  }
+  return taglen;
+}
+
+/// Extract the language code from the BCP-47 `tag`
+///
+/// @param tag        The BCP-47 tag
+/// @param lang_code  Caller-provided character array that will receive
+///                   the language code extracted from `tag`
+/// @param capacity   Array size of `lang_code`
+/// @return TRUE if successful, otherwise FALSE
+int bcp47_get_language_code(const char* tag, char* lang_code, int capacity) {
+  UErrorCode status = U_ZERO_ERROR;
+  if (!tag || strlen(tag) == 0) {
+    strncpy(lang_code, "", capacity);
+    return FALSE;
+  }
+
+  // ICU 70 doesn't properly treat `und`
+  if (strncmp(tag, "und", 3) == 0) {
+    strncpy(lang_code, "und", capacity);
+    return TRUE;
+  }
+
+  uloc_getLanguage(tag, lang_code, capacity, &status);
+  if (U_FAILURE(status)) {
+    g_error("%s: uloc_getLanguage returned %0x", __FUNCTION__, status);
+    return FALSE;
+  }
+
+  return TRUE;
+}
diff --git a/linux/ibus-keyman/src/bcp47util.h b/linux/ibus-keyman/src/bcp47util.h
new file mode 100644
index 0000000000..8a3a057f85
--- /dev/null
+++ b/linux/ibus-keyman/src/bcp47util.h
@@ -0,0 +1,7 @@
+#ifndef __BCP47UTIL_H__
+#define __BCP47UTIL_H__
+
+int bcp47_minimize(const char* tag, char* minimizedTag, int capacity);
+int bcp47_get_language_code(const char* tag, char* lang_code, int capacity);
+
+#endif // __BCP47UTIL_H__
diff --git a/linux/ibus-keyman/src/keymanutil.c b/linux/ibus-keyman/src/keymanutil.c
index 4efd941bec..6726aee4e3 100644
--- a/linux/ibus-keyman/src/keymanutil.c
+++ b/linux/ibus-keyman/src/keymanutil.c
@@ -57,6 +57,7 @@
 #include <string.h>
 #include <keyman/keyboardprocessor.h>
 
+#include "bcp47util.h"
 #include "keymanutil.h"
 #include "kmpdetails.h"
 #include "keyman-version.h"
@@ -163,8 +164,6 @@ ibus_keyman_add_engines(GList * engines, GList * kmpdir_list)
         get_kmp_details(kmp_dir, details);
 
         for (k=details->keyboards; k != NULL; k = k->next) {
-            gchar *lang=NULL;
-            gchar *name_with_lang = NULL;
             kmp_keyboard *keyboard = (kmp_keyboard *) k->data;
             gboolean alreadyexists = FALSE;
 
@@ -191,35 +190,50 @@ ibus_keyman_add_engines(GList * engines, GList * kmpdir_list)
                     for (l=keyboard->languages; l != NULL; l = l->next) {
                         kmp_language *language = (kmp_language *) l->data;
                         if (language->id != NULL) {
-                            gchar **tagparts = g_strsplit(language->id, "-", 2);
-                            lang = g_strdup(tagparts[0]);
-                            g_strfreev(tagparts);
-                            // If ibus doesn't know about the language then append the
-                            // language name to the keyboard name
-                            if (language->name != NULL) {
-                                if (g_strcmp0(ibus_get_untranslated_language_name (lang), "Other") == 0) {
-                                    name_with_lang = g_strjoin(" - ", keyboard->name, language->name, NULL);
-                                }
+                          int capacity          = 255;
+                          gchar *name_with_lang = NULL;
+                          gchar *minimized_tag  = g_new0(gchar, capacity);
+                          int result = bcp47_minimize(language->id, minimized_tag, capacity);
+                          if (result < 0) {
+                            g_strlcpy(minimized_tag, language->id, capacity);
+                          }
+
+                          gchar *lang_code = g_new0(gchar, capacity);
+                          if (!bcp47_get_language_code(minimized_tag, lang_code, capacity)) {
+                            g_strlcpy(lang_code, minimized_tag, capacity);
+                          }
+
+                          // If ibus doesn't know about the language then append the
+                          // language name to the keyboard name
+                          if (language->name != NULL) {
+                            gchar *ibus_lang = ibus_get_untranslated_language_name(lang_code);
+                            g_debug("%s: untranslated ibus language for %s: %s", __FUNCTION__, minimized_tag, ibus_lang);
+                            if (g_strcmp0(ibus_lang, "Other") == 0) {
+                              name_with_lang = g_strjoin(" - ", keyboard->name, language->name, NULL);
                             }
+                            g_free(ibus_lang);
+                          }
 
-                            gchar *id_with_lang = g_strjoin(":", language->id, abs_kmx, NULL);
+                          gchar *id_with_lang = g_strjoin(":", minimized_tag, abs_kmx, NULL);
 
-                            g_message("adding engine %s", id_with_lang);
-                            engines = g_list_append (engines,
-                                ibus_keyman_engine_desc_new (id_with_lang, // lang:kmx full path
-                                        name_with_lang ? name_with_lang : keyboard->name, // longname
-                                        kbd_details->description, // description
-                                        details->info.copyright, // copyright if available
-                                        lang, // language, most are ignored by ibus except major languages
-                                        kbd_details->license, // license
-                                        details->info.author_desc, // author name only, not email
-                                        keyman_get_icon_file(abs_kmx), // icon full path
-                                        "us", // layout defaulting to us (en-US)
-                                        keyboard->version));
-                            g_free(lang);
-                            g_free(id_with_lang);
-                            g_free(name_with_lang);
-                            name_with_lang = NULL;
+                          g_message("adding engine %s", id_with_lang);
+                          engines = g_list_append(
+                              engines,
+                              ibus_keyman_engine_desc_new(
+                                  id_with_lang,                                      // lang:kmx full path
+                                  name_with_lang ? name_with_lang : keyboard->name,  // longname
+                                  kbd_details->description,                          // description
+                                  details->info.copyright,                           // copyright if available
+                                  lang_code,                      // language, most are ignored by ibus except major languages
+                                  kbd_details->license,           // license
+                                  details->info.author_desc,      // author name only, not email
+                                  keyman_get_icon_file(abs_kmx),  // icon full path
+                                  "us",                           // layout defaulting to us (en-US)
+                                  keyboard->version));
+                          g_free(lang_code);
+                          g_free(minimized_tag);
+                          g_free(id_with_lang);
+                          g_free(name_with_lang);
                         }
                     }
                 }
@@ -230,7 +244,7 @@ ibus_keyman_add_engines(GList * engines, GList * kmpdir_list)
                                 keyboard->name, // longname
                                 kbd_details->description, // description
                                 details->info.copyright, // copyright if available
-                                lang, // language, most are ignored by ibus except major languages
+                                NULL, // language, most are ignored by ibus except major languages
                                 kbd_details->license, // license
                                 details->info.author_desc, // author name only, not email
                                 keyman_get_icon_file(abs_kmx), // icon full path
diff --git a/linux/ibus-keyman/src/meson.build b/linux/ibus-keyman/src/meson.build
index 2e83a65106..f7fe8ada1d 100644
--- a/linux/ibus-keyman/src/meson.build
+++ b/linux/ibus-keyman/src/meson.build
@@ -1,6 +1,7 @@
 util_files = files(
   'keymanutil.c',
   'kmpdetails.c',
+  'bcp47util.c',
 )
 
 engine_files = files(
@@ -16,7 +17,7 @@ include_dirs = [
   include_directories(meson.current_build_dir() / '..'),
 ]
 
-deps = [ibus, gtk, json_glib, kmnkbp_lib, systemd]
+deps = [gtk, ibus, icu, json_glib, kmnkbp_lib, systemd]
 
 prefix = get_option('prefix')
 cfg = configuration_data()
diff --git a/linux/ibus-keyman/src/test/bcp47util_tests.c b/linux/ibus-keyman/src/test/bcp47util_tests.c
new file mode 100644
index 0000000000..a82d30476f
--- /dev/null
+++ b/linux/ibus-keyman/src/test/bcp47util_tests.c
@@ -0,0 +1,100 @@
+#include <glib-object.h>
+#include <glib.h>
+#include <gtk/gtk.h>
+#include "bcp47util.h"
+
+typedef struct {
+} Bcp47UtilFixture;
+
+typedef struct {
+  const char *tag;
+  const char *expected;
+  int expectedResult;
+} TestData;
+
+static void
+test_bcp47_minimize(Bcp47UtilFixture *fixture, gconstpointer user_data) {
+  TestData *testData = (TestData *)user_data;
+  int capacity        = 255;
+  char minimizedTag[capacity];
+  int result = bcp47_minimize(testData->tag, minimizedTag, capacity);
+  g_assert_cmpint(result, ==, testData->expectedResult);
+  g_assert_cmpstr(minimizedTag, ==, testData->expected);
+}
+
+static void
+test_bcp47_get_language_code(Bcp47UtilFixture *fixture, gconstpointer user_data) {
+  TestData *testData = (TestData *)user_data;
+  int capacity       = 255;
+  char lang_code[capacity];
+  int result = bcp47_get_language_code(testData->tag, lang_code, capacity);
+  g_assert_cmpint(result, ==, testData->expectedResult);
+  g_assert_cmpstr(lang_code, ==, testData->expected);
+}
+
+int
+main(int argc, char *argv[]) {
+  gtk_init(&argc, &argv);
+  g_test_init(&argc, &argv, NULL);
+  g_test_set_nonfatal_assertions();
+
+  TestData testData1 = {NULL, "", -1};
+  g_test_add("/bcp47util/minimize/NULL", Bcp47UtilFixture, &testData1, NULL, test_bcp47_minimize, NULL);
+
+  TestData testData2 = {"", "", -1};
+  g_test_add("/bcp47util/minimize/EmptyString", Bcp47UtilFixture, &testData2, NULL, test_bcp47_minimize, NULL);
+
+  TestData testData3 = { "fuf", "fuf", 3 };
+  g_test_add("/bcp47util/minimize/fuf", Bcp47UtilFixture, &testData3, NULL, test_bcp47_minimize, NULL);
+
+  TestData testData4 = {"fuf-Latn", "fuf", 3};
+  g_test_add("/bcp47util/minimize/fuf-Latn", Bcp47UtilFixture, &testData4, NULL, test_bcp47_minimize, NULL);
+
+  TestData testData5 = {"fuf-Arab", "fuf-Arab", 8};
+  g_test_add("/bcp47util/minimize/fuf-Arab", Bcp47UtilFixture, &testData5, NULL, test_bcp47_minimize, NULL);
+
+  TestData testData6 = {"fuf-Adlm-ML", "fuf-Adlm-ML", 11};
+  g_test_add("/bcp47util/minimize/fuf-Adlm-ML", Bcp47UtilFixture, &testData6, NULL, test_bcp47_minimize, NULL);
+
+  TestData testData7 = {"und", "und", 3};
+  g_test_add("/bcp47util/minimize/und", Bcp47UtilFixture, &testData7, NULL, test_bcp47_minimize, NULL);
+
+  TestData testData8 = {"und-Latn", "und-Latn", 8};
+  g_test_add("/bcp47util/minimize/und-Latn", Bcp47UtilFixture, &testData8, NULL, test_bcp47_minimize, NULL);
+
+  TestData testData9 = {"und-fonipa", "und-fonipa", 10};
+  g_test_add("/bcp47util/minimize/und-fonipa", Bcp47UtilFixture, &testData9, NULL, test_bcp47_minimize, NULL);
+
+  TestData testData10 = {"und-Latn-fonipa", "und-fonipa", 10};
+  g_test_add("/bcp47util/minimize/und-Latn-fonipa", Bcp47UtilFixture, &testData10, NULL, test_bcp47_minimize, NULL);
+
+  TestData testData11 = {"mul", "mul", 3};
+  g_test_add("/bcp47util/minimize/mul", Bcp47UtilFixture, &testData11, NULL, test_bcp47_minimize, NULL);
+
+  // bcp47_get_language_code tests
+  TestData testData21 = {NULL, "", FALSE};
+  g_test_add("/bcp47util/langcode/NULL", Bcp47UtilFixture, &testData21, NULL, test_bcp47_get_language_code, NULL);
+
+  TestData testData22 = {"", "", FALSE};
+  g_test_add("/bcp47util/langcode/EmptyString", Bcp47UtilFixture, &testData22, NULL, test_bcp47_get_language_code, NULL);
+
+  TestData testData23 = {"fuf", "fuf", TRUE};
+  g_test_add("/bcp47util/langcode/fuf", Bcp47UtilFixture, &testData23, NULL, test_bcp47_get_language_code, NULL);
+
+  TestData testData24 = {"fuf-Latn", "fuf", TRUE};
+  g_test_add("/bcp47util/langcode/fuf-Latn", Bcp47UtilFixture, &testData24, NULL, test_bcp47_get_language_code, NULL);
+
+  TestData testData25 = {"fuf-Arab", "fuf", TRUE};
+  g_test_add("/bcp47util/langcode/fuf-Arab", Bcp47UtilFixture, &testData25, NULL, test_bcp47_get_language_code, NULL);
+
+  TestData testData26 = {"fuf-Adlm-ML", "fuf", TRUE};
+  g_test_add("/bcp47util/langcode/fuf-Adlm-ML", Bcp47UtilFixture, &testData26, NULL, test_bcp47_get_language_code, NULL);
+
+  TestData testData27 = {"und", "und", TRUE};
+  g_test_add("/bcp47util/langcode/und", Bcp47UtilFixture, &testData27, NULL, test_bcp47_get_language_code, NULL);
+
+  TestData testData28 = {"und-Latn", "und", TRUE};
+  g_test_add("/bcp47util/langcode/und-Latn", Bcp47UtilFixture, &testData28, NULL, test_bcp47_get_language_code, NULL);
+
+  return g_test_run();
+}
diff --git a/linux/ibus-keyman/src/test/meson.build b/linux/ibus-keyman/src/test/meson.build
index 7e5cd21dbe..1ad841c6d6 100644
--- a/linux/ibus-keyman/src/test/meson.build
+++ b/linux/ibus-keyman/src/test/meson.build
@@ -3,11 +3,13 @@ keymanutil_sources = [
   util_files,
 ]
 
-keymanutil_deps = [ibus, gtk, json_glib, kmnkbp_lib]
+keymanutil_deps = [gtk, ibus, icu, json_glib, kmnkbp_lib]
 
 test_env = [
   'G_TEST_SRCDIR=' + meson.current_source_dir(),
   'G_TEST_BUILDDIR=' + meson.current_build_dir(),
+  'TOP_SRCDIR=' + meson.global_source_root(),
+  'TOP_BINDIR=' + meson.build_root(),
 ]
 
 test_include_dirs = [
@@ -16,47 +18,102 @@ test_include_dirs = [
   include_directories(meson.current_build_dir() / '..'),
 ]
 
-executable(
+env_file = '/tmp/env-src-test.txt'
+pid_file = '/tmp/ibus-keyman-src-test-pids'
+
+setup_src_test_tests = find_program('setup-tests.sh', dirs: [meson.current_source_dir()])
+teardown_tests = find_program('teardown-tests.sh', dirs: [meson.current_source_dir() / '../../tests/scripts'])
+run_src_test = find_program('run-single-test.sh', dirs: [meson.current_source_dir()])
+
+keymanutil_tests = executable(
   'keymanutil-tests',
   sources: keymanutil_sources,
   dependencies: keymanutil_deps,
   include_directories : test_include_dirs
 )
 
+print_kmpdetails_test = executable(
+  'print_kmpdetails',
+  sources: [
+    'print_kmpdetails.c',
+    '../kmpdetails.c'
+  ],
+  dependencies: [ json_glib ],
+  include_directories: test_include_dirs
+)
+
+print_kmp_test = executable(
+  'print_kmp',
+  sources: [
+    'print_kmp.c',
+  ],
+  dependencies: [ json_glib ],
+  include_directories: test_include_dirs
+)
+
+bcp47_util_tests = executable(
+  'bcp47-util-tests',
+  sources: [
+    'bcp47util_tests.c',
+    '../bcp47util.c'
+  ],
+  dependencies: [ gtk, icu ],
+  include_directories: test_include_dirs
+)
+
+test(
+  'setup-src-test',
+  setup_src_test_tests,
+  args: ['--x11', env_file, pid_file],
+  env: test_env,
+  priority: -1,
+  is_parallel: false,
+  protocol: 'exitcode'
+)
+
+test(
+  'teardown-src-test',
+  teardown_tests,
+  args: [pid_file],
+  priority: -9,
+  is_parallel: false,
+  protocol: 'exitcode'
+)
+
 test(
   'keymanutil-tests',
-  find_program('run-tests.sh'),
+  run_src_test,
+  args: [ '--tap', '-k', '--env', env_file, '--', keymanutil_tests],
   env: test_env,
+  priority: -2,
+  is_parallel: false,
   protocol: 'tap',
 )
 
 test(
   'print-kmpdetails-test',
-  executable(
-    'print_kmpdetails',
-    sources: [
-      'print_kmpdetails.c',
-      '../kmpdetails.c'
-    ],
-    dependencies: [ json_glib ],
-    include_directories: test_include_dirs
-  ),
-  args: [ meson.current_source_dir() ],
+  run_src_test,
+  args: [ '--', print_kmpdetails_test, meson.current_source_dir() ],
   env: test_env,
+  priority: -2,
   protocol: 'exitcode',
 )
 
 test(
   'print-kmp-test',
-  executable(
-    'print_kmp',
-    sources: [
-      'print_kmp.c',
-    ],
-    dependencies: [ json_glib ],
-    include_directories: test_include_dirs
-  ),
-  args: [ meson.current_source_dir() / 'kmp.json' ],
+  run_src_test,
+  args: [ '--', print_kmp_test, meson.current_source_dir() / 'kmp.json' ],
   env: test_env,
+  priority: -2,
   protocol: 'exitcode',
 )
+
+test(
+  'bcp47-util-tests',
+  run_src_test,
+  args: [ '--tap', '-k', '--env', env_file, '--', bcp47_util_tests ],
+  env: test_env,
+  priority: -2,
+  is_parallel: false,
+  protocol: 'tap',
+)
diff --git a/linux/ibus-keyman/src/test/run-single-test.sh b/linux/ibus-keyman/src/test/run-single-test.sh
new file mode 100755
index 0000000000..1ad8714449
--- /dev/null
+++ b/linux/ibus-keyman/src/test/run-single-test.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+function help() {
+  echo "Usage:"
+  echo "  $0 [--env <envfile>] [-k] [--tap] [--] TESTFILE [TESTARGS]"
+  echo
+  echo "Arguments:"
+  echo "  --help, -h, -?          Display this help"
+  echo "  --verbose, -v           Run tests verbosely"
+  echo "  --debug                 debug test logging output"
+  echo "  -k                      passed to GLib testing framework"
+  echo "  --tap                   output in TAP format. Passed to GLib testing framework"
+  echo "  --env <envfile>         Name of the file containing environment variables to use"
+  exit 0
+}
+
+function run_tests() {
+  # Output these lines to stderr - the first line on stdout has to be the TAP version number
+  # which running ${TESTFILE} outputs
+  echo "# NOTE: When the tests fail check /tmp/ibus-engine-keyman.log and /tmp/ibus-daemon.log!" >&2
+  echo "" >&2
+
+  echo "# Starting tests..." >&2
+
+  # Note: -k and --tap are consumed by the GLib testing framework
+  # shellcheck disable=SC2086
+  "${TESTFILE}" ${ARG_K-} ${ARG_TAP-} ${ARG_VERBOSE-} ${ARG_DEBUG-} "$@"
+  echo "# Finished tests."
+}
+
+while (( $# )); do
+  case $1 in
+    --help|-h|-\?) help ;;
+    -k) ARG_K=$1 ;;
+    --tap) ARG_TAP=$1 ;;
+    --verbose|-v) ARG_VERBOSE=--verbose;;
+    --debug) ARG_DEBUG=--debug-log;;
+    --env) shift ; ARG_ENV=$1 ;;
+    --) shift ; TESTFILE=$1; shift ; break ;;
+    *) echo "Error: Unexpected argument \"$1\". Exiting." ; exit 4 ;;
+  esac
+  shift || (echo "Error: The last argument is missing a value. Exiting."; false) || exit 5
+done
+
+# shellcheck source=/dev/null
+. "$ARG_ENV"
+
+run_tests "$@"
diff --git a/linux/ibus-keyman/src/test/run-tests.sh b/linux/ibus-keyman/src/test/run-tests.sh
index 44a397582b..efeb747ddc 100755
--- a/linux/ibus-keyman/src/test/run-tests.sh
+++ b/linux/ibus-keyman/src/test/run-tests.sh
@@ -7,8 +7,8 @@ if [ -v KEYMAN_PKG_BUILD ]; then
   # During package builds we skip these tests - they often fail, e.g.
   # during Debian reproducibility testing with an error like
   # "cannot open display: :32"
-  echo "1..1"
-  echo "ok 1 # SKIP on package build"
+  echo "TAP version 14"
+  echo "1..0  # SKIP on package build"
   exit 0
 fi
 
@@ -58,4 +58,4 @@ glib-compile-schemas "$SCHEMA_DIR"
 
 export GSETTINGS_BACKEND=memory
 
-${G_TEST_BUILDDIR:-.}/keymanutil-tests "$@"
+"${G_TEST_BUILDDIR:-.}/keymanutil-tests" "$@"
diff --git a/linux/ibus-keyman/src/test/setup-tests.sh b/linux/ibus-keyman/src/test/setup-tests.sh
new file mode 100755
index 0000000000..736650270d
--- /dev/null
+++ b/linux/ibus-keyman/src/test/setup-tests.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -eu
+
+. "$(dirname "$0")/../../tests/scripts/test-helper.inc.sh"
+
+setup_display_server_only "$1" "$2" "$3"
diff --git a/linux/ibus-keyman/tests/meson.build b/linux/ibus-keyman/tests/meson.build
index 130e3b8bce..8b7b85df4a 100644
--- a/linux/ibus-keyman/tests/meson.build
+++ b/linux/ibus-keyman/tests/meson.build
@@ -10,7 +10,7 @@ kmnkbp_tests_lib = cc.find_library(
   dirs: [ core_dir / 'build/arch' / get_option('buildtype') / 'tests/kmx_test_source' ]
 )
 
-test_deps = [ibus, gtk, json_glib, kmnkbp_lib, kmnkbp_tests_lib, systemd]
+test_deps = [gtk, ibus, icu, json_glib, kmnkbp_lib, kmnkbp_tests_lib, systemd]
 dbus_deps = [gtk, systemd]
 
 test_env = [
diff --git a/linux/ibus-keyman/tests/scripts/test-helper.inc.sh b/linux/ibus-keyman/tests/scripts/test-helper.inc.sh
index 2a3735d708..5fdb5d85ca 100755
--- a/linux/ibus-keyman/tests/scripts/test-helper.inc.sh
+++ b/linux/ibus-keyman/tests/scripts/test-helper.inc.sh
@@ -10,7 +10,7 @@ function can_run_wayland() {
   fi
 }
 
-function generate_kmpjson() {
+function _generate_kmpjson() {
   local TESTDIR
   TESTDIR="$1"
   pushd "$TESTDIR" > /dev/null || exit
@@ -78,7 +78,7 @@ EOF
   popd > /dev/null || exit
 }
 
-function link_test_keyboards() {
+function _link_test_keyboards() {
   KMX_TEST_DIR=$1
   TESTDIR=$2
   TESTBASEDIR=$3
@@ -93,22 +93,23 @@ function link_test_keyboards() {
   fi
 }
 
-function setup() {
-  local DISPLAY_SERVER ENV_FILE PID_FILE TOP_SRCDIR TOP_BINDIR TESTBASEDIR TESTDIR
-  DISPLAY_SERVER=$1
-  ENV_FILE=$2
-  PID_FILE=$3
+function _setup_init() {
+  local ENV_FILE PID_FILE
+  ENV_FILE=$1
+  PID_FILE=$2
 
-  TOP_SRCDIR=${G_TEST_SRCDIR:-$(realpath "$(dirname "$0")/..")}/..
-  TOP_BINDIR=${G_TEST_BUILDDIR:-$(realpath "$(dirname "$0/..")")}/..
-  TESTBASEDIR=${XDG_DATA_HOME:-$HOME/.local/share}/keyman
-  TESTDIR=${TESTBASEDIR}/test_kmx
+  if [ -z "${TOP_SRCDIR:-}" ]; then
+    TOP_SRCDIR=${G_TEST_SRCDIR:-$(realpath "$(dirname "$0")/..")}/..
+  fi
+  if [ -z "${TOP_BINDIR:-}" ]; then
+    TOP_BINDIR=${G_TEST_BUILDDIR:-$(realpath "$(dirname "$0/..")")}/..
+  fi
 
   echo > "$ENV_FILE"
 
   if [ -f "$PID_FILE" ]; then
     # kill previous instances
-    "$(dirname "$0")"/teardown-tests.sh "$PID_FILE"
+    "$(dirname "$0")"/teardown-tests.sh "$PID_FILE" || true
   fi
 
   echo > "$PID_FILE"
@@ -127,18 +128,33 @@ function setup() {
     exit 2
   fi
 
-  link_test_keyboards "${TOP_SRCDIR}/../../common/test/keyboards/baseline" "$TESTDIR" "$TESTBASEDIR"
+  export LD_LIBRARY_PATH=${COMMON_ARCH_DIR}/src:${LD_LIBRARY_PATH-}
+  echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> "$ENV_FILE"
+}
 
-  generate_kmpjson "$TESTDIR"
+function _setup_test_dbus_server() {
+  local ENV_FILE PID_FILE
+  ENV_FILE=$1
+  PID_FILE=$2
 
-  # Start test dbus server
+  # Start test dbus server. This will create `/tmp/km-test-server.env`.
   "${TOP_BINDIR}/tests/km-dbus-test-server" &> /dev/null &
   sleep 1
-  source /tmp/km-test-server.env
+
   cat /tmp/km-test-server.env >> "$ENV_FILE"
   cat /tmp/km-test-server.env >> "$PID_FILE"
   echo "${TOP_BINDIR}/tests/stop-test-server" >> "$PID_FILE"
 
+  source /tmp/km-test-server.env
+  echo "# DBUS_SESSION_BUS_ADDRESS=$DBUS_SESSION_BUS_ADDRESS"
+}
+
+function _setup_display_server() {
+  local DISPLAY_SERVER ENV_FILE PID_FILE
+  ENV_FILE=$1
+  PID_FILE=$2
+  DISPLAY_SERVER=$3
+
   if [ "$DISPLAY_SERVER" == "wayland" ]; then
     if ! can_run_wayland; then
       # support for --headless got added in mutter 40.x
@@ -173,6 +189,12 @@ function setup() {
     export DISPLAY=:32
     echo "export DISPLAY=\"$DISPLAY\"" >> "$ENV_FILE"
   fi
+}
+
+function _setup_schema_and_gsettings() {
+  local ENV_FILE PID_FILE
+  ENV_FILE=$1
+  PID_FILE=$2
 
   # Install schema to temporary directory. This removes the build dependency on the keyman package.
   SCHEMA_DIR=$TEMP_DATA_DIR/glib-2.0/schemas
@@ -186,9 +208,6 @@ function setup() {
   cp "${TOP_SRCDIR}"/../keyman-config/resources/com.keyman.gschema.xml "$SCHEMA_DIR"/
   glib-compile-schemas "$SCHEMA_DIR"
 
-  export LD_LIBRARY_PATH=${COMMON_ARCH_DIR}/src:${LD_LIBRARY_PATH-}
-  echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> "$ENV_FILE"
-
   # Ubuntu 18.04 Bionic doesn't have ibus-memconf, and glib is not compiled with the keyfile
   # backend enabled, so we just use the default backend. Otherwise we use the keyfile
   # store which interferes less when running on a dev machine.
@@ -197,6 +216,12 @@ function setup() {
     echo "export GSETTINGS_BACKEND=\"$GSETTINGS_BACKEND\"" >> "$ENV_FILE"
     IBUS_CONFIG=--config=/usr/libexec/ibus-memconf
   fi
+}
+
+function _setup_ibus() {
+  local ENV_FILE PID_FILE
+  ENV_FILE=$1
+  PID_FILE=$2
 
   #shellcheck disable=SC2086
   ibus-daemon ${ARG_VERBOSE-} --daemonize --panel=disable --address=unix:abstract="${TEMP_DATA_DIR}/test-ibus" ${IBUS_CONFIG-} &> /tmp/ibus-daemon.log
@@ -208,11 +233,42 @@ function setup() {
 
   echo "export IBUS_ADDRESS=\"$IBUS_ADDRESS\"" >> "$ENV_FILE"
 
-  echo "# DBUS_SESSION_BUS_ADDRESS=$DBUS_SESSION_BUS_ADDRESS"
   #shellcheck disable=SC2086
   "${TOP_BINDIR}/src/ibus-engine-keyman" --testing ${ARG_VERBOSE-} &> /tmp/ibus-engine-keyman.log &
   echo "kill -9 $! || true" >> "$PID_FILE"
   sleep 1s
+
+}
+function setup() {
+  local DISPLAY_SERVER ENV_FILE PID_FILE TESTBASEDIR TESTDIR
+  DISPLAY_SERVER=$1
+  ENV_FILE=$2
+  PID_FILE=$3
+
+  _setup_init "${ENV_FILE}" "${PID_FILE}"
+
+  TESTBASEDIR=${XDG_DATA_HOME:-$HOME/.local/share}/keyman
+  TESTDIR=${TESTBASEDIR}/test_kmx
+
+  _link_test_keyboards "${TOP_SRCDIR}/../../common/test/keyboards/baseline" "$TESTDIR" "$TESTBASEDIR"
+
+  _generate_kmpjson "$TESTDIR"
+
+  _setup_test_dbus_server "${ENV_FILE}" "${PID_FILE}"
+  _setup_display_server "${ENV_FILE}" "${PID_FILE}" "${DISPLAY_SERVER}"
+  _setup_schema_and_gsettings "${ENV_FILE}" "${PID_FILE}"
+  _setup_ibus "${ENV_FILE}" "${PID_FILE}"
+}
+
+function setup_display_server_only() {
+  local DISPLAY_SERVER ENV_FILE PID_FILE TESTBASEDIR TESTDIR
+  DISPLAY_SERVER=$1
+  ENV_FILE=$2
+  PID_FILE=$3
+
+  _setup_init "${ENV_FILE}" "${PID_FILE}"
+  _setup_display_server "${ENV_FILE}" "${PID_FILE}" "${DISPLAY_SERVER}"
+  _setup_schema_and_gsettings "${ENV_FILE}" "${PID_FILE}"
 }
 
 function cleanup() {

From 6622c70fd20650006ab1baa8320e53a402ec06d3 Mon Sep 17 00:00:00 2001
From: Eberhard Beilharz <eb1@sil.org>
Date: Wed, 19 Jul 2023 16:22:31 +0200
Subject: [PATCH 06/83] chore(linux): Fix running tests with Wayland

Because we passed in a wrong parameter (`--wayland`) but checked for
just `wayland` we never run the tests with Wayland. This change
rectifies this.
---
 linux/ibus-keyman/tests/meson.build           | 88 ++++++++++---------
 .../tests/scripts/run-single-test.sh          | 13 +--
 linux/ibus-keyman/tests/scripts/run-tests.sh  |  3 +-
 .../ibus-keyman/tests/scripts/setup-tests.sh  |  6 ++
 .../tests/scripts/teardown-tests.sh           |  6 ++
 .../tests/scripts/test-helper.inc.sh          |  2 +-
 6 files changed, 70 insertions(+), 48 deletions(-)

diff --git a/linux/ibus-keyman/tests/meson.build b/linux/ibus-keyman/tests/meson.build
index 8b7b85df4a..20d82e7e2c 100644
--- a/linux/ibus-keyman/tests/meson.build
+++ b/linux/ibus-keyman/tests/meson.build
@@ -57,6 +57,10 @@ teardown_tests = find_program('teardown-tests.sh', dirs: [meson.current_source_d
 run_test = find_program('run-single-test.sh', dirs: [meson.current_source_dir() / 'scripts'])
 find_tests = find_program('find-tests.sh', dirs: [meson.current_source_dir() / 'scripts'])
 
+# Mutter 40.x added the --headless option wich we need in order to be able to run the Wayland tests
+mutter = find_program('mutter', required: false, version: '>=40')
+can_build_wayland = mutter.found()
+
 test(
   'setup-x11',
   setup_tests,
@@ -67,16 +71,6 @@ test(
   protocol: 'exitcode'
 )
 
-test(
-  'setup-wayland',
-  setup_tests,
-  args: ['--wayland', env_file, pid_file],
-  env: test_env,
-  priority: -20,
-  is_parallel: false,
-  protocol: 'exitcode'
-)
-
 test(
   'teardown-x11',
   teardown_tests,
@@ -86,14 +80,26 @@ test(
   protocol: 'exitcode'
 )
 
-test(
-  'teardown-wayland',
-  teardown_tests,
-  args: [pid_file],
-  priority: -29,
-  is_parallel: false,
-  protocol: 'exitcode'
-)
+if can_build_wayland
+  test(
+    'setup-wayland',
+    setup_tests,
+    args: ['--wayland', env_file, pid_file],
+    env: test_env,
+    priority: -20,
+    is_parallel: false,
+    protocol: 'exitcode'
+  )
+
+  test(
+    'teardown-wayland',
+    teardown_tests,
+    args: [pid_file],
+    priority: -29,
+    is_parallel: false,
+    protocol: 'exitcode'
+  )
+endif
 
 kmxtest_files = run_command(
   find_tests,
@@ -130,26 +136,28 @@ foreach kmx: kmxtest_files
     timeout: 120,
     protocol: 'tap',
   )
-  test(
-    'Wayland-' + testname + '__surrounding-text',
-    run_test,
-    args: [ '--wayland', '--surrounding-text', test_args],
-    env: test_env,
-    depends: [test_exe],
-    priority: -21,
-    is_parallel: false,
-    timeout: 120,
-    protocol: 'tap',
-  )
-  test(
-    'Wayland-' + testname + '__no-surrounding-text',
-    run_test,
-    args: [ '--wayland', '--no-surrounding-text', test_args],
-    env: test_env,
-    depends: [test_exe],
-    priority: -22,
-    is_parallel: false,
-    timeout: 120,
-    protocol: 'tap',
-  )
+  if can_build_wayland
+    test(
+      'Wayland-' + testname + '__surrounding-text',
+      run_test,
+      args: [ '--wayland', '--surrounding-text', test_args],
+      env: test_env,
+      depends: [test_exe],
+      priority: -21,
+      is_parallel: false,
+      timeout: 120,
+      protocol: 'tap',
+    )
+    test(
+      'Wayland-' + testname + '__no-surrounding-text',
+      run_test,
+      args: [ '--wayland', '--no-surrounding-text', test_args],
+      env: test_env,
+      depends: [test_exe],
+      priority: -22,
+      is_parallel: false,
+      timeout: 120,
+      protocol: 'tap',
+    )
+  endif
 endforeach
diff --git a/linux/ibus-keyman/tests/scripts/run-single-test.sh b/linux/ibus-keyman/tests/scripts/run-single-test.sh
index 19baca7dd6..b4660f8368 100755
--- a/linux/ibus-keyman/tests/scripts/run-single-test.sh
+++ b/linux/ibus-keyman/tests/scripts/run-single-test.sh
@@ -9,8 +9,8 @@ if [ -v KEYMAN_PKG_BUILD ]; then
   # ibus requires to find /var/lib/dbus/machine-id or /etc/machine-id, otherwise it fails with:
   # "Bail out! IBUS-FATAL-WARNING: Unable to load /var/lib/dbus/machine-id: Failed to open file
   # “/var/lib/dbus/machine-id”: No such file or directory"
-  echo "1..1"
-  echo "ok 1 - Integration tests # SKIP on package build"
+  echo "TAP version 14"
+  echo "1..0  # SKIP on package build"
   exit 0
 fi
 
@@ -38,10 +38,13 @@ function help() {
 }
 
 function run_tests() {
-  echo "# NOTE: When the tests fail check /tmp/ibus-engine-keyman.log and /tmp/ibus-daemon.log!"
-  echo ""
+  # Output these lines to stderr - the first line on stdout has to be the TAP version number
+  # which running ${TESTFILE} outputs
+  echo "# NOTE: When the tests fail check /tmp/ibus-engine-keyman.log and /tmp/ibus-daemon.log!" >&2
+  echo "" >&2
+
+  echo "# Starting tests..." >&2
 
-  echo "# Starting tests..."
   # Note: -k and --tap are consumed by the GLib testing framework
   # shellcheck disable=SC2086
   "${G_TEST_BUILDDIR:-.}"/ibus-keyman-tests ${ARG_K-} ${ARG_TAP-} \
diff --git a/linux/ibus-keyman/tests/scripts/run-tests.sh b/linux/ibus-keyman/tests/scripts/run-tests.sh
index 92cae307ed..ae58705f11 100755
--- a/linux/ibus-keyman/tests/scripts/run-tests.sh
+++ b/linux/ibus-keyman/tests/scripts/run-tests.sh
@@ -18,8 +18,7 @@ if [ -v KEYMAN_PKG_BUILD ]; then
   # ibus requires to find /var/lib/dbus/machine-id or /etc/machine-id, otherwise it fails with:
   # "Bail out! IBUS-FATAL-WARNING: Unable to load /var/lib/dbus/machine-id: Failed to open file
   # “/var/lib/dbus/machine-id”: No such file or directory"
-  echo "1..1"
-  echo "ok 1 - Integration tests # SKIP on package build"
+  echo "1..0 # SKIP on package build"
   exit 0
 fi
 
diff --git a/linux/ibus-keyman/tests/scripts/setup-tests.sh b/linux/ibus-keyman/tests/scripts/setup-tests.sh
index 9d591869fb..2856e5afa7 100755
--- a/linux/ibus-keyman/tests/scripts/setup-tests.sh
+++ b/linux/ibus-keyman/tests/scripts/setup-tests.sh
@@ -3,4 +3,10 @@ set -eu
 
 . "$(dirname "$0")/test-helper.inc.sh"
 
+if [ -v KEYMAN_PKG_BUILD ]; then
+  # Skip setup during package builds - can't run headless and we won't
+  # run the other tests anyway
+  exit 0
+fi
+
 setup "$1" "$2" "$3"
diff --git a/linux/ibus-keyman/tests/scripts/teardown-tests.sh b/linux/ibus-keyman/tests/scripts/teardown-tests.sh
index c3b955a13a..7e09477fda 100755
--- a/linux/ibus-keyman/tests/scripts/teardown-tests.sh
+++ b/linux/ibus-keyman/tests/scripts/teardown-tests.sh
@@ -3,4 +3,10 @@ set -eu
 
 . "$(dirname "$0")/test-helper.inc.sh"
 
+if [ -v KEYMAN_PKG_BUILD ]; then
+  # Skip setup during package builds - can't run headless and we won't
+  # run the other tests anyway
+  exit 0
+fi
+
 cleanup "$1"
diff --git a/linux/ibus-keyman/tests/scripts/test-helper.inc.sh b/linux/ibus-keyman/tests/scripts/test-helper.inc.sh
index 5fdb5d85ca..35ac9b819f 100755
--- a/linux/ibus-keyman/tests/scripts/test-helper.inc.sh
+++ b/linux/ibus-keyman/tests/scripts/test-helper.inc.sh
@@ -155,7 +155,7 @@ function _setup_display_server() {
   PID_FILE=$2
   DISPLAY_SERVER=$3
 
-  if [ "$DISPLAY_SERVER" == "wayland" ]; then
+  if [ "$DISPLAY_SERVER" == "--wayland" ]; then
     if ! can_run_wayland; then
       # support for --headless got added in mutter 40.x
       echo "ERROR: mutter doesn't support running headless. Can't run Wayland tests."

From c616be6bf15ad064b614789eceb9a9c776c78bf2 Mon Sep 17 00:00:00 2001
From: Eberhard Beilharz <eb1@sil.org>
Date: Wed, 19 Jul 2023 17:39:34 +0200
Subject: [PATCH 07/83] refactor(linux): Move common code to included file

---
 linux/ibus-keyman/tests/scripts/setup-tests.sh     | 6 +-----
 linux/ibus-keyman/tests/scripts/teardown-tests.sh  | 6 +-----
 linux/ibus-keyman/tests/scripts/test-helper.inc.sh | 8 ++++++++
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/linux/ibus-keyman/tests/scripts/setup-tests.sh b/linux/ibus-keyman/tests/scripts/setup-tests.sh
index 2856e5afa7..67bbcbb1bf 100755
--- a/linux/ibus-keyman/tests/scripts/setup-tests.sh
+++ b/linux/ibus-keyman/tests/scripts/setup-tests.sh
@@ -3,10 +3,6 @@ set -eu
 
 . "$(dirname "$0")/test-helper.inc.sh"
 
-if [ -v KEYMAN_PKG_BUILD ]; then
-  # Skip setup during package builds - can't run headless and we won't
-  # run the other tests anyway
-  exit 0
-fi
+exit_on_package_build
 
 setup "$1" "$2" "$3"
diff --git a/linux/ibus-keyman/tests/scripts/teardown-tests.sh b/linux/ibus-keyman/tests/scripts/teardown-tests.sh
index 7e09477fda..c74513f281 100755
--- a/linux/ibus-keyman/tests/scripts/teardown-tests.sh
+++ b/linux/ibus-keyman/tests/scripts/teardown-tests.sh
@@ -3,10 +3,6 @@ set -eu
 
 . "$(dirname "$0")/test-helper.inc.sh"
 
-if [ -v KEYMAN_PKG_BUILD ]; then
-  # Skip setup during package builds - can't run headless and we won't
-  # run the other tests anyway
-  exit 0
-fi
+exit_on_package_build
 
 cleanup "$1"
diff --git a/linux/ibus-keyman/tests/scripts/test-helper.inc.sh b/linux/ibus-keyman/tests/scripts/test-helper.inc.sh
index 35ac9b819f..9ea5ffe3fc 100755
--- a/linux/ibus-keyman/tests/scripts/test-helper.inc.sh
+++ b/linux/ibus-keyman/tests/scripts/test-helper.inc.sh
@@ -283,3 +283,11 @@ function cleanup() {
     echo "# Finished shutdown of processes."
   fi
 }
+
+function exit_on_package_build() {
+  if [ -v KEYMAN_PKG_BUILD ]; then
+    # Skip setup during package builds - can't run headless and we won't
+    # run the other tests anyway
+    exit 0
+  fi
+}

From 9fc62790b91897b3c6ac39b01c5518c522bb9b9e Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Wed, 19 Jul 2023 15:29:38 -0500
Subject: [PATCH 08/83] =?UTF-8?q?feat(core):=20transform/reorder=20process?=
 =?UTF-8?q?ing=20=20a=20little=20less=20broken=20=20=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

#7375
---
 core/src/ldml/ldml_transforms.cpp        | 1 +
 core/src/ldml/ldml_transforms.hpp        | 2 +-
 core/tests/unit/ldml/test_transforms.cpp | 5 +++++
 3 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/core/src/ldml/ldml_transforms.cpp b/core/src/ldml/ldml_transforms.cpp
index 7d6920fe57..8e425cb0a3 100644
--- a/core/src/ldml/ldml_transforms.cpp
+++ b/core/src/ldml/ldml_transforms.cpp
@@ -430,6 +430,7 @@ transforms::apply(const std::u32string &input, std::u32string &output) {
         output.append(str2);
         updatedInput.resize(0);
         updatedInput.append(str2);
+        matched = output.length();
       }
     }
     // else: continue to next group
diff --git a/core/src/ldml/ldml_transforms.hpp b/core/src/ldml/ldml_transforms.hpp
index bd278d059a..cf5f993493 100644
--- a/core/src/ldml/ldml_transforms.hpp
+++ b/core/src/ldml/ldml_transforms.hpp
@@ -24,7 +24,7 @@ using km::kbp::kmx::USet;
  * Type of a group
  */
 enum any_group_type {
-  transform = LDML_TRAN_GROUP_TYPE_REORDER,
+  transform = LDML_TRAN_GROUP_TYPE_TRANSFORM,
   reorder   = LDML_TRAN_GROUP_TYPE_REORDER,
 };
 
diff --git a/core/tests/unit/ldml/test_transforms.cpp b/core/tests/unit/ldml/test_transforms.cpp
index a28ca46ef3..03c2772292 100644
--- a/core/tests/unit/ldml/test_transforms.cpp
+++ b/core/tests/unit/ldml/test_transforms.cpp
@@ -339,8 +339,13 @@ test_reorder_standalone() {
       // try all-at-once
       {
         std::u32string text = roast;
+        std::cout << " Starting: " << roast << std::endl;
         if (!tr.apply(text)) {
           std::cout << " (did not apply)" << std::endl;
+        } else if (text == roast) {
+          std::cout << " (suboptimal: apply returned true but made no change)" << std::endl;
+        } else {
+          std::cout << " changed to " << text;
         }
         zassert_string_equal(text, expect);
         std::cout << " matched (converting all at once)!" << std::endl;

From 7f07f951ae9d6ee9310d3ed43db4007b88e5908a Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Wed, 19 Jul 2023 18:07:19 -0500
Subject: [PATCH 09/83] =?UTF-8?q?feat(core):=20transform/reorder=20passing?=
 =?UTF-8?q?=20tests!=20=20=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- reorder now handles runs, where a run starts at primary=0
- reorder still returns as if it had matched/altered the entire string

#7375
---
 core/src/ldml/ldml_transforms.cpp             | 54 +++++++++----------
 core/src/ldml/ldml_transforms.hpp             |  1 +
 .../keyboards/k_200_reorder_nod_Lana-test.xml | 20 +++++--
 .../ldml/keyboards/k_200_reorder_nod_Lana.xml |  6 ++-
 core/tests/unit/ldml/test_transforms.cpp      | 41 ++++++++++++--
 5 files changed, 85 insertions(+), 37 deletions(-)

diff --git a/core/src/ldml/ldml_transforms.cpp b/core/src/ldml/ldml_transforms.cpp
index 8e425cb0a3..9320a10dd8 100644
--- a/core/src/ldml/ldml_transforms.cpp
+++ b/core/src/ldml/ldml_transforms.cpp
@@ -96,6 +96,11 @@ reorder_sort_key::operator<(const reorder_sort_key &other) const {
   return (compare(other) < 0);
 }
 
+bool
+reorder_sort_key::operator>(const reorder_sort_key &other) const {
+  return (compare(other) > 0);
+}
+
 std::deque<reorder_sort_key>
 reorder_sort_key::from(const std::u32string &str) {
   // construct a 'baseline' sort key, that is, in the absence of
@@ -210,21 +215,16 @@ reorder_group::apply(std::u32string &str) const {
   // get a baseline sort key
   auto sort_keys = reorder_sort_key::from(str);
 
-  // DebugLog("Baseline Keys:");
-  // for (auto e = sort_keys.begin(); e < sort_keys.end(); e++) {
-  //   e->dump();
-  // }
-
   // apply ALL reorders in the group.
-  // size_t c = 0;
-  for (auto r = list.begin(); r < list.end(); r++) {
+  for (const auto &r : list) {
     // work backward from end of string forward
+    // That is, see if "abc" matches "abc" or "ab" or "a"
     for (size_t s = str.size(); s > 0; s--) {
-      size_t submatch = r->match_end(str, 0, s);
+      size_t submatch = r.match_end(str, 0, s);
       if (submatch != 0) {
         // update the sort key
         size_t sub_match_start = s - submatch;
-        r->elements.update_sort_key(sub_match_start, sort_keys);
+        r.elements.update_sort_key(sub_match_start, sort_keys);
         some_match = true;
       }
     }
@@ -235,30 +235,28 @@ reorder_group::apply(std::u32string &str) const {
     return false;  // nothing matched, so no work.
   }
 
-  size_t match_len = str.size();  // TODO-LDML: for now, assume entire match
-
-  // DebugLog("Updated Keys:");
-  // for (auto e = sort_keys.begin(); e < sort_keys.end(); e++) {
-  //   e->dump();
-  // }
+  size_t match_len = str.size();  // TODO-LDML: for now, assume matches entire string
 
   std::u32string prefix = str;
   prefix.resize(str.size() - match_len);  // just the part before the matched part.
   // just the suffix (the matched part)
   std::u32string suffix = str.substr(prefix.size(), match_len);
-  // sort it! Here's where the reorder happens
-  // TODO: need to sort only between primary bases…
-  std::sort(sort_keys.begin(), sort_keys.end());
-#if 0
-  // TODO-LDML :need to sort sub-runs
-  for(auto e = sort_keys.end(); !applied && e > sort_keys.begin(); e--) {
-    if (e->primary == 0) {
-      // Got it.
-      std::sort(e, sort_keys.end());
-      // DebugLog("… sorting at q=%d", (int)e->quaternary);
+
+  /** pointer to the beginning of the current run. */
+  std::deque<reorder_sort_key>::iterator run_start = sort_keys.begin();
+  for(auto e = run_start; e != sort_keys.end(); e++) {
+    e->dump();
+    if ((e->primary == 0) && (e != run_start)) { // it's a base
+      auto run_end = e - 1;
+      std::sort(run_start, run_end); // reversed because it's a reverse iterator…?
+      // move the start
+      run_start = e; // next run starts here
     }
   }
-#endif
+  // sort the last run in the string as well.
+  if (run_start != sort_keys.end()) {
+    std::sort(run_start, sort_keys.end()); // reversed because it's a reverse iterator…?
+  }
   // recombine into a str
   std::u32string newSuffix;
   size_t q = sort_keys.begin()->quaternary;  //
@@ -269,10 +267,6 @@ reorder_group::apply(std::u32string &str) const {
     newSuffix.append(1, e->ch);
   }
   if (applied) {
-    // DebugLog("Final Sort");
-    // for (auto e = sort_keys.begin(); e < sort_keys.end(); e++) {
-    //   e->dump();
-    // }
     str.resize(prefix.size());
     str.append(newSuffix);
   } else {
diff --git a/core/src/ldml/ldml_transforms.hpp b/core/src/ldml/ldml_transforms.hpp
index cf5f993493..3746d3f864 100644
--- a/core/src/ldml/ldml_transforms.hpp
+++ b/core/src/ldml/ldml_transforms.hpp
@@ -116,6 +116,7 @@ struct reorder_sort_key {
   */
   int compare(const reorder_sort_key &other) const;
   bool operator<(const reorder_sort_key &other) const;
+  bool operator>(const reorder_sort_key &other) const;
 
   /** create a 'baseline' sort key, all 0 primary weights */
   static std::deque<reorder_sort_key> from(const std::u32string &str);
diff --git a/core/tests/unit/ldml/keyboards/k_200_reorder_nod_Lana-test.xml b/core/tests/unit/ldml/keyboards/k_200_reorder_nod_Lana-test.xml
index 5fcfd321e2..d23862ca2b 100644
--- a/core/tests/unit/ldml/keyboards/k_200_reorder_nod_Lana-test.xml
+++ b/core/tests/unit/ldml/keyboards/k_200_reorder_nod_Lana-test.xml
@@ -3,16 +3,30 @@
 <keyboardTest conformsTo="techpreview">
   <info keyboard="k_200_reorder_nod_Lana.xml" author="Team Keyboard" name="nod-Lana" />
   <tests name="reorder-tests">
-    <test name="roast0-orig">
+    <test name="roast00-orig-emit">
+      <startContext to="" />
+      <emit to="\u1A21\u1A60\u1A45\u1A6B\u1A76" />
+      <check result="\u1A21\u1A60\u1A45\u1A6B\u1A76" />
+    </test>
+    <test name="roast01-orig-key">
       <startContext to="" />
       <keystroke key="kha" />
+      <check result="\u1A21" />
       <keystroke key="sakot" />
+      <check result="\u1A21\u1A60" />
       <keystroke key="wa" />
+      <check result="\u1A21\u1A60\u1A45" />
       <keystroke key="o" />
+      <check result="\u1A21\u1A60\u1A45\u1A6B" />
       <keystroke key="t2" />
       <check result="\u1A21\u1A60\u1A45\u1A6B\u1A76" />
     </test>
-    <!-- <test name="roast1-upper-first">
+    <test name="roast02-orig-1key">
+      <startContext to="" />
+      <keystroke key="roast" />
+      <check result="\u1A21\u1A60\u1A45\u1A6B\u1A76" />
+    </test>
+    <test name="roast1-upper-first">
       <startContext to="" />
       <keystroke key="kha" />
       <keystroke key="o" />
@@ -38,6 +52,6 @@
       <keystroke key="wa" />
       <keystroke key="t2" />
       <check result="\u1A21\u1A60\u1A45\u1A6B\u1A76" />
-    </test> -->
+    </test>
   </tests>
 </keyboardTest>
diff --git a/core/tests/unit/ldml/keyboards/k_200_reorder_nod_Lana.xml b/core/tests/unit/ldml/keyboards/k_200_reorder_nod_Lana.xml
index da16c18717..8df0d5ffd7 100644
--- a/core/tests/unit/ldml/keyboards/k_200_reorder_nod_Lana.xml
+++ b/core/tests/unit/ldml/keyboards/k_200_reorder_nod_Lana.xml
@@ -13,16 +13,20 @@
   </names>
 
   <keys>
+    <!-- keys in order for 'roast' -->
     <key id="kha" to="\u{1A21}" />
     <key id="sakot" to="\u{1A60}" />
     <key id="wa" to="\u{1A45}" />
     <key id="o" to="\u{1A6B}" />
     <key id="t2" to="\u{1A76}" />
+
+    <!-- and a shortcut key for this common word -->
+    <key id="roast" to="\u{1A21}\u{1A60}\u{1A45}\u{1A6B}\u{1A76}" />
   </keys>
 
   <layers form="us">
     <layer modifier="none" id="base">
-      <row keys="gap" />
+      <row keys="roast" />
       <row keys="gap wa gap gap t2 gap gap gap o gap" />
       <row keys="gap sakot gap gap gap gap gap kha gap" />
       <row keys="gap gap gap gap gap gap gap" />
diff --git a/core/tests/unit/ldml/test_transforms.cpp b/core/tests/unit/ldml/test_transforms.cpp
index 03c2772292..dc658f7781 100644
--- a/core/tests/unit/ldml/test_transforms.cpp
+++ b/core/tests/unit/ldml/test_transforms.cpp
@@ -288,7 +288,7 @@ test_reorder_standalone() {
 
       // <reorder from="\u1A60" order="127" />
       element_list e0;
-      e0.emplace_back(U'\u1A6B', 127 << LDML_ELEM_FLAGS_ORDER_BITSHIFT);
+      e0.emplace_back(U'\u1A60', 127 << LDML_ELEM_FLAGS_ORDER_BITSHIFT);
       rg.list.emplace_back(e0);
 
       // <reorder from="\u1A6B" order="42" />
@@ -334,12 +334,28 @@ test_reorder_standalone() {
     std::cout << __FILE__ << ":" << __LINE__ << " - back to nod-Lana " << std::endl;
     // TODO-LDML: move this into test code perhaps
     for (size_t r = 0; r < sizeof(roasts) / sizeof(roasts[0]); r++) {
-      std::cout << __FILE__ << ":" << __LINE__ << " - trying roast #" << r << std::endl;
       const auto &roast = roasts[r];
+      std::cout << __FILE__ << ":" << __LINE__ << " - trying roast #" << r << "=" << roast << std::endl;
+      // try apply with string
+      {
+          std::cout << "- try apply(text, output)" << std::endl;
+          std::u32string text = roast;
+          std::u32string output;
+          size_t len = tr.apply(text, output);
+          if (len == 0) {
+            std::cout << " (did not apply)" << std::endl;
+          } else {
+            std::cout << " applied, matchLen= " << len << std::endl;
+            text.resize(text.size()-len); // shrink
+            text.append(output);
+            std::cout << " = " << text << std::endl;
+          }
+          zassert_string_equal(text, expect);
+      }
       // try all-at-once
       {
+        std::cout << "- try apply(text)" << std::endl;
         std::u32string text = roast;
-        std::cout << " Starting: " << roast << std::endl;
         if (!tr.apply(text)) {
           std::cout << " (did not apply)" << std::endl;
         } else if (text == roast) {
@@ -352,6 +368,7 @@ test_reorder_standalone() {
       }
       // simulate typing this one char at a time;
       {
+        std::cout << "- try key-at-a-time" << std::endl;
         std::u32string text;
         for (auto ch = roast.begin(); ch < roast.end(); ch++) {
           // append the string
@@ -367,6 +384,24 @@ test_reorder_standalone() {
         std::cout << std::endl;
       }
     }
+    // special test
+    {
+      std::cout << __FILE__ << ":" << __LINE__ << " - special test " << std::endl;
+      const std::u32string expect = U"\u1A21\u1A60\u1A45"; // this string shouldn't mutate at all.
+      {
+        std::u32string text = expect;
+        tr.apply(text);
+        zassert_string_equal(text, expect);
+      }
+      {
+        // try submatch
+        std::u32string text = expect;
+        std::u32string output;
+        size_t len = tr.apply(text, output);
+        zassert_string_equal(output, U"");
+        assert_equal(len, 0);
+      }
+    }
   }
   return EXIT_SUCCESS;
 }

From 96b11d8a7349066744702ec3e7f667dca0c47ee7 Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Wed, 19 Jul 2023 18:14:01 -0500
Subject: [PATCH 10/83] =?UTF-8?q?feat(core):=20transform/reorder=20passing?=
 =?UTF-8?q?=20tests!=20=20=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- remove a debug print call

#7375
---
 core/src/ldml/ldml_transforms.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/ldml/ldml_transforms.cpp b/core/src/ldml/ldml_transforms.cpp
index 9320a10dd8..0a4c83f8f1 100644
--- a/core/src/ldml/ldml_transforms.cpp
+++ b/core/src/ldml/ldml_transforms.cpp
@@ -172,6 +172,7 @@ element_list::update_sort_key(size_t offset, std::deque<reorder_sort_key> &key)
     if (!e->matches(k.ch)) {
       DebugLog("!! updateSortKey(%d+%d): element did not re-match the sortkey", offset, c);
       k.dump();
+      // TODO-LDML: assertion follows
     }
     assert(e->matches(k.ch));        // double check that this element matches
     k.primary  = e->get_order();
@@ -245,7 +246,6 @@ reorder_group::apply(std::u32string &str) const {
   /** pointer to the beginning of the current run. */
   std::deque<reorder_sort_key>::iterator run_start = sort_keys.begin();
   for(auto e = run_start; e != sort_keys.end(); e++) {
-    e->dump();
     if ((e->primary == 0) && (e != run_start)) { // it's a base
       auto run_end = e - 1;
       std::sort(run_start, run_end); // reversed because it's a reverse iterator…?

From 8f72f61cbb11c5cebbcd3cb575ff94055ed63871 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 20 Jul 2023 11:17:39 +0000
Subject: [PATCH 11/83] chore(deps-dev): bump word-wrap from 1.2.3 to 1.2.4

Bumps [word-wrap](https://github.com/jonschlinkert/word-wrap) from 1.2.3 to 1.2.4.
- [Release notes](https://github.com/jonschlinkert/word-wrap/releases)
- [Commits](https://github.com/jonschlinkert/word-wrap/compare/1.2.3...1.2.4)

---
updated-dependencies:
- dependency-name: word-wrap
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 package-lock.json | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 658eb2d26c..f07cd1bd6d 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -10590,9 +10590,9 @@
       }
     },
     "node_modules/word-wrap": {
-      "version": "1.2.3",
-      "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",
-      "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==",
+      "version": "1.2.4",
+      "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.4.tgz",
+      "integrity": "sha512-2V81OA4ugVo5pRo46hAoD2ivUJx8jXmWXfUkY4KFNw0hEptvN0QfH3K4nHiwzGeKl5rFKedV48QVoqYavy4YpA==",
       "dev": true,
       "engines": {
         "node": ">=0.10.0"

From ac6dbb62918a6297341b4672d93d7a473d8bb49e Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Thu, 20 Jul 2023 12:04:50 -0500
Subject: [PATCH 12/83] =?UTF-8?q?chore(core):=20scale=20back=20kmxplus=20d?=
 =?UTF-8?q?ebugging=20=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- define KMXPLUS_DEBUG_LOAD, 0 by default, to get details such as 'each string' and 'each key'
- overall debugging and anomalies still DebugLog by default.
- new macro DebugLoad
- also KMXPLUS_DEBUG_TRANSFORM similarly and DebugTran
---
 core/src/kmx/kmx_plus.cpp         | 40 +++++++++++++++++++++----------
 core/src/ldml/ldml_transforms.cpp | 40 +++++++++++++++++++++++++++----
 2 files changed, 64 insertions(+), 16 deletions(-)

diff --git a/core/src/kmx/kmx_plus.cpp b/core/src/kmx/kmx_plus.cpp
index e5132193f5..db4e79598c 100644
--- a/core/src/kmx/kmx_plus.cpp
+++ b/core/src/kmx/kmx_plus.cpp
@@ -18,6 +18,20 @@ namespace km {
 namespace kbp {
 namespace kmx {
 
+/**
+ * \def KMXPLUS_DEBUG_LOAD set to 1 to print messages on KMXPLUS loading.
+ * Off by default.
+*/
+#ifndef KMXPLUS_DEBUG_LOAD
+#define KMXPLUS_DEBUG_LOAD 0
+#endif
+
+#if KMXPLUS_DEBUG_LOAD
+#define DebugLoad(msg,...) DebugLog(msg, __VA_ARGS__)
+#else
+#define DebugLoad(msg,...)
+#endif
+
 // double check these modifier mappings
 static_assert(LCTRLFLAG == LDML_KEYS_MOD_CTRLL, "LDML modifier bitfield vs. kmx_file.h #define mismatch");
 static_assert(RCTRLFLAG == LDML_KEYS_MOD_CTRLR, "LDML modifier bitfield vs. kmx_file.h #define mismatch");
@@ -231,7 +245,7 @@ COMP_KMXPLUS_DISP::valid(KMX_DWORD _kmn_unused(length)) const {
     DebugLog("disp: baseCharacter str#0x%X", baseCharacter);
   }
   for (KMX_DWORD i=0; i<count; i++) {
-    DebugLog("disp#%d: to: str0x%X -> str0x%X", i, entries[i].to, entries[i].display);
+    DebugLoad("disp#%d: to: str0x%X -> str0x%X", i, entries[i].to, entries[i].display);
     if (entries[i].to == 0 || entries[i].display == 0) {
       DebugLog("disp to: or display: has a zero string");
       assert(false);
@@ -265,7 +279,7 @@ COMP_KMXPLUS_STRS::valid(KMX_DWORD _kmn_unused(length)) const {
       return false;
     }
     // TODO-LDML: validate valid UTF-16LE?
-    DebugLog("strs #0x%X: '%s'", i, Debug_UnicodeString(start));
+    DebugLoad("strs #0x%X: '%s'", i, Debug_UnicodeString(start));
   }
   return true;
 }
@@ -740,7 +754,7 @@ COMP_KMXPLUS_KEYS_Helper::setKeys(const COMP_KMXPLUS_KEYS *newKeys) {
     for(KMX_DWORD i = 0; is_valid && i < key2->keyCount; i++) {
       const auto &key = keys[i];
       // is the count off the end?
-      DebugLog( "<key #%d> id=0x%X, to=0x%X, flicks=%d", i, key.id, key.to, key.flicks); // TODO-LDML: could dump more fields here
+      DebugLoad( "<key #%d> id=0x%X, to=0x%X, flicks=%d", i, key.id, key.to, key.flicks); // TODO-LDML: could dump more fields here
       if (key.flicks >0 && key.flicks >= key2->flicksCount) {
         DebugLog("key[%d] has invalid flicks index %d", i, key.flicks);
         is_valid = false;
@@ -750,7 +764,7 @@ COMP_KMXPLUS_KEYS_Helper::setKeys(const COMP_KMXPLUS_KEYS *newKeys) {
     for(KMX_DWORD i = 0; is_valid && i < key2->flicksCount; i++) {
       const auto &e = flickLists[i];
       // is the count off the end?
-      DebugLog("<flicks> %d: index %d, count %d", i, e.flick, e.count);
+      DebugLoad("<flicks> %d: index %d, count %d", i, e.flick, e.count);
       if (i == 0) {
         if (e.flick != 0 || e.count != 0) {
           DebugLog("Error: Invalid Flick #0");
@@ -766,17 +780,16 @@ COMP_KMXPLUS_KEYS_Helper::setKeys(const COMP_KMXPLUS_KEYS *newKeys) {
     for(KMX_DWORD i = 0; is_valid && i < key2->flickCount; i++) {
       const auto &e = flickElements[i];
       // is the count off the end?
-      DebugLog("<flick> %d: to=0x%X, directions=0x%X, flags=0x%X", i, e.to, e.directions, e.flags);
+      DebugLoad("<flick> %d: to=0x%X, directions=0x%X, flags=0x%X", i, e.to, e.directions, e.flags);
     }
     // now the kmap
     DebugLog(" kmap count: #0x%X", key2->kmapCount);
     for (KMX_DWORD i = 0; i < key2->kmapCount; i++) {
-      // These are pretty noisy, drop them from the log
-      // DebugLog(" #0x%d\n", i);
+      DebugLoad(" #0x%d\n", i);
       auto &entry = kmap[i];
-      // DebugLog("  vkey\t0x%X", entry.vkey);
-      // DebugLog("  mod\t0x%X", entry.mod);
-      // DebugLog("  key\t#0x%X", entry.key);
+      DebugLoad("  vkey\t0x%X", entry.vkey);
+      DebugLoad("  mod\t0x%X", entry.mod);
+      DebugLoad("  key\t#0x%X", entry.key);
       if (!LDML_IS_VALID_MODIFIER_BITS(entry.mod)) {
         DebugLog("Invalid modifier value");
         assert(false);
@@ -930,7 +943,7 @@ COMP_KMXPLUS_LIST_Helper::setList(const COMP_KMXPLUS_LIST *newList) {
     }
     for (KMX_DWORD i = 0; is_valid && i < list->indexCount; i++) {
       const auto &e = indices[i];
-      DebugLog(" index %d: str 0x%X", i, e);
+      DebugLoad(" index %d: str 0x%X", i, e);
     }
   }
   // Return results
@@ -977,7 +990,7 @@ COMP_KMXPLUS_USET_Helper::COMP_KMXPLUS_USET_Helper() : uset(nullptr), is_valid(f
 
 bool
 COMP_KMXPLUS_USET_Helper::setUset(const COMP_KMXPLUS_USET *newUset) {
-  DebugLog("validating newUset=%p", newUset);
+  DebugLoad("validating newUset=%p", newUset);
   is_valid = true;
   if (newUset == nullptr) {
     // Note: kmx_plus::kmx_plus has already called section_from_bytes()
@@ -1083,6 +1096,9 @@ kmx_plus::kmx_plus(const COMP_KEYBOARD *keyboard, size_t length)
     : bksp(nullptr), disp(nullptr), elem(nullptr), key2(nullptr), layr(nullptr), list(nullptr), loca(nullptr), meta(nullptr),
       sect(nullptr), strs(nullptr), tran(nullptr), vars(nullptr), vkey(nullptr), valid(false) {
   DebugLog("kmx_plus: Got a COMP_KEYBOARD at %p\n", keyboard);
+#if !KMXPLUS_DEBUG_LOAD
+  DebugLog("Note: define KMXPLUS_DEBUG_LOAD=1 at compile time for more verbosity in loading");
+#endif
   if (!(keyboard->dwFlags & KF_KMXPLUS)) {
     DebugLog("Err: flags COMP_KEYBOARD.dwFlags did not have KF_KMXPLUS set");
     valid = false;
diff --git a/core/src/ldml/ldml_transforms.cpp b/core/src/ldml/ldml_transforms.cpp
index 0a4c83f8f1..e08c0c77c2 100644
--- a/core/src/ldml/ldml_transforms.cpp
+++ b/core/src/ldml/ldml_transforms.cpp
@@ -20,6 +20,16 @@ namespace km {
 namespace kbp {
 namespace ldml {
 
+#ifndef KMXPLUS_DEBUG_TRANSFORM
+#define KMXPLUS_DEBUG_TRANSFORM 1
+#endif
+
+#if KMXPLUS_DEBUG_TRANSFORM
+#define DebugTran(msg, ...) DebugLog(msg, __VA_ARGS__)
+#else
+#define DebugTran(msg, ...)
+#endif
+
 element::element(const USet &new_u, KMX_DWORD new_flags)
     : chr(), uset(new_u), flags((new_flags & ~LDML_ELEM_FLAGS_TYPE) | LDML_ELEM_FLAGS_TYPE_USET) {
 }
@@ -216,6 +226,13 @@ reorder_group::apply(std::u32string &str) const {
   // get a baseline sort key
   auto sort_keys = reorder_sort_key::from(str);
 
+#if 0 && KMXPLUS_DEBUG_TRANSFORM
+  DebugTran("Baseline sortkey");
+  for (const auto &r : sort_keys) {
+    r.dump();
+  }
+#endif
+
   // apply ALL reorders in the group.
   for (const auto &r : list) {
     // work backward from end of string forward
@@ -232,10 +249,17 @@ reorder_group::apply(std::u32string &str) const {
     // c++;
   }
   if (!some_match) {
-    // DebugLog("Skip: No reorder elements matched.");
+    DebugTran("Skip: No reorder elements matched.");
     return false;  // nothing matched, so no work.
   }
 
+#if KMXPLUS_DEBUG_TRANSFORM
+  DebugTran("Updated sortkey");
+  for (const auto &r : sort_keys) {
+    r.dump();
+  }
+#endif
+
   size_t match_len = str.size();  // TODO-LDML: for now, assume matches entire string
 
   std::u32string prefix = str;
@@ -248,13 +272,15 @@ reorder_group::apply(std::u32string &str) const {
   for(auto e = run_start; e != sort_keys.end(); e++) {
     if ((e->primary == 0) && (e != run_start)) { // it's a base
       auto run_end = e - 1;
-      std::sort(run_start, run_end); // reversed because it's a reverse iterator…?
+      DebugTran("Sorting subrange quaternary=[%d..]", run_start->quaternary);
+      std::sort(run_start, run_end);  // reversed because it's a reverse iterator…?
       // move the start
       run_start = e; // next run starts here
     }
   }
   // sort the last run in the string as well.
-  if (run_start != sort_keys.end()) {
+  if (run_start != sort_keys.end()) { // TODO-LDML: skip if a single-char run
+    DebugTran("Sorting final subrange quaternary=[%d..]", run_start->quaternary);
     std::sort(run_start, sort_keys.end()); // reversed because it's a reverse iterator…?
   }
   // recombine into a str
@@ -270,8 +296,14 @@ reorder_group::apply(std::u32string &str) const {
     str.resize(prefix.size());
     str.append(newSuffix);
   } else {
-    // DebugLog("Skip: no reordering change detected");
+    DebugTran("Skip: sorting caused no reordering");
   }
+#if KMXPLUS_DEBUG_TRANSFORM
+  DebugTran("Sorted sortkey");
+  for (const auto &r : sort_keys) {
+    r.dump();
+  }
+#endif
   return applied;
 }
 

From 6ea90935bcbde3d6a010105ad870c45664c24303 Mon Sep 17 00:00:00 2001
From: "Joshua A. Horton" <joshua_horton@sil.org>
Date: Fri, 21 Jul 2023 08:04:37 +0700
Subject: [PATCH 13/83] fix(web): missed a closing bracket in suggestion

---
 web/src/engine/package-cache/src/keyboardRequisitioner.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/web/src/engine/package-cache/src/keyboardRequisitioner.ts b/web/src/engine/package-cache/src/keyboardRequisitioner.ts
index aeedc27300..7bed7b12ac 100644
--- a/web/src/engine/package-cache/src/keyboardRequisitioner.ts
+++ b/web/src/engine/package-cache/src/keyboardRequisitioner.ts
@@ -108,6 +108,7 @@ export default class KeyboardRequisitioner {
         registration.forEach((entry) => {
           this.cache.addStub(entry);
         });
+      }
     });
   }
 

From 35d45af53cb5a4d0454c03519998ea5e2216b34e Mon Sep 17 00:00:00 2001
From: "Joshua A. Horton" <joshua_horton@sil.org>
Date: Fri, 21 Jul 2023 08:25:58 +0700
Subject: [PATCH 14/83] fix(web): eventemitter import style affected unit test

---
 web/src/engine/package-cache/src/cloud/queryEngine.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/web/src/engine/package-cache/src/cloud/queryEngine.ts b/web/src/engine/package-cache/src/cloud/queryEngine.ts
index 219dd911df..3fe0bcc025 100644
--- a/web/src/engine/package-cache/src/cloud/queryEngine.ts
+++ b/web/src/engine/package-cache/src/cloud/queryEngine.ts
@@ -1,4 +1,4 @@
-import { EventEmitter } from 'eventemitter3';
+import EventEmitter from 'eventemitter3';
 
 import { PathConfiguration } from 'keyman/engine/paths';
 

From 4d9fd28181d09983a93ed33b6cdd58077d97d0c9 Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Fri, 21 Jul 2023 11:42:32 -0500
Subject: [PATCH 15/83] =?UTF-8?q?feat(core):=20fix=20UMR=20in=20Uset=20?=
 =?UTF-8?q?=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- USet struct was depending on the COMP_KEYBOARD_EX (the .kmx data), but was used after load completed during processing
- Change the USet structure to contain a std::list<COMP_KMXPLUS_USET_RANGE> i.e. copying it
- but, while we're at it, added functions to validate that characters are within valid Unicode ranges.

#7375
---
 core/src/kmx/kmx_plus.cpp                   | 50 ++++++++++++++++++---
 core/src/kmx/kmx_plus.h                     | 14 +++---
 core/src/kmx/kmx_xstring.h                  | 16 +++++++
 core/src/ldml/ldml_transforms.cpp           | 44 ++++++++++++++----
 core/src/ldml/ldml_transforms.hpp           |  3 ++
 core/tests/unit/kmnkbd/test_kmx_xstring.cpp | 25 +++++++++++
 core/tests/unit/ldml/test_transforms.cpp    |  2 +-
 7 files changed, 133 insertions(+), 21 deletions(-)

diff --git a/core/src/kmx/kmx_plus.cpp b/core/src/kmx/kmx_plus.cpp
index db4e79598c..43e43e825a 100644
--- a/core/src/kmx/kmx_plus.cpp
+++ b/core/src/kmx/kmx_plus.cpp
@@ -982,7 +982,13 @@ COMP_KMXPLUS_USET::valid(KMX_DWORD _kmn_unused(length)) const {
     assert(false);
     return false;
   }
-  return true;
+  return true; // see helper
+}
+
+COMP_KMXPLUS_USET_RANGE::COMP_KMXPLUS_USET_RANGE(KMX_DWORD s, KMX_DWORD e) : start(s), end(e) {
+}
+
+COMP_KMXPLUS_USET_RANGE::COMP_KMXPLUS_USET_RANGE(const COMP_KMXPLUS_USET_RANGE &other) : start(other.start), end(other.end) {
 }
 
 COMP_KMXPLUS_USET_Helper::COMP_KMXPLUS_USET_Helper() : uset(nullptr), is_valid(false), usets(nullptr), ranges(nullptr) {
@@ -1030,9 +1036,13 @@ COMP_KMXPLUS_USET_Helper::setUset(const COMP_KMXPLUS_USET *newUset) {
     } else {
       /** last lastEnd value */
       KMX_DWORD lastEnd = 0x0;
-      for (KMX_DWORD r = 0; r < e.count; r++) {
+      for (KMX_DWORD r = 0; is_valid && r < e.count; r++) {
         const auto &range = ranges[e.range + r];  // already range-checked 'r' above
-        if (range.end < range.start) {
+        if (!Uni_IsValid(range.start) || !Uni_IsValid(range.end)) {
+          DebugLog("uset[%d][%d] not valid: [U+%04X-U+%04X]", i, r, range.start, range.end);
+          is_valid = false;
+          assert(is_valid);
+        } else if (range.end < range.start) {
           // range swapped
           DebugLog("uset[%d]: range[%d+%d] end 0x%X<start 0x%X", i, e.range, r, range.end, range.start);
           is_valid = false;
@@ -1055,15 +1065,17 @@ COMP_KMXPLUS_USET_Helper::setUset(const COMP_KMXPLUS_USET *newUset) {
   return is_valid;
 }
 
-USet::USet(const COMP_KMXPLUS_USET_RANGE *newRange, size_t newCount) : ranges(newRange), count(newCount) {
+USet::USet(const COMP_KMXPLUS_USET_RANGE *newRange, size_t newCount)  {
+  for (size_t i = 0; i < newCount; i++) {
+    ranges.emplace_back(newRange[i].start, newRange[i].end);
+  }
 }
 
-USet::USet() : ranges(nullptr), count(0) {
+USet::USet() {
 }
 
 bool USet::contains(km_kbp_usv ch) const {
-  for (size_t i = 0; i < count; i++) {
-    const auto &range = ranges[i];
+  for (const auto &range : ranges) {
     if (range.start <= ch && range.end >= ch) {
       return true;
     }
@@ -1071,6 +1083,30 @@ bool USet::contains(km_kbp_usv ch) const {
   return false;
 }
 
+bool
+USet::valid() const {
+  // double check
+  for (const auto &range : ranges) {
+    if (!Uni_IsValid(range.start) || !Uni_IsValid(range.end)) {
+      DebugLog("Invalid UnicodeSet (contains noncharacters): [U+%04X,U+%04X]", (int)range.start, (int)range.end);
+      return false;
+    }
+  }
+  return true;
+}
+
+void
+USet::dump() const {
+  DebugLog(" - USet size=%d", ranges.size());
+  for (const auto &range : ranges) {
+    if (range.start == range.end) {
+      DebugLog("  - [U+%04X]", (uint32_t)range.start);
+    } else {
+      DebugLog("  - [U+%04X-U+%04X]", (uint32_t)range.start, (uint32_t)range.end);
+    }
+  }
+}
+
 USet
 COMP_KMXPLUS_USET_Helper::getUset(KMXPLUS_USET i) const {
   if (!valid() || i >= uset->usetCount) {
diff --git a/core/src/kmx/kmx_plus.h b/core/src/kmx/kmx_plus.h
index cbcdb4a323..684da2a192 100644
--- a/core/src/kmx/kmx_plus.h
+++ b/core/src/kmx/kmx_plus.h
@@ -11,6 +11,7 @@
 #include <kmx/kmx_base.h>
 #include <kmx_file.h>
 #include <ldml/keyboardprocessor_ldml.h>
+#include <list>
 
 namespace km {
 namespace kbp {
@@ -679,24 +680,27 @@ struct COMP_KMXPLUS_USET_USET {
 struct COMP_KMXPLUS_USET_RANGE {
   km_kbp_usv start;
   km_kbp_usv end;
+  public:
+    COMP_KMXPLUS_USET_RANGE(const COMP_KMXPLUS_USET_RANGE& other);
+    COMP_KMXPLUS_USET_RANGE(KMX_DWORD start, KMX_DWORD end);
 };
 
 /**
  * represents one of the uset elements
- * Aliases, does not copy memory.
- * The original KMX+ memory must stay around while this object is held.
  */
 class USet {
   public:
-    /** construct a set over the specified range. */
+    /** construct a set over the specified range. Data is copied. */
     USet(const COMP_KMXPLUS_USET_RANGE* newStart, size_t newCount);
     /** empty set */
     USet();
     /** true if the uset contains this char */
     bool contains(km_kbp_usv ch) const;
+    /** debugging */
+    void dump() const;
+    bool valid() const;
   private:
-    const COMP_KMXPLUS_USET_RANGE *ranges;
-    size_t count;
+    std::list<COMP_KMXPLUS_USET_RANGE> ranges;
 };
 
 class COMP_KMXPLUS_USET_Helper {
diff --git a/core/src/kmx/kmx_xstring.h b/core/src/kmx/kmx_xstring.h
index fafa277ed7..68f1edd5ce 100644
--- a/core/src/kmx/kmx_xstring.h
+++ b/core/src/kmx/kmx_xstring.h
@@ -17,6 +17,12 @@ namespace kmx {
  */
 #define Uni_IsSurrogate2(ch) ((ch) >= 0xDC00 && (ch) <= 0xDFFF)
 
+/**
+ * @brief True if any surrogate
+ * \def UniIsSurrogate
+*/
+#define Uni_IsSurrogate(ch) (Uni_IsSurrogate1(ch) || Uni_IsSurrogate2(ch))
+
 /**
  * @brief Returns true if BMP (Plane 0)
  * \def Uni_IsBMP
@@ -41,6 +47,16 @@ namespace kmx {
 #define Uni_UTF32ToSurrogate1(ch) (char16_t)(((ch) - 0x10000) / 0x400 + 0xD800)
 #define Uni_UTF32ToSurrogate2(ch) (char16_t)(((ch) - 0x10000) % 0x400 + 0xDC00)
 
+#define Uni_IsNoncharacter(ch) ((ch >= 0xFDD0 && ch <= 0xFDEF) || ((ch & 0xFFFE) == 0xFFFE))
+
+#define Uni_InCodespace(ch) (ch <= 0x10FFFF)
+
+/**
+ * @brief True if in codespace and NOT a surrogate or noncharacter.
+ * \def Uni_IsValid
+*/
+#define Uni_IsValid(ch) true // (Uni_InCodespace(ch) && !Uni_IsSurrogate(ch) && !Uni_IsNoncharacter(ch))
+
 /**
  * char16_t array big enough to hold a single Unicode codepoint,
  * including trailing null.
diff --git a/core/src/ldml/ldml_transforms.cpp b/core/src/ldml/ldml_transforms.cpp
index e08c0c77c2..913bffbe6d 100644
--- a/core/src/ldml/ldml_transforms.cpp
+++ b/core/src/ldml/ldml_transforms.cpp
@@ -21,7 +21,7 @@ namespace kbp {
 namespace ldml {
 
 #ifndef KMXPLUS_DEBUG_TRANSFORM
-#define KMXPLUS_DEBUG_TRANSFORM 1
+#define KMXPLUS_DEBUG_TRANSFORM 0
 #endif
 
 #if KMXPLUS_DEBUG_TRANSFORM
@@ -79,6 +79,16 @@ element::matches(km_kbp_usv ch) const {
   }
 }
 
+void
+element::dump() const {
+  if (is_uset()) {
+    DebugLog("element order=%d USET", (int)get_order());
+    uset.dump();
+  } else {
+    DebugLog("element order=%d U+%04X", (int)get_order(), (int)chr);
+  }
+}
+
 int
 reorder_sort_key::compare(const reorder_sort_key &other) const {
   int primaryResult    = (int)primary    - (int)other.primary;
@@ -164,6 +174,12 @@ element_list::load(const kmx::kmx_plus &kplus, kmx::KMXPLUS_ELEM id) {
       emplace_back(e.element, flags); // char
     } else if (type == LDML_ELEM_FLAGS_TYPE_USET) {
       auto u = kplus.usetHelper.getUset(e.element);
+      if (!u.valid()) {
+        DebugLog("Error, invalid UnicodeSet at element %d", (int)i);
+        u.dump();
+        assert(u.valid());
+        return false;
+      }
       emplace_back(u, e.flags);
     } else {
       // not handled
@@ -171,6 +187,10 @@ element_list::load(const kmx::kmx_plus &kplus, kmx::KMXPLUS_ELEM id) {
       return false;
     }
   }
+#if KMXPLUS_DEBUG_TRANSFORM
+  DebugTran("Loaded:");
+  dump();
+#endif
   return true;
 }
 
@@ -187,11 +207,21 @@ element_list::update_sort_key(size_t offset, std::deque<reorder_sort_key> &key)
     assert(e->matches(k.ch));        // double check that this element matches
     k.primary  = e->get_order();
     k.tertiary = e->get_tertiary();  // TODO-LDML: need more detailed tertiary work
+    DebugTran("Updating at +%d", c);
+    k.dump();
     c++;
   }
   return key;
 }
 
+void
+element_list::dump() const {
+  DebugLog("element_list[%d]", size());
+  for (const auto &e : *this) {
+    e.dump();
+  }
+}
+
 reorder_entry::reorder_entry(const element_list &new_elements) : elements(new_elements), before() {
 }
 reorder_entry::reorder_entry(const element_list &new_elements, const element_list &new_before) : elements(new_elements), before(new_before) {
@@ -226,13 +256,6 @@ reorder_group::apply(std::u32string &str) const {
   // get a baseline sort key
   auto sort_keys = reorder_sort_key::from(str);
 
-#if 0 && KMXPLUS_DEBUG_TRANSFORM
-  DebugTran("Baseline sortkey");
-  for (const auto &r : sort_keys) {
-    r.dump();
-  }
-#endif
-
   // apply ALL reorders in the group.
   for (const auto &r : list) {
     // work backward from end of string forward
@@ -240,6 +263,10 @@ reorder_group::apply(std::u32string &str) const {
     for (size_t s = str.size(); s > 0; s--) {
       size_t submatch = r.match_end(str, 0, s);
       if (submatch != 0) {
+#if KMXPLUS_DEBUG_TRANSFORM
+        DebugTran("Matched: %S (off=%d, len=%d)", str, 0, s);
+        r.elements.dump();
+#endif
         // update the sort key
         size_t sub_match_start = s - submatch;
         r.elements.update_sort_key(sub_match_start, sort_keys);
@@ -564,6 +591,7 @@ transforms::load(
         if (load_ok) {
           newGroup.list.emplace_back(elements, before);
         } else {
+          DebugLog("reorder elements(%d+%d) failed to load", group->index, itemNumber);
           return nullptr;
         }
       }
diff --git a/core/src/ldml/ldml_transforms.hpp b/core/src/ldml/ldml_transforms.hpp
index 3746d3f864..be1d1abac4 100644
--- a/core/src/ldml/ldml_transforms.hpp
+++ b/core/src/ldml/ldml_transforms.hpp
@@ -48,6 +48,7 @@ public:
   KMX_DWORD get_flags() const;
   /** @returns true if matches this character*/
   bool matches(km_kbp_usv ch) const;
+  void dump() const;
 
 private:
   // TODO-LDML: support multi-char strings
@@ -145,6 +146,8 @@ public:
   /** construct from KMX+ elem id*/
   bool
   load(const kmx::kmx_plus& kplus, kmx::KMXPLUS_ELEM id);
+
+  void dump() const;
 };
 
 class reorder_entry {
diff --git a/core/tests/unit/kmnkbd/test_kmx_xstring.cpp b/core/tests/unit/kmnkbd/test_kmx_xstring.cpp
index d578fa7e38..38e3468449 100644
--- a/core/tests/unit/kmnkbd/test_kmx_xstring.cpp
+++ b/core/tests/unit/kmnkbd/test_kmx_xstring.cpp
@@ -1238,6 +1238,7 @@ test_xstrlen_ignoreifopt() {
 
 void
 test_utf32() {
+  std::cout << "== " << __FUNCTION__ << std::endl;
   const KMX_DWORD u295 =     0x0127; // ħ
 
   assert(Uni_IsBMP(u295));
@@ -1270,6 +1271,7 @@ test_utf32() {
 
 void
 test_u16string_to_u32string() {
+  std::cout << "== " << __FUNCTION__ << std::endl;
   // normal cases
   {
     const std::u32string str = u16string_to_u32string(u"");
@@ -1327,6 +1329,28 @@ test_u16string_to_u32string() {
   }
 }
 
+void test_is_valid() {
+  std::cout << "== " << __FUNCTION__ << std::endl;
+  // valid
+  assert_equal(Uni_IsValid(0x0000), true);
+  assert_equal(Uni_IsValid(0x0127), true);
+  assert_equal(Uni_IsValid(U'🙀'), true);
+
+  // invalid
+  assert_equal(Uni_IsValid(0xDECAFBAD), false); // out of range
+  assert_equal(Uni_IsValid(0x566D4128), false);
+  assert_equal(Uni_IsValid(0xFFFF), false); // nonchar
+  assert_equal(Uni_IsValid(0xFFFE), false); // nonchar
+  assert_equal(Uni_IsValid(0x10FFFF), false); // nonchar
+  assert_equal(Uni_IsValid(0x10FFFE), false); // nonchar
+  assert_equal(Uni_IsValid(0x01FFFF), false); // nonchar
+  assert_equal(Uni_IsValid(0x01FFFE), false); // nonchar
+  assert_equal(Uni_IsValid(0x02FFFF), false); // nonchar
+  assert_equal(Uni_IsValid(0x02FFFE), false); // nonchar
+  assert_equal(Uni_IsValid(0xFDD1), false); // nonchar
+  assert_equal(Uni_IsValid(0xFDD0), false); // nonchar  
+}
+
 constexpr const auto help_str = u"\
 test_kmx_xstring [--color]\n\
 \n\
@@ -1349,6 +1373,7 @@ int main(int argc, char *argv []) {
   test_xstrlen_ignoreifopt();
   test_utf32();
   test_u16string_to_u32string();
+  test_is_valid();
 
   return 0;
 }
diff --git a/core/tests/unit/ldml/test_transforms.cpp b/core/tests/unit/ldml/test_transforms.cpp
index dc658f7781..7097bcb64b 100644
--- a/core/tests/unit/ldml/test_transforms.cpp
+++ b/core/tests/unit/ldml/test_transforms.cpp
@@ -163,7 +163,7 @@ test_reorder_standalone() {
     const std::u32string expect = roasts[0];
     // now setup the rules
     const COMP_KMXPLUS_USET_RANGE ranges[]      = {// 0
-                                              {0x1A75, 0x1A79}};
+                                              COMP_KMXPLUS_USET_RANGE(0x1A75, 0x1A79)};
     const COMP_KMXPLUS_USET_USET usets[]        = {{0, 1, 0xFFFFFFFF}};
     const COMP_KMXPLUS_USET_USET &toneMarksUset = usets[0];
     const USet toneMarks(&ranges[toneMarksUset.range], toneMarksUset.count);

From 78a5879b72a6f05359157e43779e06b42e40f6ab Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Fri, 21 Jul 2023 11:43:58 -0500
Subject: [PATCH 16/83] =?UTF-8?q?feat(core):=20fix=20testing=20string=20?=
 =?UTF-8?q?=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- oops, had commented out a section for testing

#7375
---
 core/src/kmx/kmx_xstring.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/kmx/kmx_xstring.h b/core/src/kmx/kmx_xstring.h
index 68f1edd5ce..06fb8a0758 100644
--- a/core/src/kmx/kmx_xstring.h
+++ b/core/src/kmx/kmx_xstring.h
@@ -55,7 +55,7 @@ namespace kmx {
  * @brief True if in codespace and NOT a surrogate or noncharacter.
  * \def Uni_IsValid
 */
-#define Uni_IsValid(ch) true // (Uni_InCodespace(ch) && !Uni_IsSurrogate(ch) && !Uni_IsNoncharacter(ch))
+#define Uni_IsValid(ch) (Uni_InCodespace(ch) && !Uni_IsSurrogate(ch) && !Uni_IsNoncharacter(ch))
 
 /**
  * char16_t array big enough to hold a single Unicode codepoint,

From f88ffbff944ee635e72c52a5957bd1b451ad57e4 Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Fri, 21 Jul 2023 12:20:46 -0500
Subject: [PATCH 17/83] =?UTF-8?q?chore(core):=20improve=20validity=20check?=
 =?UTF-8?q?=20on=20transform=20=20=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- also remove unnecessary loops with unused parameters

#7375
---
 core/src/kmx/kmx_plus.cpp         | 9 ++++++++-
 core/src/ldml/ldml_transforms.cpp | 2 +-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/core/src/kmx/kmx_plus.cpp b/core/src/kmx/kmx_plus.cpp
index 43e43e825a..46521961fb 100644
--- a/core/src/kmx/kmx_plus.cpp
+++ b/core/src/kmx/kmx_plus.cpp
@@ -779,7 +779,12 @@ COMP_KMXPLUS_KEYS_Helper::setKeys(const COMP_KMXPLUS_KEYS *newKeys) {
     }
     for(KMX_DWORD i = 0; is_valid && i < key2->flickCount; i++) {
       const auto &e = flickElements[i];
-      // is the count off the end?
+      // validate to is present
+      if (e.to == 0 || e.directions == 0) {
+        DebugLog("flickElement[%d] has empty to=%0x%X or directions=%0x%X", i, e.to, e.directions);
+        is_valid = false;
+        assert(is_valid);
+      }
       DebugLoad("<flick> %d: to=0x%X, directions=0x%X, flags=0x%X", i, e.to, e.directions, e.flags);
     }
     // now the kmap
@@ -941,10 +946,12 @@ COMP_KMXPLUS_LIST_Helper::setList(const COMP_KMXPLUS_LIST *newList) {
         assert(is_valid);
       }
     }
+#if KMXPLUS_DEBUG_LOAD
     for (KMX_DWORD i = 0; is_valid && i < list->indexCount; i++) {
       const auto &e = indices[i];
       DebugLoad(" index %d: str 0x%X", i, e);
     }
+#endif
   }
   // Return results
   DebugLog("COMP_KMXPLUS_LIST_Helper.setList(): %s", is_valid ? "valid" : "invalid");
diff --git a/core/src/ldml/ldml_transforms.cpp b/core/src/ldml/ldml_transforms.cpp
index 913bffbe6d..95565feb0d 100644
--- a/core/src/ldml/ldml_transforms.cpp
+++ b/core/src/ldml/ldml_transforms.cpp
@@ -25,7 +25,7 @@ namespace ldml {
 #endif
 
 #if KMXPLUS_DEBUG_TRANSFORM
-#define DebugTran(msg, ...) DebugLog(msg, __VA_ARGS__)
+#define DebugTran(msg, ...) DebugLog(msg, ##__VA_ARGS__)
 #else
 #define DebugTran(msg, ...)
 #endif

From 11d5a428fd42a52fd171c1994d43f02afe4b952d Mon Sep 17 00:00:00 2001
From: Keyman Build Agent <buildagent@keyman.com>
Date: Fri, 21 Jul 2023 14:03:22 -0400
Subject: [PATCH 18/83] auto: increment master version to 17.0.146

---
 HISTORY.md | 7 +++++++
 VERSION.md | 2 +-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/HISTORY.md b/HISTORY.md
index 335f2ccc79..0b02854e5c 100644
--- a/HISTORY.md
+++ b/HISTORY.md
@@ -1,5 +1,12 @@
 # Keyman Version History
 
+## 17.0.145 alpha 2023-07-21
+
+* fix(linux): Fix logging (#9310)
+* fix(windows): open pdf in an external browser (#9295)
+* fix(linux): Fix installation of keyboards with lang tag `mul` (#9027)
+* fix(web): allows registering precached keyboards (#9304)
+
 ## 17.0.144 alpha 2023-07-20
 
 * refactor(linux): Use better way to get username (#9313)
diff --git a/VERSION.md b/VERSION.md
index 56d15fc7ec..4775541429 100644
--- a/VERSION.md
+++ b/VERSION.md
@@ -1 +1 @@
-17.0.145
\ No newline at end of file
+17.0.146
\ No newline at end of file

From 93f05d6456ce57890d92618fcb44eefb0d0e1d2c Mon Sep 17 00:00:00 2001
From: Eberhard Beilharz <eb1@sil.org>
Date: Mon, 24 Jul 2023 19:44:05 +0200
Subject: [PATCH 19/83] docs(core): Document how to build Core on Linux

Closes #8913.
---
 docs/build/linux-ubuntu.md            | 95 +++++++++++++++++----------
 docs/linux/README.md                  | 32 ++++-----
 linux/Dockerfile                      | 26 ++++++--
 linux/keyman-system-service/README.md |  6 +-
 4 files changed, 98 insertions(+), 61 deletions(-)

diff --git a/docs/build/linux-ubuntu.md b/docs/build/linux-ubuntu.md
index ee45d4eb4a..44c4056862 100644
--- a/docs/build/linux-ubuntu.md
+++ b/docs/build/linux-ubuntu.md
@@ -4,24 +4,25 @@
 
 On Linux, you can build the following projects:
 
-* [Keyman for Linux](#keyman-for-linux)
-* [Keyman Core](#keyman-core) (Linux only) (aka core)
-* [Keyman for Android](#keyman-for-android)
-<!-- TODO: document how to build for Web, Core-Wasm and Common/Web on Linux. See TC build agent for details. -->
-* Keyman Core (wasm targets)
-* Common/Web
-* KeymanWeb
+- [Keyman for Linux](#keyman-for-linux)
+- [Keyman Core](#keyman-core) (Linux only) (aka core)
+- [Keyman for Android](#keyman-for-android)
+<!-- TODO: document how to build for Web, Core-Wasm and Common/Web on Linux.
+     See TC build agent for details. -->
+- Keyman Core (wasm targets)
+- Common/Web
+- KeymanWeb
 
 The following projects **cannot** be built on Linux:
 
-* Keyman for Windows
-* Keyman Developer
-* Keyman for macOS
-* Keyman for iOS
+- Keyman for Windows
+- Keyman Developer
+- Keyman for macOS
+- Keyman for iOS
 
 ## System Requirements
 
-* Minimum Ubuntu version: Ubuntu 20.04
+- Minimum Ubuntu version: Ubuntu 20.04
 
 Other Linux distributions will also work if appropriate dependencies are installed.
 
@@ -53,21 +54,42 @@ sudo mk-build-deps --install linux/debian/control
 
 Node.js v18 is required for Core build, Web tests, and Developer command line tools.
 
+You can install it with:
+
+```shell
+curl -sL https://deb.nodesource.com/setup_18.x | bash
+apt-get -q -y install nodejs
+```
+
 ## Keyman for Linux
 
-All dependencies are already installed if you followed the instructions under [Prerequisites](#Prerequisites).
+All dependencies are already installed if you followed the instructions
+under [Prerequisites](#prerequisites).
 
-Building:
+### Building Keyman for Linux
 
-* [Building Keyman for Linux](../../linux/README.md)
+- [Building Keyman for Linux](../../linux/README.md)
 
 ## Keyman Core
 
-All dependencies are already installed if you followed the instructions under [Prerequisites](#Prerequisites).
+Most dependencies are already installed if you followed the instructions under
+[Prerequisites](#prerequisites). You'll still have to install `emscripten`:
 
-Building:
+```shell
+git clone https://github.com/emscripten-core/emsdk.git
+cd emsdk
+./emsdk install latest
+./emsdk activate latest
+source ./emsdk_env.sh
+export EMSDK_NODE=/usr/bin/node
+```
 
-* [Building Keyman Core](../../core/doc/BUILDING.md)
+*emscripten* comes with an older version of node, so it's important to set
+the `EMSDK_NODE` environment variable to the node version we need.
+
+### Building Keyman Core
+
+- [Building Keyman Core](../../core/doc/BUILDING.md)
 
 ## Docker Builder
 
@@ -86,50 +108,55 @@ Once the image is built, it may be used to build parts of Keyman.
 
 ```shell
 # build 'core' in docker
-cd ../core
+cd $(git rev-parse --show-toplevel)/core
 # keep linux build artifacts separate
 mkdir -p build/linux
-docker run -it --rm -v $(pwd)/..:/home/build -v $(pwd)/build/linux:/home/build/core/build keymanapp/keyman-linux-builder:latest bash -c 'core/build.sh --debug'
+docker run -it --rm -v $(pwd)/..:/home/build/build \
+  -v $(pwd)/build/linux:/home/build/build/core/build \
+  keymanapp/keyman-linux-builder:latest \
+  bash -c 'core/build.sh --debug'
 ```
 
 - linux
 
 ```shell
 # build 'linux' installation in docker
-cd keymanapp/keyman
-docker run -it --rm -v $(pwd):/home/build/src/keyman -w /home/build/src/keyman keymanapp/keyman-linux-builder:latest bash -c "DESTDIR=. linux/build.sh --debug build install"
+cd $(git rev-parse --show-toplevel)
+docker run -it --rm -v $(pwd):/home/build/build \
+  keymanapp/keyman-linux-builder:latest \
+  bash -c 'DESTDIR=/home/build linux/build.sh --debug build install'
 ```
 
 ## Keyman for Android
 
 **Dependencies:**
 
-* [Base](#base-dependencies)
-* [Web](./windows#web-dependencies)
+- [Base](#base-dependencies)
+- [Web](./windows#web-dependencies)
 
 **Additional requirements:**
 
-* Android SDK
-* [Android Studio](https://developer.android.com/studio/install#linux)
-* Gradle
-* Maven
-* OpenJDK 11 (for Keyman 17.0+)
-* pandoc
+- Android SDK
+- [Android Studio](https://developer.android.com/studio/install#linux)
+- Gradle
+- Maven
+- OpenJDK 11 (for Keyman 17.0+)
+- pandoc
 
 Run Android Studio once after installation to install additional components
 such as emulator images and SDK updates.
 
 **Required environment variable:**
 
-* `ANDROID_HOME` pointing to Android SDK (`$HOME/Android/Sdk`)
+- `ANDROID_HOME` pointing to Android SDK (`$HOME/Android/Sdk`)
 
 **Recommended environment variable:**
 
-* [`JAVA_HOME`](#java_home)
+- [`JAVA_HOME`](#java_home)
 
 Building:
 
-* [Building Keyman for Android](../../android/README.md)
+- [Building Keyman for Android](../../android/README.md)
 
 ## Prerequisites
 
@@ -139,7 +166,7 @@ Many dependencies are only required for specific projects.
 
 **Environment variables:**
 
-* --
+- --
 
 ## Notes on Environment Variables
 
diff --git a/docs/linux/README.md b/docs/linux/README.md
index 18e65699ad..d848e215ed 100644
--- a/docs/linux/README.md
+++ b/docs/linux/README.md
@@ -2,27 +2,18 @@
 
 ## Projects
 
-- [keyman-config](../../linux/keyman-config) - km-config and some other tools to install, uninstall
-  and view information about Keyman keyboard packages.
+- [keyman-config](../../linux/keyman-config) - `km-config` and some other tools
+  to install, uninstall and view information about Keyman keyboard packages.
 - [ibus-keyman](../../linux/ibus-keyman) - IBUS integration to use .kmp Keyman keyboards
+- [keyman-system-service](../../linux/keyman-system-service) - A DBus system service
+  that allows to perform keyboard related actions when running under Wayland.
 - [core](../../core) - common keyboardprocessor library
 
 See [license information](../../linux/LICENSE.md) about licensing.
 
 ## Linux Requirements/Setup
 
-- It is helpful to be using the [packages.sil.org](http://packages.sil.org) repo
-
-- Install packages required for building and developing Keyman for Linux.
-  The list of required packages can be seen in `linux/debian/control`.
-  It is easiest to use the `mk-build-deps` tool to install the
-  dependencies:
-
-  ```bash
-  sudo apt update
-  sudo apt install devscripts equivs
-  sudo mk-build-deps --install linux/debian/control
-  ```
+See [document in ../build](../build/linux-ubuntu.md).
 
 ## Compiling from Command Line
 
@@ -61,15 +52,18 @@ for details on building Linux packages for Keyman.
 
 ## Testing
 
-### keyman-config
-
-The unit tests can be run with the following command:
+The tests can be run with the following command:
 
 ```bash
-cd linux/keyman-config
-./run-tests.sh
+linux/build.sh test
 ```
 
+To just run the unit tests without integration tests, add the
+`--no-integration` parameter.
+
+It's also possible to only run the tests for one of the subprojects. You
+can use `build.sh` in the subdirectory for that.
+
 ### ibus-keyman
 
 If you want to run the ibus-keyman tests with Wayland, you'll have to
diff --git a/linux/Dockerfile b/linux/Dockerfile
index d6cf7d0c63..49c696cc25 100644
--- a/linux/Dockerfile
+++ b/linux/Dockerfile
@@ -1,4 +1,4 @@
-# Copyright (c) 2022 SIL International. All rights reserved.
+# Copyright (c) 2022-2023 SIL International. All rights reserved.
 #
 # builder image for a linux build
 # see ../docs/build/linux-ubuntu.md
@@ -7,12 +7,13 @@ FROM --platform=amd64 ubuntu:latest
 LABEL org.opencontainers.image.authors="SIL International."
 LABEL org.opencontainers.image.url="https://github.com/keymanapp/keyman.git"
 LABEL org.opencontainers.image.title="Keyman Linux Build Image"
+
 # We will switch to a build user after some installation
 USER root
-RUN useradd -c "Build user" -d $HOME -m build
 ENV HOME /home/build
-VOLUME /home/build
-WORKDIR /home/build
+RUN useradd -c "Build user" --home-dir $HOME --create-home --shell /usr/bin/bash build
+VOLUME /home/build/build
+WORKDIR /home/build/build
 ENV DEBIAN_FRONTEND noninteractive
 ENV DEBIAN_PRIORITY critical
 ENV DEBCONF_NOWARNINGS yes
@@ -21,13 +22,28 @@ ENV DEBCONF_NOWARNINGS yes
 RUN apt-get -q -y update &&  \
   apt-get -q -y install devscripts equivs meson python3 python3-setuptools software-properties-common && \
   add-apt-repository ppa:keymanapp/keyman && \
-  add-apt-repository ppa:keymanapp/keyman-alpha && \
+  add-apt-repository ppa:keymanapp/keyman-alpha
+RUN apt-get -q -y update &&  \
   apt-get -q -y upgrade
 
+# Install dependencies
 ADD debian/control /tmp/control
 # Answer 'yes' to install questions
 RUN (yes | mk-build-deps --install /tmp/control) || true
+
+# Install Node
 RUN curl -sL https://deb.nodesource.com/setup_18.x | bash
 RUN apt-get -q -y install nodejs
+
+# Install emscripten
+RUN cd /usr/share && \
+  git clone https://github.com/emscripten-core/emsdk.git && \
+  cd emsdk && \
+  ./emsdk install latest && \
+  ./emsdk activate latest && \
+  echo 'source "/usr/share/emsdk/emsdk_env.sh"' >> $HOME/.bashrc && \
+  echo "export EMSDK_NODE=/usr/bin/node" >> $HOME/.bashrc && \
+  echo 'echo "node $(node --version)"' >> $HOME/.bashrc
+
 # now, switch to build user
 USER build
diff --git a/linux/keyman-system-service/README.md b/linux/keyman-system-service/README.md
index 358004af8f..8b6ca2eba6 100644
--- a/linux/keyman-system-service/README.md
+++ b/linux/keyman-system-service/README.md
@@ -1,8 +1,8 @@
 # keyman-system-service
 
-A DBus system service that allows to access /dev/input/* devices
-to toggle capslock and perform other keyboard related actions when
-running under Wayland.
+A DBus system service that allows to access `/dev/input/*` devices
+to toggle capslock and perform other keyboard related actions. This is
+required when running under Wayland, but also used with X11.
 
 See <https://0pointer.net/blog/the-new-sd-bus-api-of-systemd.html>,
 <https://stackoverflow.com/a/44281937>,

From 6553be2eb692ef0c0aff6a550f8e727a62d9a313 Mon Sep 17 00:00:00 2001
From: Keyman Build Agent <buildagent@keyman.com>
Date: Mon, 24 Jul 2023 14:02:14 -0400
Subject: [PATCH 20/83] auto: increment master version to 17.0.147

---
 HISTORY.md | 4 ++++
 VERSION.md | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/HISTORY.md b/HISTORY.md
index 0b02854e5c..21e708b523 100644
--- a/HISTORY.md
+++ b/HISTORY.md
@@ -1,5 +1,9 @@
 # Keyman Version History
 
+## 17.0.146 alpha 2023-07-24
+
+* chore(deps-dev): bump word-wrap from 1.2.3 to 1.2.4 (#9314)
+
 ## 17.0.145 alpha 2023-07-21
 
 * fix(linux): Fix logging (#9310)
diff --git a/VERSION.md b/VERSION.md
index 4775541429..2b985d5cd9 100644
--- a/VERSION.md
+++ b/VERSION.md
@@ -1 +1 @@
-17.0.146
\ No newline at end of file
+17.0.147
\ No newline at end of file

From 168a98d8bd90802475964bb3481de76fc82a90e0 Mon Sep 17 00:00:00 2001
From: Eberhard Beilharz <eb1@sil.org>
Date: Mon, 24 Jul 2023 11:41:22 +0200
Subject: [PATCH 21/83] chore(linux): Update debian changelog

(cherry picked from commit 04a52e9b8e54121239132ca2b52231a204751bba)
---
 linux/debian/changelog | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/linux/debian/changelog b/linux/debian/changelog
index 9f69cc9a12..4e672643b3 100644
--- a/linux/debian/changelog
+++ b/linux/debian/changelog
@@ -1,3 +1,10 @@
+keyman (16.0.140-1) unstable; urgency=medium
+
+  * New upstream release (closes: #1037707).
+  * Re-release to Debian
+
+ -- Eberhard Beilharz <eb1@sil.org>  Mon, 24 Jul 2023 11:41:07 +0200
+
 keyman (16.0.139-4) unstable; urgency=medium
 
   * debian/tests: Revert previous change and ignore s390x from autopkgtests

From a97a966d727a4bf5beb9d33a6ef4dcc9d340afd9 Mon Sep 17 00:00:00 2001
From: Eberhard Beilharz <eb1@sil.org>
Date: Tue, 25 Jul 2023 12:39:03 +0200
Subject: [PATCH 22/83] chore(linux): Update supported Ubuntu versions

- remove Ubuntu 22.10 Kinetic which is no longer supported
- add future Ubuntu 23.10 Mantic
---
 linux/.pbuilderrc          | 2 +-
 linux/Jenkinsfile          | 4 ++--
 linux/scripts/cow.sh       | 2 +-
 linux/scripts/deb.sh       | 2 +-
 linux/scripts/launchpad.sh | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/linux/.pbuilderrc b/linux/.pbuilderrc
index 1384eec453..6a77a0ab08 100644
--- a/linux/.pbuilderrc
+++ b/linux/.pbuilderrc
@@ -14,7 +14,7 @@ DEBIAN_SUITES=($UNSTABLE_CODENAME $TESTING_CODENAME $STABLE_CODENAME $STABLE_BAC
     "experimental" "unstable" "testing" "stable")
 
 # List of Ubuntu suites. Update these when needed.
-UBUNTU_SUITES=("lunar" "kinetic" "jammy" "focal")
+UBUNTU_SUITES=("mantic" "lunar" "jammy" "focal")
 
 # Mirrors to use. Update these to your preferred mirror.
 DEBIAN_MIRROR="deb.debian.org"
diff --git a/linux/Jenkinsfile b/linux/Jenkinsfile
index b70cb95d70..1607d74d44 100644
--- a/linux/Jenkinsfile
+++ b/linux/Jenkinsfile
@@ -1,11 +1,11 @@
 #!groovy
-// Copyright (c) 2019-2022 SIL International
+// Copyright (c) 2019-2023 SIL International
 // This software is licensed under the MIT license (http://opensource.org/licenses/MIT)
 
 @Library('lsdev-pipeline-library') _
 
 keymanPackaging {
-  distributionsToPackage = 'focal jammy kinetic lunar'
+  distributionsToPackage = 'focal jammy lunar mantic'
   arches = 'amd64 i386'
   packagesToBuild = ['keyman']
 }
diff --git a/linux/scripts/cow.sh b/linux/scripts/cow.sh
index e175a666ef..201896a7e7 100755
--- a/linux/scripts/cow.sh
+++ b/linux/scripts/cow.sh
@@ -3,7 +3,7 @@
 # If needed set cowbuilder up for building Keyman Debian packages
 # Then cowbuilder update
 
-distributions='focal jammy kinetic lunar'
+distributions='focal jammy lunar mantic'
 
 if ! dpkg-query -l cowbuilder; then
     echo "installing pbuilder and cowbuilder"
diff --git a/linux/scripts/deb.sh b/linux/scripts/deb.sh
index 3c87af0de8..35a476a55c 100755
--- a/linux/scripts/deb.sh
+++ b/linux/scripts/deb.sh
@@ -11,7 +11,7 @@
 
 set -e
 
-all_distributions="focal jammy"
+all_distributions="focal jammy lunar mantic"
 distributions=""
 echo "all_distributions: ${all_distributions}"
 
diff --git a/linux/scripts/launchpad.sh b/linux/scripts/launchpad.sh
index 30495d2b0e..7437272c03 100755
--- a/linux/scripts/launchpad.sh
+++ b/linux/scripts/launchpad.sh
@@ -33,7 +33,7 @@ else
 fi
 echo "ppa: ${ppa}"
 
-distributions="${DIST:-focal jammy kinetic lunar}"
+distributions="${DIST:-focal jammy lunar mantic}"
 packageversion="${PACKAGEVERSION:-1~sil1}"
 
 BASEDIR=$(pwd)

From 77895ad9583f05a6bc266872a1dcbf252fd9e827 Mon Sep 17 00:00:00 2001
From: Keyman Build Agent <buildagent@keyman.com>
Date: Tue, 25 Jul 2023 14:02:09 -0400
Subject: [PATCH 23/83] auto: increment master version to 17.0.148

---
 HISTORY.md | 4 ++++
 VERSION.md | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/HISTORY.md b/HISTORY.md
index 21e708b523..0a6941afc2 100644
--- a/HISTORY.md
+++ b/HISTORY.md
@@ -1,5 +1,9 @@
 # Keyman Version History
 
+## 17.0.147 alpha 2023-07-25
+
+* chore(linux): Update debian changelog (#9327)
+
 ## 17.0.146 alpha 2023-07-24
 
 * chore(deps-dev): bump word-wrap from 1.2.3 to 1.2.4 (#9314)
diff --git a/VERSION.md b/VERSION.md
index 2b985d5cd9..0e228b36ba 100644
--- a/VERSION.md
+++ b/VERSION.md
@@ -1 +1 @@
-17.0.147
\ No newline at end of file
+17.0.148
\ No newline at end of file

From 0a2aa6c5cab88a4cf6d0d501831a4bd98564119b Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Tue, 25 Jul 2023 16:30:56 -0500
Subject: [PATCH 24/83] Apply suggestions from code review

Co-authored-by: Marc Durdin <marc@durdin.net>
---
 core/src/kmx/kmx_plus.cpp        |  2 +-
 core/src/kmx/kmx_xstring.h       |  4 ++--
 core/src/ldml/ldml_processor.cpp | 10 +++++-----
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/core/src/kmx/kmx_plus.cpp b/core/src/kmx/kmx_plus.cpp
index 46521961fb..61305f2c83 100644
--- a/core/src/kmx/kmx_plus.cpp
+++ b/core/src/kmx/kmx_plus.cpp
@@ -788,7 +788,7 @@ COMP_KMXPLUS_KEYS_Helper::setKeys(const COMP_KMXPLUS_KEYS *newKeys) {
       DebugLoad("<flick> %d: to=0x%X, directions=0x%X, flags=0x%X", i, e.to, e.directions, e.flags);
     }
     // now the kmap
-    DebugLog(" kmap count: #0x%X", key2->kmapCount);
+    DebugLoad(" kmap count: #0x%X", key2->kmapCount);
     for (KMX_DWORD i = 0; i < key2->kmapCount; i++) {
       DebugLoad(" #0x%d\n", i);
       auto &entry = kmap[i];
diff --git a/core/src/kmx/kmx_xstring.h b/core/src/kmx/kmx_xstring.h
index 06fb8a0758..24f02c54da 100644
--- a/core/src/kmx/kmx_xstring.h
+++ b/core/src/kmx/kmx_xstring.h
@@ -47,9 +47,9 @@ namespace kmx {
 #define Uni_UTF32ToSurrogate1(ch) (char16_t)(((ch) - 0x10000) / 0x400 + 0xD800)
 #define Uni_UTF32ToSurrogate2(ch) (char16_t)(((ch) - 0x10000) % 0x400 + 0xDC00)
 
-#define Uni_IsNoncharacter(ch) ((ch >= 0xFDD0 && ch <= 0xFDEF) || ((ch & 0xFFFE) == 0xFFFE))
+#define Uni_IsNoncharacter(ch) (((ch) >= 0xFDD0 && (ch) <= 0xFDEF) || (((ch) & 0xFFFE) == 0xFFFE))
 
-#define Uni_InCodespace(ch) (ch <= 0x10FFFF)
+#define Uni_InCodespace(ch) ((ch) <= 0x10FFFF)
 
 /**
  * @brief True if in codespace and NOT a surrogate or noncharacter.
diff --git a/core/src/ldml/ldml_processor.cpp b/core/src/ldml/ldml_processor.cpp
index 9e341e62ef..66a03836bb 100644
--- a/core/src/ldml/ldml_processor.cpp
+++ b/core/src/ldml/ldml_processor.cpp
@@ -279,8 +279,8 @@ ldml_processor::process_event(
 
           // TODO-LDML: unroll ctxt into a str. Would be better to have transforms be able to process a vector
           std::u32string ctxtstr;
-          for (size_t i = 0; i < ctxt.size(); i++) {
-            ctxtstr.append(ctxt[i]);
+          for (const auto &ch : ctxt) {
+            ctxtstr.append(ch);
           }
           const size_t matchedContext = transforms->apply(ctxtstr, outputString);
 
@@ -293,9 +293,9 @@ ldml_processor::process_event(
               state->actions().push_backspace(KM_KBP_BT_CHAR, deletedChar);  // Cause prior char to be removed
             }
             // Now, add in the updated text
-            for (size_t i = 0; i < outputString.length(); i++) {
-              state->context().push_character(outputString[i]);
-              state->actions().push_character(outputString[i]);
+            for (const auto &ch : outputString) {
+              state->context().push_character(ch);
+              state->actions().push_character(ch);
             }
           }
         }

From 84dc28f512e89628007605fb9c504eb038bdb575 Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Tue, 25 Jul 2023 16:32:42 -0500
Subject: [PATCH 25/83] =?UTF-8?q?feat(core):=20transform/reorder=20=20?=
 =?UTF-8?q?=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- remove another debug print call

#7375
---
 core/src/ldml/ldml_transforms.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/core/src/ldml/ldml_transforms.cpp b/core/src/ldml/ldml_transforms.cpp
index 95565feb0d..d791e00618 100644
--- a/core/src/ldml/ldml_transforms.cpp
+++ b/core/src/ldml/ldml_transforms.cpp
@@ -207,8 +207,10 @@ element_list::update_sort_key(size_t offset, std::deque<reorder_sort_key> &key)
     assert(e->matches(k.ch));        // double check that this element matches
     k.primary  = e->get_order();
     k.tertiary = e->get_tertiary();  // TODO-LDML: need more detailed tertiary work
+#if KMXPLUS_DEBUG_TRANSFORM
     DebugTran("Updating at +%d", c);
     k.dump();
+#endif
     c++;
   }
   return key;

From 2f381279af47f8185e86df1e2459677a27e67de5 Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Tue, 25 Jul 2023 17:52:19 -0500
Subject: [PATCH 26/83] =?UTF-8?q?chore(developer):=20make=20unknown=20vkey?=
 =?UTF-8?q?=20a=20hint,=20not=20error=20=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Note that feat(core): ldml vkey support 🙀 #7135 is future,
in the future this may be an Error again.

- also, add an internal error to the visual-keyboard-compiler on missing keys
- also, update fr-t-k0-azerty to fix a missing key (simultaneous fix has been made in CLDR)
- also, reinstate CLDR's fr-t-k0-azerty - now all 'stock' keyboards build in keyman!
- keep the 'internal' azerty as k_020 - it has some additional transform goodies

Fixes: #9236
---
 .../{fr-t-k0-azerty-test.xml => k_020_fr-test.xml} |  5 +++--
 .../keyboards/{fr-t-k0-azerty.xml => k_020_fr.xml} | 13 +++++++------
 core/tests/unit/ldml/keyboards/meson.build         |  4 ++--
 developer/src/kmc-ldml/src/compiler/messages.ts    |  8 ++++----
 .../src/compiler/visual-keyboard-compiler.ts       |  4 ++++
 developer/src/kmc-ldml/src/compiler/vkey.ts        |  8 ++++----
 developer/src/kmc-ldml/test/test-vkey.ts           | 14 +++++++-------
 .../techpreview/3.0/fr-t-k0-azerty.xml             |  3 ---
 8 files changed, 31 insertions(+), 28 deletions(-)
 rename core/tests/unit/ldml/keyboards/{fr-t-k0-azerty-test.xml => k_020_fr-test.xml} (85%)
 rename core/tests/unit/ldml/keyboards/{fr-t-k0-azerty.xml => k_020_fr.xml} (94%)

diff --git a/core/tests/unit/ldml/keyboards/fr-t-k0-azerty-test.xml b/core/tests/unit/ldml/keyboards/k_020_fr-test.xml
similarity index 85%
rename from core/tests/unit/ldml/keyboards/fr-t-k0-azerty-test.xml
rename to core/tests/unit/ldml/keyboards/k_020_fr-test.xml
index 21f9d64cc4..04066ae220 100644
--- a/core/tests/unit/ldml/keyboards/fr-t-k0-azerty-test.xml
+++ b/core/tests/unit/ldml/keyboards/k_020_fr-test.xml
@@ -1,10 +1,11 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!-- Note: similar to, but slightly ahead of, fr-t-k0-azerty -->
 <!DOCTYPE keyboardTest SYSTEM "../../../../../resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboardTest.dtd">
 <keyboardTest conformsTo="techpreview">
-    <info keyboard="fr-t-k0-azerty.xml" author="Team Keyboard" name="fr-test" />
+    <info keyboard="k_020_fr.xml" author="Team Keyboard" name="fr-test-updated" />
     <repertoire name="simple-repertoire" chars="[a b c d e \u0022]" type="simple" /> <!-- verify that these outputs are all available from simple keys on any layer, for all form factors -->
     <repertoire name="chars-repertoire" chars="[á é ó]" type="gesture" /> <!-- verify that these outputs are all available from simple or gesture keys on any layer, for touch -->
-    <tests name="key-tests">
+    <tests name="key-tests-updated">
         <test name="key-test">
             <startContext to="abc\u0022..."/>
             <!-- tests by pressing key ids -->
diff --git a/core/tests/unit/ldml/keyboards/fr-t-k0-azerty.xml b/core/tests/unit/ldml/keyboards/k_020_fr.xml
similarity index 94%
rename from core/tests/unit/ldml/keyboards/fr-t-k0-azerty.xml
rename to core/tests/unit/ldml/keyboards/k_020_fr.xml
index 8a634bda65..1f6fc2c32f 100644
--- a/core/tests/unit/ldml/keyboards/fr-t-k0-azerty.xml
+++ b/core/tests/unit/ldml/keyboards/k_020_fr.xml
@@ -1,4 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!-- Note: similar to, but slightly ahead of, fr-t-k0-azerty -->
 <!DOCTYPE keyboard SYSTEM "../../../../../resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboard.dtd">
 <keyboard locale="fr-t-k0-azerty" conformsTo="techpreview">
 	<locales>
@@ -7,11 +8,11 @@
 	<!-- 'conformsTo' gives CLDR spec conformance. Distinguishes from prior
 		keyboard formats -->
 	<!-- 'version' element is now optional -->
-	<version number="1.0.0" />
+	<version number="1.0.1" />
 	<info author="Team Keyboard" normalization="NFC" layout="AZERTY" indicator="FR" />
 	<names>
-		<name value="French Test" />
-		<name value="French Test AZERTY" />
+		<name value="French Test Updated" />
+		<name value="French Test AZERTY Updated" />
 	</names>
 
 	<settings fallback="omit" /> <!-- "hide" makes this act like a dead keyTBD -->
@@ -22,15 +23,15 @@
     <vkey from="A" to="Q" />
     <vkey from="Z" to="W" />
     <vkey from="GRAVE" to="QUOTE" /> <!-- TODO-LDML: GRAVE=BKQUOTE in ldml? -->
-    <!-- <vkey from="MINUS" to="LBRACKET" /> --> <!-- TODO-LDML: missing MINUS, LBRKT=LBRACKET in ldml? -->
+    <vkey from="MINUS" to="LBRACKET" /> <!-- TODO-LDML: missing MINUS, LBRKT=LBRACKET in ldml? -->
     <!-- i.e., OEM_3 >>> OEM_7 -->
     <vkey from="LBRACKET" to="RBRACKET" />  <!-- TODO-LDML: RBRKT=RBRACKET in ldml? -->
-    <!-- <vkey from="COLON" to="M" /> --> <!-- TODO-LDML: missing COLON-->
+    <vkey from="COLON" to="M" /> <!-- TODO-LDML: missing COLON-->
     <vkey from="QUOTE" to="GRAVE" /> <!-- TODO-LDML: GRAVE=BKQUOTE in ldml? -->
     <vkey from="M" to="COMMA" />
 		<vkey from="COMMA" to="PERIOD" />
 		<vkey from="PERIOD" to="SLASH" />
-    <!-- <vkey from="SLASH" to="EXCLAMATION" />  --> <!-- TODO-LDML: missing: EXCLAMATION -->
+    <vkey from="SLASH" to="EXCLAMATION" />  <!-- TODO-LDML: missing: EXCLAMATION -->
     <!-- i.e., OEM_2 >>> OEM_8 -->
 	</vkeys>
 
diff --git a/core/tests/unit/ldml/keyboards/meson.build b/core/tests/unit/ldml/keyboards/meson.build
index b4ba719548..93ccfaad63 100644
--- a/core/tests/unit/ldml/keyboards/meson.build
+++ b/core/tests/unit/ldml/keyboards/meson.build
@@ -10,7 +10,7 @@
 tests_from_cldr = [
   'ja-Latn',
   'pt-k0-abnt2',
-  # 'fr-t-k0-azerty', # vkey issues
+  'fr-t-k0-azerty',
 ]
 
 tests_without_testdata = [
@@ -32,7 +32,7 @@ tests_without_testdata = [
 # These tests have a k_001_tiny-test.xml file as well.
 tests_with_testdata = [
   'k_001_tiny',
-  'fr-t-k0-azerty', # TODO-LDML: move to cldr above (fix vkey)
+  'k_020_fr', # TODO-LDML: move to cldr above (fix vkey)
   'k_200_reorder_nod_Lana',
 ]
 
diff --git a/developer/src/kmc-ldml/src/compiler/messages.ts b/developer/src/kmc-ldml/src/compiler/messages.ts
index f0feb6f599..a6a8b4ff49 100644
--- a/developer/src/kmc-ldml/src/compiler/messages.ts
+++ b/developer/src/kmc-ldml/src/compiler/messages.ts
@@ -2,7 +2,7 @@ import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m
 
 const SevInfo = CompilerErrorSeverity.Info | CompilerErrorNamespace.LdmlKeyboardCompiler;
 const SevHint = CompilerErrorSeverity.Hint | CompilerErrorNamespace.LdmlKeyboardCompiler;
-// const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.KeyboardCompiler;
+// const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.LdmlKeyboardCompiler;
 const SevError = CompilerErrorSeverity.Error | CompilerErrorNamespace.LdmlKeyboardCompiler;
 const SevFatal = CompilerErrorSeverity.Fatal | CompilerErrorNamespace.LdmlKeyboardCompiler;
 
@@ -35,9 +35,9 @@ export class CompilerMessages {
     m(this.HINT_LocaleIsNotMinimalAndClean, `Locale '${o.sourceLocale}' is not minimal or correctly formatted and should be '${o.locale}'`);
   static HINT_LocaleIsNotMinimalAndClean = SevHint | 0x0008;
 
-  static Error_VkeyIsNotValid = (o:{vkey: string}) =>
-    m(this.ERROR_VkeyIsNotValid, `Virtual key '${o.vkey}' is not found in the CLDR VKey Enum table.`);
-  static ERROR_VkeyIsNotValid = SevError | 0x0009;
+  static Hint_VkeyIsNotValid = (o:{vkey: string}) =>
+    m(this.HINT_VkeyIsNotValid, `Virtual key '${o.vkey}' is not found in the CLDR VKey Enum table.`);
+  static HINT_VkeyIsNotValid = SevHint | 0x0009;
 
   static Hint_VkeyIsRedundant = (o:{vkey: string}) =>
     m(this.HINT_VkeyIsRedundant, `Virtual key '${o.vkey}' is mapped to itself, which is redundant.`);
diff --git a/developer/src/kmc-ldml/src/compiler/visual-keyboard-compiler.ts b/developer/src/kmc-ldml/src/compiler/visual-keyboard-compiler.ts
index d951080b9b..0d0c802095 100644
--- a/developer/src/kmc-ldml/src/compiler/visual-keyboard-compiler.ts
+++ b/developer/src/kmc-ldml/src/compiler/visual-keyboard-compiler.ts
@@ -44,6 +44,10 @@ export class LdmlKeyboardVisualKeyboardCompiler {
 
         let keydef = source.keyboard.keys?.key?.find(x => x.id == key);
 
+        if (!keydef) {
+          throw Error(`Internal Error: could not find key id="${key}" in layer "${layer.id || '<none>'}", row "${y}"`);
+        }
+
         vk.keys.push({
           flags: VisualKeyboard.VisualKeyboardKeyFlags.kvkkUnicode,
           shift: shift,
diff --git a/developer/src/kmc-ldml/src/compiler/vkey.ts b/developer/src/kmc-ldml/src/compiler/vkey.ts
index 1b7d71f300..79967e1d3b 100644
--- a/developer/src/kmc-ldml/src/compiler/vkey.ts
+++ b/developer/src/kmc-ldml/src/compiler/vkey.ts
@@ -19,13 +19,13 @@ export class VkeyCompiler extends SectionCompiler {
 
       this.keyboard.vkeys.vkey.forEach(vk => {
         if(LdmlVkeyNames[vk.from] === undefined) {
-          this.callbacks.reportMessage(CompilerMessages.Error_VkeyIsNotValid({vkey: vk.from}));
-          valid = false;
+          this.callbacks.reportMessage(CompilerMessages.Hint_VkeyIsNotValid({vkey: vk.from}));
+          return;
         }
 
         if(LdmlVkeyNames[vk.to] === undefined) {
-          this.callbacks.reportMessage(CompilerMessages.Error_VkeyIsNotValid({vkey: vk.to}));
-          valid = false;
+          this.callbacks.reportMessage(CompilerMessages.Hint_VkeyIsNotValid({vkey: vk.to}));
+          return;
         }
 
         if(vk.from == vk.to) {
diff --git a/developer/src/kmc-ldml/test/test-vkey.ts b/developer/src/kmc-ldml/test/test-vkey.ts
index 03c27cbd3f..dec24125e2 100644
--- a/developer/src/kmc-ldml/test/test-vkey.ts
+++ b/developer/src/kmc-ldml/test/test-vkey.ts
@@ -37,19 +37,19 @@ describe('vkey compiler', function () {
     assert.deepEqual(compilerTestCallbacks.messages[0], CompilerMessages.Info_MultipleVkeysHaveSameTarget({vkey: 'Q'}));
   });
 
-  it('should error on invalid "from" vkey', async function() {
+  it('should hint on invalid "from" vkey', async function() {
     let vkey = await loadSectionFixture(VkeyCompiler, 'sections/vkey/invalid-from-vkey.xml', compilerTestCallbacks) as Vkey;
-    assert.isNull(vkey);
+    assert.isNotNull(vkey);
     assert.equal(compilerTestCallbacks.messages.length, 2);
-    assert.deepEqual(compilerTestCallbacks.messages[0], CompilerMessages.Error_VkeyIsNotValid({vkey: 'q'}));
-    assert.deepEqual(compilerTestCallbacks.messages[1], CompilerMessages.Error_VkeyIsNotValid({vkey: 'HYFEN'}));
+    assert.deepEqual(compilerTestCallbacks.messages[0], CompilerMessages.Hint_VkeyIsNotValid({vkey: 'q'}));
+    assert.deepEqual(compilerTestCallbacks.messages[1], CompilerMessages.Hint_VkeyIsNotValid({vkey: 'HYFEN'}));
   });
 
-  it('should error on invalid "to" vkey', async function() {
+  it('should hint on invalid "to" vkey', async function() {
     let vkey = await loadSectionFixture(VkeyCompiler, 'sections/vkey/invalid-to-vkey.xml', compilerTestCallbacks) as Vkey;
-    assert.isNull(vkey);
+    assert.isNotNull(vkey);
     assert.equal(compilerTestCallbacks.messages.length, 1);
-    assert.deepEqual(compilerTestCallbacks.messages[0], CompilerMessages.Error_VkeyIsNotValid({vkey: 'A-ACUTE'}));
+    assert.deepEqual(compilerTestCallbacks.messages[0], CompilerMessages.Hint_VkeyIsNotValid({vkey: 'A-ACUTE'}));
   });
 
   it('should error on repeated vkeys', async function() {
diff --git a/resources/standards-data/ldml-keyboards/techpreview/3.0/fr-t-k0-azerty.xml b/resources/standards-data/ldml-keyboards/techpreview/3.0/fr-t-k0-azerty.xml
index 7a1916d1df..156a195c05 100644
--- a/resources/standards-data/ldml-keyboards/techpreview/3.0/fr-t-k0-azerty.xml
+++ b/resources/standards-data/ldml-keyboards/techpreview/3.0/fr-t-k0-azerty.xml
@@ -63,12 +63,9 @@
 		<key id="symbol" switch="symbol" />
 		<key id="base" switch="base" />
 
-		<!--
-			TODO: need discussion
 		<key id="bksp" gap="true" />
 		<key id="extra" gap="true" />
 		<key id="enter" to="\u{000A}" />
-		-->
 
 		<!-- extra keys -->
 		<key id="u-grave" to="ü" />

From 7ee574b8ba223392aa6433f0c920f70939f43f98 Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Tue, 25 Jul 2023 17:53:58 -0500
Subject: [PATCH 27/83] =?UTF-8?q?chore(developer):=20make=20unknown=20vkey?=
 =?UTF-8?q?=20a=20hint,=20not=20error=20=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- add a TODO mentioning #7135

Fixes: #9236
---
 developer/src/kmc-ldml/src/compiler/vkey.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/developer/src/kmc-ldml/src/compiler/vkey.ts b/developer/src/kmc-ldml/src/compiler/vkey.ts
index 79967e1d3b..2d6d44d9d0 100644
--- a/developer/src/kmc-ldml/src/compiler/vkey.ts
+++ b/developer/src/kmc-ldml/src/compiler/vkey.ts
@@ -19,11 +19,13 @@ export class VkeyCompiler extends SectionCompiler {
 
       this.keyboard.vkeys.vkey.forEach(vk => {
         if(LdmlVkeyNames[vk.from] === undefined) {
+          // TODO-LDML: When we do #7135 this may need to change back to an error.
           this.callbacks.reportMessage(CompilerMessages.Hint_VkeyIsNotValid({vkey: vk.from}));
           return;
         }
 
         if(LdmlVkeyNames[vk.to] === undefined) {
+          // TODO-LDML: When we do #7135 this may need to change back to an error.
           this.callbacks.reportMessage(CompilerMessages.Hint_VkeyIsNotValid({vkey: vk.to}));
           return;
         }

From 352328fbbe48eff7b9baf8f53a5d051df927922f Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Tue, 25 Jul 2023 18:42:29 -0500
Subject: [PATCH 28/83] =?UTF-8?q?fix(core):=20Better=20range=20check=20for?=
 =?UTF-8?q?=20Uni=5FIsValid()=20=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- rewrite macros as inlines
- brute force approach

For: #7375
---
 core/src/kmx/kmx_plus.cpp                   |  4 +-
 core/src/kmx/kmx_xstring.h                  | 46 ++++++++++++++++++---
 core/tests/unit/kmnkbd/test_kmx_xstring.cpp |  6 ++-
 3 files changed, 47 insertions(+), 9 deletions(-)

diff --git a/core/src/kmx/kmx_plus.cpp b/core/src/kmx/kmx_plus.cpp
index 61305f2c83..b02fec77d8 100644
--- a/core/src/kmx/kmx_plus.cpp
+++ b/core/src/kmx/kmx_plus.cpp
@@ -1045,7 +1045,7 @@ COMP_KMXPLUS_USET_Helper::setUset(const COMP_KMXPLUS_USET *newUset) {
       KMX_DWORD lastEnd = 0x0;
       for (KMX_DWORD r = 0; is_valid && r < e.count; r++) {
         const auto &range = ranges[e.range + r];  // already range-checked 'r' above
-        if (!Uni_IsValid(range.start) || !Uni_IsValid(range.end)) {
+        if (!Uni_IsValid(range.start, range.end)) {
           DebugLog("uset[%d][%d] not valid: [U+%04X-U+%04X]", i, r, range.start, range.end);
           is_valid = false;
           assert(is_valid);
@@ -1094,7 +1094,7 @@ bool
 USet::valid() const {
   // double check
   for (const auto &range : ranges) {
-    if (!Uni_IsValid(range.start) || !Uni_IsValid(range.end)) {
+    if (!Uni_IsValid(range.start, range.end)) {
       DebugLog("Invalid UnicodeSet (contains noncharacters): [U+%04X,U+%04X]", (int)range.start, (int)range.end);
       return false;
     }
diff --git a/core/src/kmx/kmx_xstring.h b/core/src/kmx/kmx_xstring.h
index 24f02c54da..4572a276ad 100644
--- a/core/src/kmx/kmx_xstring.h
+++ b/core/src/kmx/kmx_xstring.h
@@ -47,15 +47,21 @@ namespace kmx {
 #define Uni_UTF32ToSurrogate1(ch) (char16_t)(((ch) - 0x10000) / 0x400 + 0xD800)
 #define Uni_UTF32ToSurrogate2(ch) (char16_t)(((ch) - 0x10000) % 0x400 + 0xDC00)
 
-#define Uni_IsNoncharacter(ch) (((ch) >= 0xFDD0 && (ch) <= 0xFDEF) || (((ch) & 0xFFFE) == 0xFFFE))
-
-#define Uni_InCodespace(ch) ((ch) <= 0x10FFFF)
+/**
+ * @returns true if the character is a noncharacter
+*/
+bool Uni_IsNonCharacter(km_kbp_usv ch);
 
 /**
- * @brief True if in codespace and NOT a surrogate or noncharacter.
- * \def Uni_IsValid
+ * @returns true if the character is a valid Unicode code point
 */
-#define Uni_IsValid(ch) (Uni_InCodespace(ch) && !Uni_IsSurrogate(ch) && !Uni_IsNoncharacter(ch))
+bool Uni_IsValid(km_kbp_usv ch);
+
+/**
+ * @returns true if the character is a valid Unicode code point range, that is, [start-end] are all
+ * valid.
+*/
+bool Uni_IsValid(km_kbp_usv start, km_kbp_usv range);
 
 /**
  * char16_t array big enough to hold a single Unicode codepoint,
@@ -149,6 +155,34 @@ u16string_to_u32string(const std::u16string &source) {
   return out;
 }
 
+
+inline bool Uni_IsNoncharacter(km_kbp_usv ch) {
+  return (((ch) >= 0xFDD0 && (ch) <= 0xFDEF) || (((ch) & 0xFFFE) == 0xFFFE));
+}
+
+inline bool Uni_InCodespace(km_kbp_usv ch) {
+  return ((ch) <= 0x10FFFF);
+};
+
+inline bool Uni_IsValid(km_kbp_usv ch) {
+  return (Uni_InCodespace(ch) && !Uni_IsSurrogate(ch) && !Uni_IsNoncharacter(ch));
+}
+
+inline bool Uni_IsValid(km_kbp_usv start, km_kbp_usv end) {
+  // quicker check
+  if (!Uni_IsValid(start) || !Uni_IsValid(end)) {
+    return false;
+  }
+
+  // brute force it
+  for (km_kbp_usv i = start; i <= end; i++) {
+    if (!Uni_IsValid(i)) return false;
+  }
+
+  return true;
+}
+
+
 } // namespace kmx
 } // namespace kbp
 } // namespace km
diff --git a/core/tests/unit/kmnkbd/test_kmx_xstring.cpp b/core/tests/unit/kmnkbd/test_kmx_xstring.cpp
index 38e3468449..aedd249b8c 100644
--- a/core/tests/unit/kmnkbd/test_kmx_xstring.cpp
+++ b/core/tests/unit/kmnkbd/test_kmx_xstring.cpp
@@ -1348,7 +1348,11 @@ void test_is_valid() {
   assert_equal(Uni_IsValid(0x02FFFF), false); // nonchar
   assert_equal(Uni_IsValid(0x02FFFE), false); // nonchar
   assert_equal(Uni_IsValid(0xFDD1), false); // nonchar
-  assert_equal(Uni_IsValid(0xFDD0), false); // nonchar  
+  assert_equal(Uni_IsValid(0xFDD0), false); // nonchar
+
+  // range test
+  assert_equal(Uni_IsValid(0, 0x10FFFF), false); // ends with nonchar
+  assert_equal(Uni_IsValid(0, 0x10FFFD), false); // contains lots o' nonchars
 }
 
 constexpr const auto help_str = u"\

From 480fd3bcadd4d52982017fc659dfc18893a527b9 Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Tue, 25 Jul 2023 19:02:46 -0500
Subject: [PATCH 29/83] =?UTF-8?q?fix(core):=20More=20range=20checks=20for?=
 =?UTF-8?q?=20Uni=5FIsValid()=20=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

For: #7375
---
 core/src/kmx/kmx_xstring.h                  |  7 +++--
 core/tests/unit/kmnkbd/test_kmx_xstring.cpp | 32 ++++++++++++++++++---
 2 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/core/src/kmx/kmx_xstring.h b/core/src/kmx/kmx_xstring.h
index 4572a276ad..e86724d037 100644
--- a/core/src/kmx/kmx_xstring.h
+++ b/core/src/kmx/kmx_xstring.h
@@ -155,9 +155,12 @@ u16string_to_u32string(const std::u16string &source) {
   return out;
 }
 
+inline bool Uni_IsEndOfPlaneNonCharacter(km_kbp_usv ch) {
+  return (((ch) & 0xFFFE) == 0xFFFE);
+}
 
 inline bool Uni_IsNoncharacter(km_kbp_usv ch) {
-  return (((ch) >= 0xFDD0 && (ch) <= 0xFDEF) || (((ch) & 0xFFFE) == 0xFFFE));
+  return (((ch) >= 0xFDD0 && (ch) <= 0xFDEF) || Uni_IsEndOfPlaneNonCharacter(ch));
 }
 
 inline bool Uni_InCodespace(km_kbp_usv ch) {
@@ -170,7 +173,7 @@ inline bool Uni_IsValid(km_kbp_usv ch) {
 
 inline bool Uni_IsValid(km_kbp_usv start, km_kbp_usv end) {
   // quicker check
-  if (!Uni_IsValid(start) || !Uni_IsValid(end)) {
+  if (!Uni_IsValid(end) || (end < start)) { // start is checked below
     return false;
   }
 
diff --git a/core/tests/unit/kmnkbd/test_kmx_xstring.cpp b/core/tests/unit/kmnkbd/test_kmx_xstring.cpp
index aedd249b8c..e3c4077622 100644
--- a/core/tests/unit/kmnkbd/test_kmx_xstring.cpp
+++ b/core/tests/unit/kmnkbd/test_kmx_xstring.cpp
@@ -1350,10 +1350,34 @@ void test_is_valid() {
   assert_equal(Uni_IsValid(0xFDD1), false); // nonchar
   assert_equal(Uni_IsValid(0xFDD0), false); // nonchar
 
-  // range test
-  assert_equal(Uni_IsValid(0, 0x10FFFF), false); // ends with nonchar
-  assert_equal(Uni_IsValid(0, 0x10FFFD), false); // contains lots o' nonchars
-}
+
+ // positive range test
+  assert_equal(Uni_IsValid(0x100000, 0x10FFFD), true);
+  assert_equal(Uni_IsValid(0x10, 0x20), true);
+  assert_equal(Uni_IsValid(0x100000, 0x10FFFD), true);
+
+  // all valid ranges in BMP
+  assert_equal(Uni_IsValid(0x0000, 0xD7FF), true);
+  assert_equal(Uni_IsValid(0xD800, 0xDFFF), false);
+  assert_equal(Uni_IsValid(0xE000, 0xFDCF), true);
+  assert_equal(Uni_IsValid(0xFDD0, 0xFDEF), false);
+  assert_equal(Uni_IsValid(0xFDF0, 0xFDFF), true);
+  assert_equal(Uni_IsValid(0xFDF0, 0xFFFD), true);
+
+  // negative range test
+  assert_equal(Uni_IsValid(0, 0x10FFFF), false);         // ends with nonchar
+  assert_equal(Uni_IsValid(0, 0x10FFFD), false);         // contains lots o' nonchars
+  assert_equal(Uni_IsValid(0x20, 0x10), false);          // swapped
+  assert_equal(Uni_IsValid(0xFDEF, 0xFDF0), false);      // just outside range
+  assert_equal(Uni_IsValid(0x0000, 0x010000), false);    // crosses noncharacter plane boundary and other stuff
+  assert_equal(Uni_IsValid(0x010000, 0x020000), false);  // crosses noncharacter plane boundary
+  assert_equal(Uni_IsValid(0x0000, 0xFFFF), false);      // crosses other BMP prohibited and plane boundary
+  assert_equal(Uni_IsValid(0x0000, 0xFFFD), false);      // crosses other BMP prohibited
+  assert_equal(Uni_IsValid(0x0000, 0xE000), false);      // crosses surrogate space
+  assert_equal(Uni_IsValid(0x0000, 0x20FFFF), false);      // out of bounds
+  assert_equal(Uni_IsValid(0x10FFFD, 0x20FFFF), false);      // out of bounds
+
+ }
 
 constexpr const auto help_str = u"\
 test_kmx_xstring [--color]\n\

From 97b312082971b232cc7cedb997d872e19a4e05cc Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Wed, 26 Jul 2023 14:50:49 -0500
Subject: [PATCH 30/83] =?UTF-8?q?fix(core):=20Optimize=20checks=20for=20Un?=
 =?UTF-8?q?i=5FIsValid()=20=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

For: #7375
---
 core/src/kmx/kmx_xstring.h | 57 ++++++++++++++++++++++++++++----------
 1 file changed, 43 insertions(+), 14 deletions(-)

diff --git a/core/src/kmx/kmx_xstring.h b/core/src/kmx/kmx_xstring.h
index e86724d037..e16ee7f78f 100644
--- a/core/src/kmx/kmx_xstring.h
+++ b/core/src/kmx/kmx_xstring.h
@@ -6,16 +6,30 @@ namespace km {
 namespace kbp {
 namespace kmx {
 
+const char16_t Uni_LEAD_SURROGATE_START  = 0xD800;
+const char16_t Uni_LEAD_SURROGATE_END    = 0xDBFF;
+const char16_t Uni_TRAIL_SURROGATE_START = 0xDC00;
+const char16_t Uni_TRAIL_SURROGATE_END   = 0xDFFF;
+const char16_t Uni_SURROGATE_START       = Uni_LEAD_SURROGATE_START;
+const char16_t Uni_SURROGATE_END         = Uni_TRAIL_SURROGATE_END;
+const char16_t Uni_FD_NONCHARACTER_START = 0xFDD0;
+const char16_t Uni_FD_NONCHARACTER_END   = 0xFDEF;
+const char16_t Uni_FFFE_NONCHARACTER     = 0xFFFE;
+const char16_t Uni_FFFF_NONCHARACTER     = 0xFFFF;
+const char16_t Uni_BMP_END               = 0xFFFF;
+const km_kbp_usv Uni_SMP_START           = 0x010000;
+const km_kbp_usv Uni_PLANE_MASK          = 0x1F0000;
+
 /**
  * @brief True if a lead surrogate
  * \def Uni_IsSurrogate1
  */
-#define Uni_IsSurrogate1(ch) ((ch) >= 0xD800 && (ch) <= 0xDBFF)
+#define Uni_IsSurrogate1(ch) ((ch) >= km::kbp::kmx::Uni_LEAD_SURROGATE_START && (ch) <= km::kbp::kmx::Uni_LEAD_SURROGATE_END)
 /**
  * @brief True if a trail surrogate
  * \def Uni_IsSurrogate2
  */
-#define Uni_IsSurrogate2(ch) ((ch) >= 0xDC00 && (ch) <= 0xDFFF)
+#define Uni_IsSurrogate2(ch) ((ch) >= km::kbp::kmx::Uni_TRAIL_SURROGATE_START && (ch) <= km::kbp::kmx::Uni_TRAIL_SURROGATE_END)
 
 /**
  * @brief True if any surrogate
@@ -27,7 +41,7 @@ namespace kmx {
  * @brief Returns true if BMP (Plane 0)
  * \def Uni_IsBMP
  */
-#define Uni_IsBMP(ch) ((ch) < 0x10000)
+#define Uni_IsBMP(ch) ((ch) <=  km::kbp::kmx::Uni_BMP_END)
 
 /**
  * @brief Convert two UTF-16 surrogates into one UTF-32 codepoint
@@ -35,17 +49,17 @@ namespace kmx {
  * @param cl trail surrogate - Uni_IsSurrogate2(cl) must == true
  * \def Uni_SurrogateToUTF
  */
-#define Uni_SurrogateToUTF32(ch, cl) (((ch) - 0xD800) * 0x400 + ((cl) - 0xDC00) + 0x10000)
+#define Uni_SurrogateToUTF32(ch, cl) (((ch) - km::kbp::kmx::Uni_LEAD_SURROGATE_START) * 0x400 + ((cl) - km::kbp::kmx::Uni_TRAIL_SURROGATE_START) + km::kbp::kmx::Uni_SMP_START)
 
 /**
  * @brief Convert UTF-32 BMP to UTF-16 BMP
  * @param ch codepoint - Uni_IsBMP(ch) must == true
  * \def Uni_UTF32BMPToUTF16
  */
-#define Uni_UTF32BMPToUTF16(ch) (ch & 0xFFFF)
+#define Uni_UTF32BMPToUTF16(ch) ((ch) & Uni_FFFF_NONCHARACTER)
 
-#define Uni_UTF32ToSurrogate1(ch) (char16_t)(((ch) - 0x10000) / 0x400 + 0xD800)
-#define Uni_UTF32ToSurrogate2(ch) (char16_t)(((ch) - 0x10000) % 0x400 + 0xDC00)
+#define Uni_UTF32ToSurrogate1(ch) (char16_t)(((ch) - km::kbp::kmx::Uni_SMP_START) / 0x400 + km::kbp::kmx::Uni_LEAD_SURROGATE_START)
+#define Uni_UTF32ToSurrogate2(ch) (char16_t)(((ch) - km::kbp::kmx::Uni_SMP_START) % 0x400 + km::kbp::kmx::Uni_TRAIL_SURROGATE_START)
 
 /**
  * @returns true if the character is a noncharacter
@@ -53,13 +67,15 @@ namespace kmx {
 bool Uni_IsNonCharacter(km_kbp_usv ch);
 
 /**
- * @returns true if the character is a valid Unicode code point
+ * @returns true if the character is a valid Unicode code point.
+ * Surrogates belong to UTF-16 and are invalid.
 */
 bool Uni_IsValid(km_kbp_usv ch);
 
 /**
  * @returns true if the character is a valid Unicode code point range, that is, [start-end] are all
  * valid.
+ * Surrogates belong to UTF-16 and are invalid.
 */
 bool Uni_IsValid(km_kbp_usv start, km_kbp_usv range);
 
@@ -156,11 +172,11 @@ u16string_to_u32string(const std::u16string &source) {
 }
 
 inline bool Uni_IsEndOfPlaneNonCharacter(km_kbp_usv ch) {
-  return (((ch) & 0xFFFE) == 0xFFFE);
+  return (((ch) & Uni_FFFE_NONCHARACTER) == Uni_FFFE_NONCHARACTER); // matches FFFF or FFFE
 }
 
 inline bool Uni_IsNoncharacter(km_kbp_usv ch) {
-  return (((ch) >= 0xFDD0 && (ch) <= 0xFDEF) || Uni_IsEndOfPlaneNonCharacter(ch));
+  return (((ch) >= Uni_FD_NONCHARACTER_START && (ch) <= Uni_FD_NONCHARACTER_END) || Uni_IsEndOfPlaneNonCharacter(ch));
 }
 
 inline bool Uni_InCodespace(km_kbp_usv ch) {
@@ -173,13 +189,26 @@ inline bool Uni_IsValid(km_kbp_usv ch) {
 
 inline bool Uni_IsValid(km_kbp_usv start, km_kbp_usv end) {
   // quicker check
-  if (!Uni_IsValid(end) || (end < start)) { // start is checked below
+  if (!Uni_IsValid(end) || !Uni_IsValid(start) || (end < start)) {  // start is checked below
     return false;
   }
 
-  // brute force it
-  for (km_kbp_usv i = start; i <= end; i++) {
-    if (!Uni_IsValid(i)) return false;
+  // If 'start' is low enough in the BMP, we need to avoid (1) Surrogates and (2) the FDxx nonchars
+  if (start < Uni_FD_NONCHARACTER_END) {
+    if ((start <= Uni_SURROGATE_END) && (end >= Uni_SURROGATE_START)) {
+      return false; // contains some of the surrogate range
+    } else if ((start <= Uni_FD_NONCHARACTER_END) && (end >= Uni_FD_NONCHARACTER_START)) {
+      return false; // contains some of the noncharacter range
+    }
+  }
+
+  // Are the end-of-plane noncharacters contained?
+  // As a reminder, we already checked that start/end are themselves valid,
+  // so we know that 'end' is not on a noncharacter at end of plane.
+  if ((start & Uni_PLANE_MASK) != (end & Uni_PLANE_MASK)) {
+    // start and end are on different planes, meaning that the U+__FFFE/U+__FFFF noncharacters
+    // are contained.  Invalid.
+    return false;
   }
 
   return true;

From 875d59987ac3d2f7c3459a72d0aacdefc811a751 Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Wed, 26 Jul 2023 16:16:57 -0500
Subject: [PATCH 31/83] =?UTF-8?q?chore(core):=20update=20documentation=20i?=
 =?UTF-8?q?n=20transform=20logic=20and=20processor=20=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- drop some unneeded variables also, should be no semantic change

For: #7375
---
 core/src/ldml/ldml_processor.cpp  |  14 ++--
 core/src/ldml/ldml_transforms.cpp | 122 +++++++++++++++++++++++-------
 core/src/ldml/ldml_transforms.hpp |  31 ++++----
 3 files changed, 119 insertions(+), 48 deletions(-)

diff --git a/core/src/ldml/ldml_processor.cpp b/core/src/ldml/ldml_processor.cpp
index 66a03836bb..533147565e 100644
--- a/core/src/ldml/ldml_processor.cpp
+++ b/core/src/ldml/ldml_processor.cpp
@@ -240,14 +240,15 @@ ldml_processor::process_event(
         // Construct a context buffer of all the KM_KBP_BT_CHAR items
         // Extract the context into 'ctxt' for transforms to process
         if (!!transforms) {
-          // if no transforms, no reason to do this extraction
+          // if no transforms, no reason to do this extraction (ctxt will remain empty)
           auto &cp = state->context();
           // We're only interested in as much of the context as is a KM_KBP_BT_CHAR.
           uint8_t last_type = KM_KBP_BT_UNKNOWN;
           for (auto c = cp.rbegin(); c != cp.rend(); c++) {
             last_type = c->type;
             if (last_type != KM_KBP_BT_CHAR) {
-              // not a char, get out
+              // not a char, stop here
+              // TODO-LDML: markers?
               break;
             }
             ctxt.emplace_front(1, c->character);
@@ -258,16 +259,16 @@ ldml_processor::process_event(
         // Look up the key
         const std::u16string str = keys.lookup(vk, modifier_state);
         if (str.empty()) {
-          // not found
+          // not found, so pass the keystroke on to the
           state->actions().push_invalidate_context();
           state->actions().push_emit_keystroke();
           break; // ----- commit and exit
         }
         // found the correct string - push it into the context and actions
         const std::u32string str32 = kmx::u16string_to_u32string(str);
-        for(size_t i=0; i<str32.length(); i++) {
-          state->context().push_character(str32[i]);
-          state->actions().push_character(str32[i]);
+        for (const auto &ch : str32) {
+          state->context().push_character(ch);
+          state->actions().push_character(ch);
         }
         // Now process transforms
         // Process the transforms
@@ -282,6 +283,7 @@ ldml_processor::process_event(
           for (const auto &ch : ctxt) {
             ctxtstr.append(ch);
           }
+          // check if the context matched, and if so how much (at the end)
           const size_t matchedContext = transforms->apply(ctxtstr, outputString);
 
           if (matchedContext > 0) {
diff --git a/core/src/ldml/ldml_transforms.cpp b/core/src/ldml/ldml_transforms.cpp
index d791e00618..ae13181387 100644
--- a/core/src/ldml/ldml_transforms.cpp
+++ b/core/src/ldml/ldml_transforms.cpp
@@ -20,8 +20,14 @@ namespace km {
 namespace kbp {
 namespace ldml {
 
+/**
+ * \def KMXPLUS_DEBUG_TRANSFORM
+ * define KMXPLUS_DEBUG_TRANSFORM=1 to enable verbose processing of transforms/reorders
+ * The default is 0, which only notes initialization and exceptional cases
+*/
+
 #ifndef KMXPLUS_DEBUG_TRANSFORM
-#define KMXPLUS_DEBUG_TRANSFORM 0
+#define KMXPLUS_DEBUG_TRANSFORM 1
 #endif
 
 #if KMXPLUS_DEBUG_TRANSFORM
@@ -105,7 +111,10 @@ reorder_sort_key::compare(const reorder_sort_key &other) const {
   } else if (quaternaryResult) {
     return quaternaryResult;
   } else {
-    assert(quaternaryResult);  // quaternary is a string index, should always be !=
+    // We don't expect to get here.  quaternaryResult is the string index, which
+    // should be unequal.
+    assert(quaternaryResult);
+    // We have the underlying character, so use the binary order as a tiebreaker.
     int identityResult = (int)ch - (int)other.ch;  // tie breaker
     return identityResult;
   }
@@ -129,6 +138,10 @@ reorder_sort_key::from(const std::u32string &str) {
   auto s   = str.begin();  // str iterator
   size_t c = 0;            // str index
   for (auto e = str.begin(); e < str.end(); e++, s++, c++) {
+    // primary    weight: 0
+    // seconary   weight: c (the string index)
+    // tertiary   weight: 0
+    // quaternary weight: c (the index again)
     keylist.emplace_back(reorder_sort_key{*s, 0, c, 0, c});
   }
   return keylist;
@@ -143,9 +156,13 @@ reorder_sort_key::dump() const {
 size_t
 element_list::match_end(const std::u32string &str) const {
   if (str.size() < size()) {
-    return 0;  // input string too short, can't possibly match
+    // input string too short, can't possibly match.
+    // This assumes each element is a single char, no string elements.
+    return 0;
   }
   // s: iterate from end to front of string
+  // For example, if str = 'abcd', we try to match 'd', then 'c', then 'b', then 'a'
+  // starting with the end of the element list.
   auto s = str.rbegin();
   // e: end to front on elements.
   // we know the # of elements is <= length of string,
@@ -164,15 +181,16 @@ element_list::match_end(const std::u32string &str) const {
 bool
 element_list::load(const kmx::kmx_plus &kplus, kmx::KMXPLUS_ELEM id) {
   KMX_DWORD elementsLength;
-  auto elements = kplus.elem->getElementList(id, elementsLength);
-  assert((elementsLength == 0) || (elements != nullptr));
+  auto elements = kplus.elem->getElementList(id, elementsLength); // pointer to beginning of element list
+  assert((elementsLength == 0) || (elements != nullptr)); // it could be a 0-length list
   for (size_t i = 0; i<elementsLength; i++) {
-    auto e = elements[i];
+    auto e     = elements[i];
     auto flags = e.flags;
-    auto type = flags & LDML_ELEM_FLAGS_TYPE;
+    auto type  = flags & LDML_ELEM_FLAGS_TYPE;
     if (type == LDML_ELEM_FLAGS_TYPE_CHAR) {
       emplace_back(e.element, flags); // char
     } else if (type == LDML_ELEM_FLAGS_TYPE_USET) {
+      // need to load a USet
       auto u = kplus.usetHelper.getUset(e.element);
       if (!u.valid()) {
         DebugLog("Error, invalid UnicodeSet at element %d", (int)i);
@@ -182,7 +200,7 @@ element_list::load(const kmx::kmx_plus &kplus, kmx::KMXPLUS_ELEM id) {
       }
       emplace_back(u, e.flags);
     } else {
-      // not handled
+      // reorders don't use 'string' element types, so we don't expect them here.
       assert((type != LDML_ELEM_FLAGS_TYPE_USET) && (type != LDML_ELEM_FLAGS_TYPE_CHAR));
       return false;
     }
@@ -196,22 +214,27 @@ element_list::load(const kmx::kmx_plus &kplus, kmx::KMXPLUS_ELEM id) {
 
 std::deque<reorder_sort_key> &
 element_list::update_sort_key(size_t offset, std::deque<reorder_sort_key> &key) const {
+  /** string index */
   size_t c = 0;
-  for (auto e = begin(); e < end(); e++) {
+  for (auto e = begin(); e < end(); e++, c++) {
+    /** update this key */
     auto &k = key.at(offset + c);
+    // we double check that the character matches. otherwise something
+    // has really gone awry, because we shouldn't be here if this element list doesn't apply.
     if (!e->matches(k.ch)) {
-      DebugLog("!! updateSortKey(%d+%d): element did not re-match the sortkey", offset, c);
+      DebugLog("!! Internal Error: updateSortKey(%d+%d): element did not re-match the sortkey", offset, c);
       k.dump();
       // TODO-LDML: assertion follows
+      assert(e->matches(k.ch));        // double check that this element matches
     }
-    assert(e->matches(k.ch));        // double check that this element matches
+    // we only update primary and tertiary weights
     k.primary  = e->get_order();
-    k.tertiary = e->get_tertiary();  // TODO-LDML: need more detailed tertiary work
+    // TODO-LDML: need more detailed tertiary work
+    k.tertiary = e->get_tertiary();
 #if KMXPLUS_DEBUG_TRANSFORM
     DebugTran("Updating at +%d", c);
     k.dump();
 #endif
-    c++;
   }
   return key;
 }
@@ -232,11 +255,14 @@ reorder_entry::reorder_entry(const element_list &new_elements, const element_lis
 size_t
 reorder_entry::match_end(std::u32string &str, size_t offset, size_t len) const {
   auto substr      = str.substr(offset, len);
+  // first, see if the elements match. If not, this entry doesn't apply
   size_t match_len = elements.match_end(substr);
   if (match_len == 0) {
     return 0;
   }
 
+  // Now we need to check if there is a "before=" element string that
+  // is also a precondition.
   if (!before.empty()) {
     // does not match before offset
     std::u32string prefix = substr.substr(0, substr.size() - match_len);
@@ -266,18 +292,20 @@ reorder_group::apply(std::u32string &str) const {
       size_t submatch = r.match_end(str, 0, s);
       if (submatch != 0) {
 #if KMXPLUS_DEBUG_TRANSFORM
-        DebugTran("Matched: %S (off=%d, len=%d)", str, 0, s);
+        DebugTran("Matched: %S (off=%d, len=%d)", str.c_str(), 0, s);
         r.elements.dump();
 #endif
         // update the sort key
         size_t sub_match_start = s - submatch;
         r.elements.update_sort_key(sub_match_start, sort_keys);
-        some_match = true;
+        some_match = true; // record that there was a match
       }
     }
-    // c++;
   }
   if (!some_match) {
+    // get out if nothing matched.
+    // the sortkey won't be "interesting", and the sort
+    // will be a no-op.
     DebugTran("Skip: No reorder elements matched.");
     return false;  // nothing matched, so no work.
   }
@@ -289,12 +317,37 @@ reorder_group::apply(std::u32string &str) const {
   }
 #endif
 
-  size_t match_len = str.size();  // TODO-LDML: for now, assume matches entire string
+  // TODO-LDML: for now, assume matches entire string.
+  // A needed optimization here would be to detect a common substring
+  // at the end of the old and new strings, and keep the match_len
+  // minimal. This reduces thrash in core's context.
+  size_t match_len = str.size();
 
+  // 'prefix' is the unmatched string before the match
+  // TODO-LDML: right now, this is empty.
   std::u32string prefix = str;
   prefix.resize(str.size() - match_len);  // just the part before the matched part.
-  // just the suffix (the matched part)
-  std::u32string suffix = str.substr(prefix.size(), match_len);
+
+  // Now, we need to actually do the sorting, but we must only sort
+  // 'runs' beginning with 0-weight keys.
+
+  // Consider the 'roast' example in the spec, you might end up with the following:
+  //    codepoint  (pri, sec, ter, quat)
+  //  U+1A21	(0, 0, 0, 0)
+  //  U+1A60	(127, 1, 0, 1)
+  //  U+1A45	(0, 2, 0, 2)
+  //  U+1A6B	(42, 3, 0, 3)
+  //  U+1A76	(55, 4, 0, 4)
+  // This example happens to be in order, but must be sorted in two diferent ranges,
+  // with secondary (index) values of [0,1] and [2,4]
+  //
+  // Another example might look like the following:
+  //  U+1A21	(0, 0, 0, 0)
+  //  U+1A6B	(42, 1, 0, 1)
+  //  U+1A76	(55, 2, 0, 2)
+  //  U+1A60	(10, 3, 0, 3)
+  //  U+1A45	(10, 4, 0, 3)
+  // Here there is only a single range to sort [0,4]
 
   /** pointer to the beginning of the current run. */
   std::deque<reorder_sort_key>::iterator run_start = sort_keys.begin();
@@ -312,13 +365,16 @@ reorder_group::apply(std::u32string &str) const {
     DebugTran("Sorting final subrange quaternary=[%d..]", run_start->quaternary);
     std::sort(run_start, sort_keys.end()); // reversed because it's a reverse iterator…?
   }
-  // recombine into a str
+  // recombine into a string by pulling out the 'ch' value
+  // that's in each sortkey element.
   std::u32string newSuffix;
-  size_t q = sort_keys.begin()->quaternary;  //
+  size_t q = sort_keys.begin()->quaternary; // start with the first quaternary
   for (auto e = sort_keys.begin(); e < sort_keys.end(); e++, q++) {
-    if (q != e->quaternary) {                // something rearranged in this subrange
+    if (q != e->quaternary) {
+      // something rearranged in this subrange, because the quaternary values are out of order.
       applied = true;
     }
+    // collect the characters
     newSuffix.append(1, e->ch);
   }
   if (applied) {
@@ -342,18 +398,24 @@ transform_entry::transform_entry(const std::u32string &from, const std::u32strin
 size_t
 transform_entry::match(const std::u32string &input) const {
   if (input.length() < fFrom.length()) {
+    // TODO-LDML: regex
+    // Too small, can't match.
     return 0;
   }
   // string at end
   auto substr = input.substr(input.length() - fFrom.length(), fFrom.length());
   if (substr != fFrom) {
+    // end of string doesn't match
     return 0;
   }
+  // match length == fFrom.length
   return substr.length();
 }
 
 std::u32string
 transform_entry::apply(const std::u32string & /*input*/, size_t /*matchLen*/) const {
+  // TODO-LDML: regex
+  // For now, we just return the 'to' string literally.
   return fTo;
 }
 
@@ -449,13 +511,14 @@ transforms::apply(const std::u32string &input, std::u32string &output) {
     // find the first match in this group (if present)
     // TODO-LDML: check if reorder
     if (group->type == any_group_type::transform) {
-      auto transform = group->transform.match(updatedInput, subMatched);
+      auto entry = group->transform.match(updatedInput, subMatched);
 
-      if (transform != nullptr) {
+      if (entry != nullptr) {
         // now apply the found transform
 
         // update subOutput (string) and subMatched
-        std::u32string subOutput = transform->apply(updatedInput, subMatched);
+        // the returned string must replace the last "subMatched" chars of the string.
+        std::u32string subOutput = entry->apply(updatedInput, subMatched);
 
         // remove the matched part of the updatedInput
         updatedInput.resize(updatedInput.length() - subMatched);  // chop of the subMatched part at end
@@ -477,7 +540,8 @@ transforms::apply(const std::u32string &input, std::u32string &output) {
         }
       }
     } else if (group->type == any_group_type::reorder) {
-      // TODO-LDML: cheesy solution
+      // TODO-LDML: cheesy solution. We should be finding a smaller
+      // common match here.
       std::u32string str2 = updatedInput;
       if (group->reorder.apply(str2)) {
         // pretend the whole thing matched
@@ -506,9 +570,9 @@ transforms::apply(const std::u32string &input, std::u32string &output) {
   return matched;
 }
 
-// simple impl
 bool
 transforms::apply(std::u32string &str) {
+  // simple implementation for tests
   std::u32string output;
   size_t matchLength = apply(str, output);
   if (matchLength == 0) {
@@ -518,7 +582,6 @@ transforms::apply(std::u32string &str) {
   str.append(output);
   return true;
 }
-// Loader
 
 transforms *
 transforms::load(
@@ -570,7 +633,8 @@ transforms::load(
         std::u16string mapFrom, mapTo;
 
         if (element->mapFrom && element->mapTo) {
-          // strings: variable name
+          // strings: variable name of from/to
+          // TODO-LDML: not implemented
           mapFrom = kplus.strs->get(element->mapFrom);
           mapTo   = kplus.strs->get(element->mapTo);
         }
diff --git a/core/src/ldml/ldml_transforms.hpp b/core/src/ldml/ldml_transforms.hpp
index be1d1abac4..6296399f2b 100644
--- a/core/src/ldml/ldml_transforms.hpp
+++ b/core/src/ldml/ldml_transforms.hpp
@@ -33,25 +33,30 @@ enum any_group_type {
  */
 class element {
 public:
-  /** from a USet */
+  /** construct from a USet */
   element(const USet &u, KMX_DWORD flags);
-  /** from a single char */
+  /** construct from a single char */
   element(km_kbp_usv ch, KMX_DWORD flags);
 
   /** @returns true if a USet type */
   bool is_uset() const;
+  /** @returns true if prebase bit set*/
   bool is_prebase() const;
+  /** @returns true if tertiary base bit set */
   bool is_tertiary_base() const;
-  signed char get_tertiary() const;
+  /** @returns the primary order */
   signed char get_order() const;
+  /** @returns the tertiary order */
+  signed char get_tertiary() const;
   /** @returns raw elem flags */
   KMX_DWORD get_flags() const;
   /** @returns true if matches this character*/
   bool matches(km_kbp_usv ch) const;
+  /** debugging: dump this element via DebugLog() */
   void dump() const;
 
 private:
-  // TODO-LDML: support multi-char strings
+  // TODO-LDML: support multi-char strings?
   const km_kbp_usv chr;
   const USet uset;
   const KMX_DWORD flags;
@@ -112,14 +117,12 @@ struct reorder_sort_key {
   signed char tertiary;  // tertiary value, defaults to 0
   size_t quaternary;     // index again
 
-  /**
-   * Return -1, 0, 1 depending on order
-  */
+  /** @returns -1, 0, 1 depending on ordering */
   int compare(const reorder_sort_key &other) const;
   bool operator<(const reorder_sort_key &other) const;
   bool operator>(const reorder_sort_key &other) const;
 
-  /** create a 'baseline' sort key, all 0 primary weights */
+  /** create a 'baseline' sort key, with each character having primary weight 0 */
   static std::deque<reorder_sort_key> from(const std::u32string &str);
 
   /** TODO-LDML: for debugging. */
@@ -138,7 +141,7 @@ public:
    * Update the deque (see reorder_sort_key::from()) with the weights from this element list
    * starting at the beginning of this element list
    * @param offset start at this offset in the deque. Still starts at the first element
-   * @param the key deque to update
+   * @param key key deque to update
    * @returns the key parameter
   */
   std::deque<reorder_sort_key> &update_sort_key(size_t offset, std::deque<reorder_sort_key> &key) const;
@@ -147,6 +150,7 @@ public:
   bool
   load(const kmx::kmx_plus& kplus, kmx::KMXPLUS_ELEM id);
 
+  /** TODO-LDML: for debugging */
   void dump() const;
 };
 
@@ -230,12 +234,13 @@ public:
   bool apply(std::u32string &str);
 
 public:
+  /** load from a kmx_plus data section, either tran or bksp */
   static transforms *
-  load(const kmx::kmx_plus &kplus, const kbp::kmx::COMP_KMXPLUS_TRAN *tran, const kbp::kmx::COMP_KMXPLUS_TRAN_Helper &tranHelper);
+  load(const kmx::kmx_plus &kplus,
+       const kbp::kmx::COMP_KMXPLUS_TRAN *tran,
+       const kbp::kmx::COMP_KMXPLUS_TRAN_Helper &tranHelper);
 };
-/**
- * Loader for transform groups (from tran or bksp)
- */
+
 }  // namespace ldml
 }  // namespace kbp
 }  // namespace km

From 99c2aff760379bda03bf2a3207815f0563b19237 Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Wed, 26 Jul 2023 16:18:53 -0500
Subject: [PATCH 32/83] =?UTF-8?q?chore(core):=20turn=20off=20debugging=20?=
 =?UTF-8?q?=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- fix a stray define

For: #7375
---
 core/src/ldml/ldml_transforms.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/ldml/ldml_transforms.cpp b/core/src/ldml/ldml_transforms.cpp
index ae13181387..4d469abfd9 100644
--- a/core/src/ldml/ldml_transforms.cpp
+++ b/core/src/ldml/ldml_transforms.cpp
@@ -27,7 +27,7 @@ namespace ldml {
 */
 
 #ifndef KMXPLUS_DEBUG_TRANSFORM
-#define KMXPLUS_DEBUG_TRANSFORM 1
+#define KMXPLUS_DEBUG_TRANSFORM 0
 #endif
 
 #if KMXPLUS_DEBUG_TRANSFORM

From f2915bbb66b37bb7029dac42249bc201ca105c0e Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Wed, 26 Jul 2023 17:04:50 -0500
Subject: [PATCH 33/83] =?UTF-8?q?spec(core):=20update=20marker=20spec=20?=
 =?UTF-8?q?=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- 'transform' doesn't have an 'after=' attribute

For: #9118
---
 core/src/ldml/C9134_ldml_markers.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/core/src/ldml/C9134_ldml_markers.md b/core/src/ldml/C9134_ldml_markers.md
index 7a0f5d6959..0b2436b97f 100644
--- a/core/src/ldml/C9134_ldml_markers.md
+++ b/core/src/ldml/C9134_ldml_markers.md
@@ -27,7 +27,6 @@ Markers can appear in both 'emitting' and 'matching-only' areas:
 #### Match only
 
 - `transform from=` to match markers
-- `transform after=` to match markers
 - `display to=` for matching keys which contain markers
 
 ## Theory / Encoding

From 0c7ad2c1af9b149993bc6b4e6d4732b8a5628e5f Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Wed, 26 Jul 2023 17:27:14 -0500
Subject: [PATCH 34/83] =?UTF-8?q?feat(developer):=20Marker=20tests=20?=
 =?UTF-8?q?=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- tests for marker validation
- TDD: it fails, hooray

For: #9119
---
 .../src/kmc-ldml/src/compiler/messages.ts     |  4 ++
 developer/src/kmc-ldml/src/compiler/vars.ts   |  7 +++
 .../sections/vars/fail-markers-badref-0.xml   | 37 ++++++++++++++++
 .../sections/vars/markers-maximal.xml         | 44 +++++++++++++++++++
 developer/src/kmc-ldml/test/test-vars.ts      | 20 +++++++++
 5 files changed, 112 insertions(+)
 create mode 100644 developer/src/kmc-ldml/test/fixtures/sections/vars/fail-markers-badref-0.xml
 create mode 100644 developer/src/kmc-ldml/test/fixtures/sections/vars/markers-maximal.xml

diff --git a/developer/src/kmc-ldml/src/compiler/messages.ts b/developer/src/kmc-ldml/src/compiler/messages.ts
index a6a8b4ff49..e3ec8bcbf2 100644
--- a/developer/src/kmc-ldml/src/compiler/messages.ts
+++ b/developer/src/kmc-ldml/src/compiler/messages.ts
@@ -130,5 +130,9 @@ export class CompilerMessages {
   static Error_CantReferenceSetFromUnicodeSet = (o:{id: string}) =>
   m(this.ERROR_CantReferenceSetFromUnicodeSet, `Illegal use of set variable from within UnicodeSet: \$[${o.id}]`);
   static ERROR_CantReferenceSetFromUnicodeSet = SevError | 0x0020;
+
+  static Error_MissingMarkers = (o: { ids: string[] }) =>
+  m(this.ERROR_MissingMarkers, `Markers used for matching but not defined: ${o.ids?.join(',')}`);
+  static ERROR_MissingMarkers = SevError | 0x0021;
 }
 
diff --git a/developer/src/kmc-ldml/src/compiler/vars.ts b/developer/src/kmc-ldml/src/compiler/vars.ts
index 19451c4db4..07ff140530 100644
--- a/developer/src/kmc-ldml/src/compiler/vars.ts
+++ b/developer/src/kmc-ldml/src/compiler/vars.ts
@@ -130,9 +130,16 @@ export class VarsCompiler extends SectionCompiler {
       }));
       valid = false;
     }
+
+    valid = valid && this.validateMarkers();
+
     return valid;
   }
 
+  private validateMarkers(): boolean {
+    return true;
+  }
+
   public compile(sections: DependencySections): Vars {
     const result =  new Vars();
 
diff --git a/developer/src/kmc-ldml/test/fixtures/sections/vars/fail-markers-badref-0.xml b/developer/src/kmc-ldml/test/fixtures/sections/vars/fail-markers-badref-0.xml
new file mode 100644
index 0000000000..59ce65fbb0
--- /dev/null
+++ b/developer/src/kmc-ldml/test/fixtures/sections/vars/fail-markers-badref-0.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+This will fail because the two markers given don't exist anywhere.
+
+-->
+<!DOCTYPE keyboard SYSTEM "../../../../../../../resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboard.dtd">
+<keyboard locale="mt" conformsTo="techpreview">
+  <names>
+    <name value="vars-markers-badref-0" />
+  </names>
+
+  <displays>
+    <!-- Not ok, nobody emits this, so this display doesn't make sense -->
+    <display to="\{doesnt-exist-1}" display="¬" />
+    <!-- OK - it si in the strings list.  Could warn that it matches no keys -->
+    <display to="\{maybe-emitted}" display="‽" />
+  </displays>
+
+  <variables>
+    <!-- This may be emitted, so at present it does not cause an error. -->
+    <!-- A linter could, however, hint on an unused variable. -->
+    <!-- Removing this variable would cause \m{maybe-emitted}  to error in the transform group -->
+    <string id="unused-variable" value="\{maybe-emitted}" />
+  </variables>
+
+  <transforms type="backspace">
+    <transformGroup>
+      <!-- not OK, nobody emits this. -->
+      <transform from="\m{doesnt-exist-2}" />
+      <!-- OK - it's in the variable list -->
+      <transform from="\m{maybe-emitted}" />
+      <!-- OK - matches all markers -->
+      <transform from="\m{.}" />
+    </transformGroup>
+  </transforms>
+</keyboard>
diff --git a/developer/src/kmc-ldml/test/fixtures/sections/vars/markers-maximal.xml b/developer/src/kmc-ldml/test/fixtures/sections/vars/markers-maximal.xml
new file mode 100644
index 0000000000..baa9383075
--- /dev/null
+++ b/developer/src/kmc-ldml/test/fixtures/sections/vars/markers-maximal.xml
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+  This keyboard won't make a lot of sense, it's mostly
+  a test of validating markers.
+
+  When we get further on in implementation, we'll shape this up
+  into something testable for implementation.
+
+-->
+<!DOCTYPE keyboard SYSTEM "../../../../../../../resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboard.dtd">
+<keyboard locale="mt" conformsTo="techpreview">
+  <names>
+    <name value="vars-markers-maximal" />
+  </names>
+
+  <displays>
+    <display to="\{m}" display="Ⓜ️" />
+  </displays>
+
+  <keys>
+    <key id="m" to="\{m}" />
+  </keys>
+
+  <!-- from spec -->
+  <variables>
+    <string id="x" value="\{x}" />
+  </variables>
+
+  <transforms type="simple">
+    <transformGroup>
+      <transform from="m" to="\{m}"/>
+      <transform from="x" to="\{x}"/>
+    </transformGroup>
+    <transformGroup>
+      <transform from="e\{x}" to="é" />
+    </transformGroup>
+  </transforms>
+  <transforms type="backspace">
+    <transformGroup>
+      <transform from="A\{.}B" />
+    </transformGroup>
+  </transforms>
+</keyboard>
diff --git a/developer/src/kmc-ldml/test/test-vars.ts b/developer/src/kmc-ldml/test/test-vars.ts
index a3cb303b82..fda20e19b9 100644
--- a/developer/src/kmc-ldml/test/test-vars.ts
+++ b/developer/src/kmc-ldml/test/test-vars.ts
@@ -183,4 +183,24 @@ describe('vars', function () {
       ],
     },
   ]);
+  describe('markers', function () {
+    this.slow(500); // 0.5 sec -- json schema validation takes a while
+
+    testCompilationCases(VarsCompiler, [
+      {
+        subpath: 'sections/vars/markers-maximal.xml',
+      },
+      {
+        subpath: 'sections/vars/fail-markers-badref-0.xml',
+        errors: [
+          CompilerMessages.Error_MissingMarkers({
+            ids: [
+              'doesnt-exist-1',
+              'doesnt-exist-2',
+            ]
+          }),
+        ],
+      },
+    ]);
+  });
 });

From ab4a0f36ab49619f55c23197a0263c7663ee34b2 Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Wed, 26 Jul 2023 18:06:22 -0500
Subject: [PATCH 35/83] =?UTF-8?q?feat(developer):=20Marker=20tests=20?=
 =?UTF-8?q?=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- wip , still not working

For: #9119
---
 .../types/src/ldml-keyboard/pattern-parser.ts |  3 ++
 developer/src/kmc-ldml/src/compiler/disp.ts   |  8 ++-
 developer/src/kmc-ldml/src/compiler/keys.ts   |  8 ++-
 developer/src/kmc-ldml/src/compiler/tran.ts   | 15 +++++-
 developer/src/kmc-ldml/src/compiler/vars.ts   | 51 ++++++++++++++++++-
 .../sections/vars/fail-markers-badref-0.xml   |  6 +++
 developer/src/kmc-ldml/test/test-vars.ts      |  1 +
 7 files changed, 86 insertions(+), 6 deletions(-)

diff --git a/common/web/types/src/ldml-keyboard/pattern-parser.ts b/common/web/types/src/ldml-keyboard/pattern-parser.ts
index 1b41b9416c..34c400b867 100644
--- a/common/web/types/src/ldml-keyboard/pattern-parser.ts
+++ b/common/web/types/src/ldml-keyboard/pattern-parser.ts
@@ -51,6 +51,9 @@ export class MarkerParser {
    * @returns `[]` or an array of all markers referenced
    */
   public static allReferences(str: string): string[] {
+    if (!str) {
+      return [];
+    }
     return matchArray(str, this.REFERENCE);
   }
 }
diff --git a/developer/src/kmc-ldml/src/compiler/disp.ts b/developer/src/kmc-ldml/src/compiler/disp.ts
index 1ed2e14b14..d83a90cca3 100644
--- a/developer/src/kmc-ldml/src/compiler/disp.ts
+++ b/developer/src/kmc-ldml/src/compiler/disp.ts
@@ -1,5 +1,5 @@
 import { constants } from "@keymanapp/ldml-keyboard-constants";
-import { KMXPlus } from '@keymanapp/common-types';
+import { KMXPlus, LDMLKeyboard, MarkerParser } from '@keymanapp/common-types';
 
 import { CompilerMessages } from "./messages.js";
 import { SectionCompiler } from "./section-compiler.js";
@@ -9,6 +9,12 @@ import Disp = KMXPlus.Disp;
 import DispItem = KMXPlus.DispItem;
 
 export class DispCompiler extends SectionCompiler {
+  static validateMarkers(keyboard: LDMLKeyboard.LKKeyboard, emitMarkers: Set<string>, matchMarkers: Set<string>): boolean {
+    keyboard.displays?.display?.forEach(({ to }) => {
+      MarkerParser.allReferences(to).forEach(marker => matchMarkers.add(marker));
+    });
+    return true;
+  }
 
   public get id() {
     return constants.section.disp;
diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts
index b3e13b70fc..614be67ebf 100644
--- a/developer/src/kmc-ldml/src/compiler/keys.ts
+++ b/developer/src/kmc-ldml/src/compiler/keys.ts
@@ -1,5 +1,5 @@
 import { constants } from '@keymanapp/ldml-keyboard-constants';
-import { LDMLKeyboard, KMXPlus, Constants } from '@keymanapp/common-types';
+import { LDMLKeyboard, KMXPlus, Constants, MarkerParser } from '@keymanapp/common-types';
 import { CompilerMessages } from './messages.js';
 import { SectionCompiler } from "./section-compiler.js";
 
@@ -10,6 +10,12 @@ import KeysFlicks = KMXPlus.KeysFlicks;
 import { allUsedKeyIdsInLayers, calculateUniqueKeys, translateLayerAttrToModifier, validModifier } from '../util/util.js';
 
 export class KeysCompiler extends SectionCompiler {
+  static validateMarkers(keyboard: LDMLKeyboard.LKKeyboard, emitMarkers: Set<string>, matchMarkers: Set<string>): boolean {
+    keyboard.keys?.key?.forEach(({ to }) => {
+      MarkerParser.allReferences(to).forEach(marker => emitMarkers.add(marker));
+    });
+    return true;
+  }
 
   public get id() {
     return constants.section.keys;
diff --git a/developer/src/kmc-ldml/src/compiler/tran.ts b/developer/src/kmc-ldml/src/compiler/tran.ts
index d0173b7bfb..2175ca2dbe 100644
--- a/developer/src/kmc-ldml/src/compiler/tran.ts
+++ b/developer/src/kmc-ldml/src/compiler/tran.ts
@@ -1,5 +1,5 @@
 import { constants, SectionIdent } from "@keymanapp/ldml-keyboard-constants";
-import { KMXPlus, LDMLKeyboard, CompilerCallbacks, VariableParser } from '@keymanapp/common-types';
+import { KMXPlus, LDMLKeyboard, CompilerCallbacks, VariableParser, MarkerParser } from '@keymanapp/common-types';
 import { SectionCompiler } from "./section-compiler.js";
 
 import Bksp = KMXPlus.Bksp;
@@ -18,7 +18,18 @@ import { CompilerMessages } from "./messages.js";
 
 type TransformCompilerType = 'simple' | 'backspace';
 
-class TransformCompiler<T extends TransformCompilerType, TranBase extends Tran> extends SectionCompiler {
+export class TransformCompiler<T extends TransformCompilerType, TranBase extends Tran> extends SectionCompiler {
+
+  static validateMarkers(keyboard: LDMLKeyboard.LKKeyboard, emitMarkers: Set<string>, matchMarkers: Set<string>): boolean {
+    keyboard?.transforms?.forEach(transforms =>
+      transforms.transformGroup.forEach(transformGroup => {
+        transformGroup.transform?.forEach(({ to, from }) => {
+          MarkerParser.allReferences(from).forEach(marker => matchMarkers.add(marker));
+          MarkerParser.allReferences(to).forEach(marker => emitMarkers.add(marker));
+        });
+      }));
+    return true;
+  }
 
   protected type: T;
 
diff --git a/developer/src/kmc-ldml/src/compiler/vars.ts b/developer/src/kmc-ldml/src/compiler/vars.ts
index 07ff140530..03647e213a 100644
--- a/developer/src/kmc-ldml/src/compiler/vars.ts
+++ b/developer/src/kmc-ldml/src/compiler/vars.ts
@@ -1,5 +1,5 @@
 import { SectionIdent, constants } from "@keymanapp/ldml-keyboard-constants";
-import { KMXPlus, LDMLKeyboard, CompilerCallbacks } from '@keymanapp/common-types';
+import { KMXPlus, LDMLKeyboard, CompilerCallbacks, MarkerParser } from '@keymanapp/common-types';
 import { VariableParser } from '@keymanapp/common-types';
 import { SectionCompiler } from "./section-compiler.js";
 import Vars = KMXPlus.Vars;
@@ -9,6 +9,9 @@ import UnicodeSetItem = KMXPlus.UnicodeSetItem;
 import DependencySections = KMXPlus.DependencySections;
 import LDMLKeyboardXMLSourceFile = LDMLKeyboard.LDMLKeyboardXMLSourceFile;
 import { CompilerMessages } from "./messages.js";
+import { KeysCompiler } from "./keys.js";
+import { TransformCompiler } from "./tran.js";
+import { DispCompiler } from "./disp.js";
 export class VarsCompiler extends SectionCompiler {
   public get id() {
     return constants.section.vars;
@@ -131,12 +134,56 @@ export class VarsCompiler extends SectionCompiler {
       valid = false;
     }
 
-    valid = valid && this.validateMarkers();
+    valid = this.validateMarkers() && valid; // accumulate validity
+
+    return valid;
+  }
+
+  private collectMarkers(emitMarkers : Set<string>, matchMarkers : Set<string>) : boolean {
+    let valid = true;
+
+    // call our friends to validate
+    valid = this.validateVarsMarkers(this.keyboard, emitMarkers, matchMarkers) && valid; // accumulate validity
+    valid = KeysCompiler.validateMarkers(this.keyboard, emitMarkers, matchMarkers) && valid; // accumulate validity
+    valid = TransformCompiler.validateMarkers(this.keyboard, emitMarkers, matchMarkers) && valid; // accumulate validity
+    valid = DispCompiler.validateMarkers(this.keyboard, emitMarkers, matchMarkers) && valid; // accumulate validity
 
     return valid;
   }
 
   private validateMarkers(): boolean {
+    /** only the markers used in emitters */
+    const emitMarkers : Set<string> = new Set<string>();
+    /** only the markers used in matchers */
+    const matchMarkers : Set<string> = new Set<string>();
+
+
+    let valid = this.collectMarkers(emitMarkers, matchMarkers);
+
+    // see if there are any matched-but-not-emitted
+    const matchedNotEmitted : string[] = [];
+    for (const m of matchMarkers.values()) {
+      if (m === '.') continue; // match-all marker
+      if (!emitMarkers.has(m)) {
+        matchedNotEmitted.push(m);
+      }
+    }
+
+    // report once
+    if (matchedNotEmitted.length) {
+      matchedNotEmitted.sort();
+      this.callbacks.reportMessage(CompilerMessages.Error_MissingMarkers({ ids: matchedNotEmitted }));
+      valid = false;
+    }
+    return valid;
+  }
+
+  validateVarsMarkers(keyboard: LDMLKeyboard.LKKeyboard, emitMarkers: Set<string>, matchMarkers: Set<string>) : boolean {
+    keyboard?.variables?.string?.forEach(({value}) =>
+          MarkerParser.allReferences(value).forEach(marker => {
+            emitMarkers.add(marker);
+            matchMarkers.add(marker);
+          }));
     return true;
   }
 
diff --git a/developer/src/kmc-ldml/test/fixtures/sections/vars/fail-markers-badref-0.xml b/developer/src/kmc-ldml/test/fixtures/sections/vars/fail-markers-badref-0.xml
index 59ce65fbb0..093849ac1a 100644
--- a/developer/src/kmc-ldml/test/fixtures/sections/vars/fail-markers-badref-0.xml
+++ b/developer/src/kmc-ldml/test/fixtures/sections/vars/fail-markers-badref-0.xml
@@ -24,6 +24,12 @@ This will fail because the two markers given don't exist anywhere.
     <string id="unused-variable" value="\{maybe-emitted}" />
   </variables>
 
+  <transforms type="simple">
+    <transformGroup>
+      <!-- not OK, nobody emits this. -->
+      <transform from="\m{doesnt-exist-3}" to="3" />
+    </transformGroup>
+  </transforms>
   <transforms type="backspace">
     <transformGroup>
       <!-- not OK, nobody emits this. -->
diff --git a/developer/src/kmc-ldml/test/test-vars.ts b/developer/src/kmc-ldml/test/test-vars.ts
index fda20e19b9..5efdc4c4a0 100644
--- a/developer/src/kmc-ldml/test/test-vars.ts
+++ b/developer/src/kmc-ldml/test/test-vars.ts
@@ -197,6 +197,7 @@ describe('vars', function () {
             ids: [
               'doesnt-exist-1',
               'doesnt-exist-2',
+              'doesnt-exist-3',
             ]
           }),
         ],

From b71b350724b3f6cb9fb47da1256cac42e8558130 Mon Sep 17 00:00:00 2001
From: Keyman Build Agent <buildagent@keyman.com>
Date: Thu, 27 Jul 2023 14:02:36 -0400
Subject: [PATCH 36/83] auto: increment master version to 17.0.149

---
 HISTORY.md | 6 ++++++
 VERSION.md | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/HISTORY.md b/HISTORY.md
index 0a6941afc2..b4d57e1c3e 100644
--- a/HISTORY.md
+++ b/HISTORY.md
@@ -1,5 +1,11 @@
 # Keyman Version History
 
+## 17.0.148 alpha 2023-07-27
+
+* feat(core): merge transform/reorder processing w/ u32 (#9293)
+* chore(developer): make unknown vkey a hint, not error (#9344)
+* chore(linux): Update supported Ubuntu versions (#9341)
+
 ## 17.0.147 alpha 2023-07-25
 
 * chore(linux): Update debian changelog (#9327)
diff --git a/VERSION.md b/VERSION.md
index 0e228b36ba..c826535c22 100644
--- a/VERSION.md
+++ b/VERSION.md
@@ -1 +1 @@
-17.0.148
\ No newline at end of file
+17.0.149
\ No newline at end of file

From b4c074ae4ffab4776f3444ed096282a16ec310a2 Mon Sep 17 00:00:00 2001
From: Eberhard Beilharz <eb1@sil.org>
Date: Thu, 27 Jul 2023 16:39:19 +0200
Subject: [PATCH 37/83] chore(linux): Fix creation of PRs after uploading to
 Debian

---
 linux/scripts/upload-to-debian.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/linux/scripts/upload-to-debian.sh b/linux/scripts/upload-to-debian.sh
index af443cac08..d7665f29f8 100755
--- a/linux/scripts/upload-to-debian.sh
+++ b/linux/scripts/upload-to-debian.sh
@@ -105,7 +105,7 @@ git add debian/changelog
 git commit -m "chore(linux): Update debian changelog"
 if [ -n "$PUSH" ]; then
     $NOOP git push --force-with-lease origin chore/linux/changelog
-    $NOOP gh pr create --draft --base "$stable_branch" --title "chore(linux): Update debian changelog" --body "@keymanapp-test-bot skip"
+    $NOOP gh pr create --draft --base "${stable_branch#origin/}" --title "chore(linux): Update debian changelog" --body "@keymanapp-test-bot skip"
 fi
 
 if $ISBETA; then
@@ -118,7 +118,7 @@ git checkout -B chore/linux/cherry-pick/changelog ${CLBRANCH}
 git cherry-pick -x chore/linux/changelog
 if [ -n "$PUSH" ]; then
     $NOOP git push --force-with-lease origin chore/linux/cherry-pick/changelog
-    $NOOP gh pr create --draft --base ${CLBRANCH} --title "chore(linux): Update debian changelog 🍒" --body "@keymanapp-test-bot skip"
+    $NOOP gh pr create --draft --base ${CLBRANCH#origin/} --title "chore(linux): Update debian changelog 🍒" --body "@keymanapp-test-bot skip"
 fi
 
 builder_heading "Finishing"

From c4720a05a11d4ef09c8f4388adb09aaf03993f2e Mon Sep 17 00:00:00 2001
From: Eberhard Beilharz <eb1@sil.org>
Date: Thu, 27 Jul 2023 16:30:20 +0200
Subject: [PATCH 38/83] chore(linux): Update debian changelog

(cherry picked from commit 0e74e37a7c52b189f47e91c93799a7e9ef850a0b)
---
 linux/debian/changelog | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/linux/debian/changelog b/linux/debian/changelog
index 4e672643b3..1ad0f861fb 100644
--- a/linux/debian/changelog
+++ b/linux/debian/changelog
@@ -1,3 +1,11 @@
+keyman (16.0.141-1) unstable; urgency=medium
+
+  * Work around mips64el build failure (#1041499)
+  * New upstream release.
+  * Re-release to Debian
+
+ -- Eberhard Beilharz <eb1@sil.org>  Thu, 27 Jul 2023 16:30:04 +0200
+
 keyman (16.0.140-1) unstable; urgency=medium
 
   * New upstream release (closes: #1037707).

From 4854cba4c9c8a11f68aa2ad31dfa0a052ac7a7a2 Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Fri, 28 Jul 2023 21:40:02 -0500
Subject: [PATCH 39/83] =?UTF-8?q?feat(developer):=20fix=20marker=20validat?=
 =?UTF-8?q?ion=20test=20=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- was using the wrong syntax, hyphen instead of underscore

#9119
---
 .../fixtures/sections/vars/fail-markers-badref-0.xml | 12 ++++++------
 developer/src/kmc-ldml/test/test-vars.ts             |  6 +++---
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/developer/src/kmc-ldml/test/fixtures/sections/vars/fail-markers-badref-0.xml b/developer/src/kmc-ldml/test/fixtures/sections/vars/fail-markers-badref-0.xml
index 093849ac1a..063b5a5d11 100644
--- a/developer/src/kmc-ldml/test/fixtures/sections/vars/fail-markers-badref-0.xml
+++ b/developer/src/kmc-ldml/test/fixtures/sections/vars/fail-markers-badref-0.xml
@@ -12,30 +12,30 @@ This will fail because the two markers given don't exist anywhere.
 
   <displays>
     <!-- Not ok, nobody emits this, so this display doesn't make sense -->
-    <display to="\{doesnt-exist-1}" display="¬" />
+    <display to="\m{doesnt_exist_1}" display="¬" />
     <!-- OK - it si in the strings list.  Could warn that it matches no keys -->
-    <display to="\{maybe-emitted}" display="‽" />
+    <display to="\m{maybe_emitted}" display="‽" />
   </displays>
 
   <variables>
     <!-- This may be emitted, so at present it does not cause an error. -->
     <!-- A linter could, however, hint on an unused variable. -->
     <!-- Removing this variable would cause \m{maybe-emitted}  to error in the transform group -->
-    <string id="unused-variable" value="\{maybe-emitted}" />
+    <string id="unused_variable" value="\m{maybe_emitted}" />
   </variables>
 
   <transforms type="simple">
     <transformGroup>
       <!-- not OK, nobody emits this. -->
-      <transform from="\m{doesnt-exist-3}" to="3" />
+      <transform from="\m{doesnt_exist_3}" to="3" />
     </transformGroup>
   </transforms>
   <transforms type="backspace">
     <transformGroup>
       <!-- not OK, nobody emits this. -->
-      <transform from="\m{doesnt-exist-2}" />
+      <transform from="\m{doesnt_exist_2}" />
       <!-- OK - it's in the variable list -->
-      <transform from="\m{maybe-emitted}" />
+      <transform from="\m{maybe_emitted}" />
       <!-- OK - matches all markers -->
       <transform from="\m{.}" />
     </transformGroup>
diff --git a/developer/src/kmc-ldml/test/test-vars.ts b/developer/src/kmc-ldml/test/test-vars.ts
index 5efdc4c4a0..d9e9d4df29 100644
--- a/developer/src/kmc-ldml/test/test-vars.ts
+++ b/developer/src/kmc-ldml/test/test-vars.ts
@@ -195,9 +195,9 @@ describe('vars', function () {
         errors: [
           CompilerMessages.Error_MissingMarkers({
             ids: [
-              'doesnt-exist-1',
-              'doesnt-exist-2',
-              'doesnt-exist-3',
+              'doesnt_exist_1',
+              'doesnt_exist_2',
+              'doesnt_exist_3',
             ]
           }),
         ],

From ccc86a942b59f2c7d75ad70c49e5cb4660deec50 Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Fri, 28 Jul 2023 21:54:09 -0500
Subject: [PATCH 40/83] =?UTF-8?q?feat(core):=20fix=20=20uset=20range=20che?=
 =?UTF-8?q?ck=20=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- review comments

#7375
---
 core/src/kmx/kmx_xstring.h | 37 ++++++++++++++++---------------------
 1 file changed, 16 insertions(+), 21 deletions(-)

diff --git a/core/src/kmx/kmx_xstring.h b/core/src/kmx/kmx_xstring.h
index e16ee7f78f..9b479dac7a 100644
--- a/core/src/kmx/kmx_xstring.h
+++ b/core/src/kmx/kmx_xstring.h
@@ -19,6 +19,7 @@ const char16_t Uni_FFFF_NONCHARACTER     = 0xFFFF;
 const char16_t Uni_BMP_END               = 0xFFFF;
 const km_kbp_usv Uni_SMP_START           = 0x010000;
 const km_kbp_usv Uni_PLANE_MASK          = 0x1F0000;
+const km_kbp_usv Uni_MAX_CODEPOINT       = 0x10FFFF;
 
 /**
  * @brief True if a lead surrogate
@@ -180,7 +181,7 @@ inline bool Uni_IsNoncharacter(km_kbp_usv ch) {
 }
 
 inline bool Uni_InCodespace(km_kbp_usv ch) {
-  return ((ch) <= 0x10FFFF);
+  return ((ch) <= Uni_MAX_CODEPOINT);
 };
 
 inline bool Uni_IsValid(km_kbp_usv ch) {
@@ -188,30 +189,24 @@ inline bool Uni_IsValid(km_kbp_usv ch) {
 }
 
 inline bool Uni_IsValid(km_kbp_usv start, km_kbp_usv end) {
-  // quicker check
-  if (!Uni_IsValid(end) || !Uni_IsValid(start) || (end < start)) {  // start is checked below
+  if (!Uni_IsValid(end) || !Uni_IsValid(start) || (end < start)) {
+    // start or end out of range, or inverted range
     return false;
-  }
-
-  // If 'start' is low enough in the BMP, we need to avoid (1) Surrogates and (2) the FDxx nonchars
-  if (start < Uni_FD_NONCHARACTER_END) {
-    if ((start <= Uni_SURROGATE_END) && (end >= Uni_SURROGATE_START)) {
-      return false; // contains some of the surrogate range
-    } else if ((start <= Uni_FD_NONCHARACTER_END) && (end >= Uni_FD_NONCHARACTER_START)) {
-      return false; // contains some of the noncharacter range
-    }
-  }
-
-  // Are the end-of-plane noncharacters contained?
-  // As a reminder, we already checked that start/end are themselves valid,
-  // so we know that 'end' is not on a noncharacter at end of plane.
-  if ((start & Uni_PLANE_MASK) != (end & Uni_PLANE_MASK)) {
+  } else if ((start <= Uni_SURROGATE_END) && (end >= Uni_SURROGATE_START)) {
+    // contains some of the surrogate range
+    return false;
+  } else if ((start <= Uni_FD_NONCHARACTER_END) && (end >= Uni_FD_NONCHARACTER_START)) {
+    // contains some of the noncharacter range
+    return false;
+  } else if ((start & Uni_PLANE_MASK) != (end & Uni_PLANE_MASK)) {
     // start and end are on different planes, meaning that the U+__FFFE/U+__FFFF noncharacters
-    // are contained.  Invalid.
+    // are contained.
+    // As a reminder, we already checked that start/end are themselves valid,
+    // so we know that 'end' is not on a noncharacter at end of plane.
     return false;
+  } else {
+    return true;
   }
-
-  return true;
 }
 
 

From c7881d99b2042cced66839d3ca2a5418eb20c865 Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Fri, 28 Jul 2023 21:55:39 -0500
Subject: [PATCH 41/83] Update core/src/ldml/ldml_processor.cpp

Co-authored-by: Marc Durdin <marc@durdin.net>
---
 core/src/ldml/ldml_processor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/ldml/ldml_processor.cpp b/core/src/ldml/ldml_processor.cpp
index 533147565e..41ed5d419f 100644
--- a/core/src/ldml/ldml_processor.cpp
+++ b/core/src/ldml/ldml_processor.cpp
@@ -259,7 +259,7 @@ ldml_processor::process_event(
         // Look up the key
         const std::u16string str = keys.lookup(vk, modifier_state);
         if (str.empty()) {
-          // not found, so pass the keystroke on to the
+          // not found, so pass the keystroke on to the Engine
           state->actions().push_invalidate_context();
           state->actions().push_emit_keystroke();
           break; // ----- commit and exit

From 52e54395f983140123682cd35b40d58beb0e00a2 Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Sat, 29 Jul 2023 14:18:37 -0500
Subject: [PATCH 42/83] =?UTF-8?q?feat(developer):=20marker=20accounting=20?=
 =?UTF-8?q?=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- split out MarkerTracker, could give us more precise messages about marker use
- for now, we parse all markers twice.
- update builder for the markers list

#9119
---
 .../src/kmx/kmx-plus-builder/build-vars.ts    |   6 +-
 .../kmx/kmx-plus-builder/kmx-plus-builder.ts  |   2 +-
 common/web/types/src/kmx/string-list.ts       |   7 +-
 developer/src/kmc-ldml/src/compiler/disp.ts   |   8 +-
 developer/src/kmc-ldml/src/compiler/keys.ts   | 111 +++++++++++++-----
 .../kmc-ldml/src/compiler/marker-tracker.ts   |  72 ++++++++++++
 developer/src/kmc-ldml/src/compiler/tran.ts   |  10 +-
 developer/src/kmc-ldml/src/compiler/vars.ts   |  61 +++++-----
 .../sections/vars/markers-maximal.xml         |  14 +--
 developer/src/kmc-ldml/test/test-vars.ts      |   6 +
 10 files changed, 217 insertions(+), 80 deletions(-)
 create mode 100644 developer/src/kmc-ldml/src/compiler/marker-tracker.ts

diff --git a/common/web/types/src/kmx/kmx-plus-builder/build-vars.ts b/common/web/types/src/kmx/kmx-plus-builder/build-vars.ts
index 5f70689e96..daa0e4a46c 100644
--- a/common/web/types/src/kmx/kmx-plus-builder/build-vars.ts
+++ b/common/web/types/src/kmx/kmx-plus-builder/build-vars.ts
@@ -2,7 +2,7 @@ import { constants } from "@keymanapp/ldml-keyboard-constants";
 import { KMXPlusData } from "../kmx-plus.js";
 import { build_strs_index, BUILDER_STR_REF, BUILDER_STRS } from "./build-strs.js";
 import { BUILDER_SECTION } from "./builder-section.js";
-import { BUILDER_LIST_REF } from "./build-list.js";
+import { build_list_index, BUILDER_LIST, BUILDER_LIST_REF } from "./build-list.js";
 import { build_elem_index, BUILDER_ELEM, BUILDER_ELEM_REF } from "./build-elem.js";
 
 
@@ -22,7 +22,7 @@ export interface BUILDER_VARS extends BUILDER_SECTION {
 /**
 * Builder for the 'vars' section
 */
-export function build_vars(kmxplus: KMXPlusData, sect_strs: BUILDER_STRS, sect_elem: BUILDER_ELEM) : BUILDER_VARS {
+export function build_vars(kmxplus: KMXPlusData, sect_strs: BUILDER_STRS, sect_elem: BUILDER_ELEM, sect_list: BUILDER_LIST) : BUILDER_VARS {
   if(!kmxplus.vars) {
     return null;
   }
@@ -49,7 +49,7 @@ export function build_vars(kmxplus: KMXPlusData, sect_strs: BUILDER_STRS, sect_e
     size: constants.length_vars +
       (constants.length_vars_item * kmxplus.vars.totalCount()),
     _offset: 0,
-    markers: 0,
+    markers: build_list_index(sect_list, kmxplus.vars.markers),
     varCount: kmxplus.vars.totalCount(),
     varEntries: [
       ...stringVars,
diff --git a/common/web/types/src/kmx/kmx-plus-builder/kmx-plus-builder.ts b/common/web/types/src/kmx/kmx-plus-builder/kmx-plus-builder.ts
index 082e5495eb..3cc58c9fe7 100644
--- a/common/web/types/src/kmx/kmx-plus-builder/kmx-plus-builder.ts
+++ b/common/web/types/src/kmx/kmx-plus-builder/kmx-plus-builder.ts
@@ -99,7 +99,7 @@ export default class KMXPlusBuilder {
     this.sect.name = build_name(this.file.kmxplus, this.sect.strs);
     this.sect.tran = build_tran(this.file.kmxplus.tran, this.sect.strs, this.sect.elem);
     this.sect.uset = build_uset(this.file.kmxplus, this.sect.strs);
-    this.sect.vars = build_vars(this.file.kmxplus, this.sect.strs, this.sect.elem);
+    this.sect.vars = build_vars(this.file.kmxplus, this.sect.strs, this.sect.elem, this.sect.list);
     this.sect.vkey = build_vkey(this.file.kmxplus);
 
     // Finalize the sect (index) section
diff --git a/common/web/types/src/kmx/string-list.ts b/common/web/types/src/kmx/string-list.ts
index 4b5716b85d..b683aae411 100644
--- a/common/web/types/src/kmx/string-list.ts
+++ b/common/web/types/src/kmx/string-list.ts
@@ -68,7 +68,12 @@ export class ListItem extends Array<ListIndex> {
       return 0;
     }
   }
+  /** for debugging, print as single string */
   toString(): string {
-    return this.map(v => v.value.value).join(' ');
+    return this.toStringArray().join(' ');
+  }
+  /** for debugging, map to string array */
+  toStringArray(): string[] {
+    return this.map(v => v.value.value);
   }
 };
diff --git a/developer/src/kmc-ldml/src/compiler/disp.ts b/developer/src/kmc-ldml/src/compiler/disp.ts
index d83a90cca3..d644f56b88 100644
--- a/developer/src/kmc-ldml/src/compiler/disp.ts
+++ b/developer/src/kmc-ldml/src/compiler/disp.ts
@@ -7,12 +7,12 @@ import { SectionCompiler } from "./section-compiler.js";
 import DependencySections = KMXPlus.DependencySections;
 import Disp = KMXPlus.Disp;
 import DispItem = KMXPlus.DispItem;
+import { MarkerTracker, MarkerUse } from "./marker-tracker.js";
 
 export class DispCompiler extends SectionCompiler {
-  static validateMarkers(keyboard: LDMLKeyboard.LKKeyboard, emitMarkers: Set<string>, matchMarkers: Set<string>): boolean {
-    keyboard.displays?.display?.forEach(({ to }) => {
-      MarkerParser.allReferences(to).forEach(marker => matchMarkers.add(marker));
-    });
+  static validateMarkers(keyboard: LDMLKeyboard.LKKeyboard, mt : MarkerTracker): boolean {
+    keyboard.displays?.display?.forEach(({ to }) =>
+      mt.add(MarkerUse.match, MarkerParser.allReferences(to)));
     return true;
   }
 
diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts
index 614be67ebf..28cc1536b8 100644
--- a/developer/src/kmc-ldml/src/compiler/keys.ts
+++ b/developer/src/kmc-ldml/src/compiler/keys.ts
@@ -8,12 +8,16 @@ import Keys = KMXPlus.Keys;
 import ListItem = KMXPlus.ListItem;
 import KeysFlicks = KMXPlus.KeysFlicks;
 import { allUsedKeyIdsInLayers, calculateUniqueKeys, translateLayerAttrToModifier, validModifier } from '../util/util.js';
+import { MarkerTracker, MarkerUse } from './marker-tracker.js';
 
 export class KeysCompiler extends SectionCompiler {
-  static validateMarkers(keyboard: LDMLKeyboard.LKKeyboard, emitMarkers: Set<string>, matchMarkers: Set<string>): boolean {
-    keyboard.keys?.key?.forEach(({ to }) => {
-      MarkerParser.allReferences(to).forEach(marker => emitMarkers.add(marker));
-    });
+  static validateMarkers(
+    keyboard: LDMLKeyboard.LKKeyboard,
+    mt: MarkerTracker
+  ): boolean {
+    keyboard.keys?.key?.forEach(({ to }) =>
+      mt.add(MarkerUse.emit, MarkerParser.allReferences(to))
+    );
     return true;
   }
 
@@ -26,7 +30,7 @@ export class KeysCompiler extends SectionCompiler {
    * @returns just the non-touch layers.
    */
   public hardwareLayers() {
-    return this.keyboard.layers?.filter(({form}) => form !== 'touch');
+    return this.keyboard.layers?.filter(({ form }) => form !== "touch");
   }
 
   public validate() {
@@ -36,7 +40,7 @@ export class KeysCompiler extends SectionCompiler {
     const usedKeys = allUsedKeyIdsInLayers(this.keyboard?.layers);
     const uniqueKeys = calculateUniqueKeys([...this.keyboard.keys?.key]);
     for (let key of uniqueKeys) {
-      const {id, flicks} = key;
+      const { id, flicks } = key;
       if (!usedKeys.has(id)) {
         continue; // unused key, ignore
       }
@@ -44,10 +48,14 @@ export class KeysCompiler extends SectionCompiler {
       if (!flicks) {
         continue; // no flicks
       }
-      const flickEntry = this.keyboard.keys?.flicks?.find(x => x.id === flicks);
-      if (!flickEntry ) {
+      const flickEntry = this.keyboard.keys?.flicks?.find(
+        (x) => x.id === flicks
+      );
+      if (!flickEntry) {
         valid = false;
-        this.callbacks.reportMessage(CompilerMessages.Error_MissingFlicks({flicks, id}));
+        this.callbacks.reportMessage(
+          CompilerMessages.Error_MissingFlicks({ flicks, id })
+        );
       }
     }
 
@@ -59,8 +67,9 @@ export class KeysCompiler extends SectionCompiler {
     if (hardwareLayers.length >= 1) {
       // validate all errors
       for (let layers of hardwareLayers) {
-        for(let layer of layers.layer) {
-          valid = this.validateHardwareLayerForKmap(layers.form, layer) && valid; // note: always validate even if previously invalid results found
+        for (let layer of layers.layer) {
+          valid =
+            this.validateHardwareLayerForKmap(layers.form, layer) && valid; // note: always validate even if previously invalid results found
         }
       }
       // TODO-LDML: } else { touch?
@@ -90,11 +99,13 @@ export class KeysCompiler extends SectionCompiler {
     /* c8 ignore next 3 */
     if (hardwareLayers.length > 1) {
       // validation should have already caught this
-      throw Error(`Internal error: Expected 0 or 1 hardware layer, not ${hardwareLayers.length}`);
+      throw Error(
+        `Internal error: Expected 0 or 1 hardware layer, not ${hardwareLayers.length}`
+      );
     } else if (hardwareLayers.length === 1) {
       const theLayers = hardwareLayers[0];
       const { form } = theLayers;
-      for(let layer of theLayers.layer) {
+      for (let layer of theLayers.layer) {
         this.compileHardwareLayerToKmap(sections, layer, sect, form);
       }
     } // else: TODO-LDML do nothing if only touch layers
@@ -104,7 +115,9 @@ export class KeysCompiler extends SectionCompiler {
 
   public loadFlicks(sections: DependencySections, sect: Keys) {
     for (let lkflicks of this.keyboard.keys.flicks) {
-      let flicks: KeysFlicks = new KeysFlicks(sections.strs.allocString(lkflicks.id));
+      let flicks: KeysFlicks = new KeysFlicks(
+        sections.strs.allocString(lkflicks.id)
+      );
 
       for (let lkflick of lkflicks.flick) {
         let flags = 0;
@@ -112,7 +125,10 @@ export class KeysCompiler extends SectionCompiler {
         if (!to.isOneChar) {
           flags |= constants.keys_flick_flags_extend;
         }
-        let directions: ListItem = sections.list.allocListFromSpaces(sections.strs, lkflick.directions);
+        let directions: ListItem = sections.list.allocListFromSpaces(
+          sections.strs,
+          lkflick.directions
+        );
         flicks.flicks.push({
           directions,
           flags,
@@ -138,19 +154,27 @@ export class KeysCompiler extends SectionCompiler {
       if (!!key.gap) {
         flags |= constants.keys_key_flags_gap;
       }
-      if (key.transform === 'no') {
+      if (key.transform === "no") {
         flags |= constants.keys_key_flags_notransform;
       }
       const id = sections.strs.allocString(key.id);
-      const longPress: ListItem = sections.list.allocListFromEscapedSpaces(sections.strs, key.longPress);
-      const longPressDefault = sections.strs.allocAndUnescapeString(key.longPressDefault);
-      const multiTap: ListItem = sections.list.allocListFromEscapedSpaces(sections.strs, key.multiTap);
+      const longPress: ListItem = sections.list.allocListFromEscapedSpaces(
+        sections.strs,
+        key.longPress
+      );
+      const longPressDefault = sections.strs.allocAndUnescapeString(
+        key.longPressDefault
+      );
+      const multiTap: ListItem = sections.list.allocListFromEscapedSpaces(
+        sections.strs,
+        key.multiTap
+      );
       const keySwitch = sections.strs.allocString(key.switch); // 'switch' is a reserved word
       const to = sections.strs.allocAndUnescapeString(key.to, true);
       if (!to.isOneChar) {
         flags |= constants.keys_key_flags_extend;
       }
-      const width = Math.ceil((key.width || 1) * 10.0);  // default, width=1
+      const width = Math.ceil((key.width || 1) * 10.0); // default, width=1
       sect.keys.push({
         flags,
         flicks,
@@ -172,12 +196,17 @@ export class KeysCompiler extends SectionCompiler {
    * @param layer
    * @returns
    */
-  private validateHardwareLayerForKmap(hardware: string, layer: LDMLKeyboard.LKLayer) {
+  private validateHardwareLayerForKmap(
+    hardware: string,
+    layer: LDMLKeyboard.LKLayer
+  ) {
     let valid = true;
 
     const { modifier } = layer;
     if (!validModifier(modifier)) {
-      this.callbacks.reportMessage(CompilerMessages.Error_InvalidModifier({ modifier, layer: layer.id }));
+      this.callbacks.reportMessage(
+        CompilerMessages.Error_InvalidModifier({ modifier, layer: layer.id })
+      );
       valid = false;
     }
 
@@ -185,21 +214,31 @@ export class KeysCompiler extends SectionCompiler {
     /* c8 ignore next 5 */
     if (!keymap) {
       // not reached due to XML validation
-      this.callbacks.reportMessage(CompilerMessages.Error_InvalidHardware({ form: hardware }));
+      this.callbacks.reportMessage(
+        CompilerMessages.Error_InvalidHardware({ form: hardware })
+      );
       valid = false;
     }
 
     const uniqueKeys = calculateUniqueKeys([...this.keyboard.keys?.key]);
     if (layer.row.length > keymap.length) {
-      this.callbacks.reportMessage(CompilerMessages.Error_HardwareLayerHasTooManyRows());
+      this.callbacks.reportMessage(
+        CompilerMessages.Error_HardwareLayerHasTooManyRows()
+      );
       valid = false;
     }
 
     for (let y = 0; y < layer.row.length && y < keymap.length; y++) {
-      const keys = layer.row[y].keys.split(' ');
+      const keys = layer.row[y].keys.split(" ");
 
       if (keys.length > keymap[y].length) {
-        this.callbacks.reportMessage(CompilerMessages.Error_RowOnHardwareLayerHasTooManyKeys({ row: y + 1, hardware, modifier }));
+        this.callbacks.reportMessage(
+          CompilerMessages.Error_RowOnHardwareLayerHasTooManyKeys({
+            row: y + 1,
+            hardware,
+            modifier,
+          })
+        );
         valid = false;
       }
 
@@ -207,14 +246,24 @@ export class KeysCompiler extends SectionCompiler {
       for (let key of keys) {
         x++;
 
-        let keydef = uniqueKeys.find(x => x.id == key);
+        let keydef = uniqueKeys.find((x) => x.id == key);
         if (!keydef) {
-          this.callbacks.reportMessage(CompilerMessages.Error_KeyNotFoundInKeyBag({ keyId: key, col: x + 1, row: y + 1, layer: layer.id, form: 'hardware' }));
+          this.callbacks.reportMessage(
+            CompilerMessages.Error_KeyNotFoundInKeyBag({
+              keyId: key,
+              col: x + 1,
+              row: y + 1,
+              layer: layer.id,
+              form: "hardware",
+            })
+          );
           valid = false;
           continue;
         }
         if (!keydef.to && !keydef.gap && !keydef.switch) {
-          this.callbacks.reportMessage(CompilerMessages.Error_KeyMissingToGapOrSwitch({ keyId: key }));
+          this.callbacks.reportMessage(
+            CompilerMessages.Error_KeyMissingToGapOrSwitch({ keyId: key })
+          );
           valid = false;
           continue;
         }
@@ -228,7 +277,7 @@ export class KeysCompiler extends SectionCompiler {
     sections: DependencySections,
     layer: LDMLKeyboard.LKLayer,
     sect: Keys,
-    hardware: string,
+    hardware: string
   ): Keys {
     const mod = translateLayerAttrToModifier(layer);
     const keymap = Constants.HardwareToKeymap.get(hardware);
@@ -237,7 +286,7 @@ export class KeysCompiler extends SectionCompiler {
     for (let row of layer.row) {
       y++;
 
-      const keys = row.keys.split(' ');
+      const keys = row.keys.split(" ");
       let x = -1;
       for (let key of keys) {
         x++;
diff --git a/developer/src/kmc-ldml/src/compiler/marker-tracker.ts b/developer/src/kmc-ldml/src/compiler/marker-tracker.ts
new file mode 100644
index 0000000000..8007f29e5b
--- /dev/null
+++ b/developer/src/kmc-ldml/src/compiler/marker-tracker.ts
@@ -0,0 +1,72 @@
+/**
+ * Verb for MarkerTracker.add()
+ */
+export enum MarkerUse {
+  /** outputs this marker into context (e.g. transform to= or key to=) */
+  emit,
+  /** consumes this marker out of the context (e.g. transform from=) */
+  consume,
+  /** matches the marker, but doesn't consume (e.g. display to=) */
+  match,
+  /** variable definition: might consume, emit, or match. */
+  variable,
+}
+
+type MarkerSet = Set<string>;
+
+/** Tracks usage of markers */
+export class MarkerTracker {
+  /** markers that were emitted */
+  emitted: MarkerSet;
+  /** markers that were consumed and removed from the context */
+  consumed: MarkerSet;
+  /** markers that were matched, but not necessarily consumed */
+  matched: MarkerSet;
+  /** all markers */
+  all: MarkerSet;
+
+  constructor() {
+    this.emitted = new Set<string>();
+    this.consumed = new Set<string>();
+    this.matched = new Set<string>();
+    this.all = new Set<string>();
+  }
+
+  /**
+   *
+   * @param verb what kind of use we are adding
+   * @param markers list of markers to add
+   */
+  add(verb: MarkerUse, markers: string[]) {
+    if (!markers.length) {
+      return; // skip if empty
+    }
+    if (verb == MarkerUse.emit) {
+      markers.forEach((m) => {
+        this.emitted.add(m);
+        this.all.add(m);
+      });
+    } else if (verb == MarkerUse.consume) {
+      markers.forEach((m) => {
+        this.consumed.add(m);
+        this.all.add(m);
+      });
+    } else if (verb == MarkerUse.match) {
+      markers.forEach((m) => {
+        this.matched.add(m);
+        this.all.add(m);
+      });
+    } else if (verb == MarkerUse.variable) {
+      markers.forEach((m) => {
+        // we don't know, so add it to all three
+        this.matched.add(m);
+        this.emitted.add(m);
+        this.consumed.add(m);
+        this.all.add(m);
+      });
+      /* c8 skip next 3 */
+    } else {
+      throw Error(`Internal error: unsupported verb ${verb} for match`);
+    }
+  }
+}
diff --git a/developer/src/kmc-ldml/src/compiler/tran.ts b/developer/src/kmc-ldml/src/compiler/tran.ts
index 2175ca2dbe..15db1b2220 100644
--- a/developer/src/kmc-ldml/src/compiler/tran.ts
+++ b/developer/src/kmc-ldml/src/compiler/tran.ts
@@ -15,19 +15,19 @@ import LKTransform = LDMLKeyboard.LKTransform;
 import LKTransforms = LDMLKeyboard.LKTransforms;
 import { verifyValidAndUnique } from "../util/util.js";
 import { CompilerMessages } from "./messages.js";
+import { MarkerTracker, MarkerUse } from "./marker-tracker.js";
 
 type TransformCompilerType = 'simple' | 'backspace';
 
 export class TransformCompiler<T extends TransformCompilerType, TranBase extends Tran> extends SectionCompiler {
 
-  static validateMarkers(keyboard: LDMLKeyboard.LKKeyboard, emitMarkers: Set<string>, matchMarkers: Set<string>): boolean {
+  static validateMarkers(keyboard: LDMLKeyboard.LKKeyboard, mt : MarkerTracker): boolean {
     keyboard?.transforms?.forEach(transforms =>
       transforms.transformGroup.forEach(transformGroup => {
         transformGroup.transform?.forEach(({ to, from }) => {
-          MarkerParser.allReferences(from).forEach(marker => matchMarkers.add(marker));
-          MarkerParser.allReferences(to).forEach(marker => emitMarkers.add(marker));
-        });
-      }));
+          mt.add(MarkerUse.emit, MarkerParser.allReferences(to));
+          mt.add(MarkerUse.consume, MarkerParser.allReferences(from));
+        })}));
     return true;
   }
 
diff --git a/developer/src/kmc-ldml/src/compiler/vars.ts b/developer/src/kmc-ldml/src/compiler/vars.ts
index 03647e213a..822f6e7d42 100644
--- a/developer/src/kmc-ldml/src/compiler/vars.ts
+++ b/developer/src/kmc-ldml/src/compiler/vars.ts
@@ -12,6 +12,7 @@ import { CompilerMessages } from "./messages.js";
 import { KeysCompiler } from "./keys.js";
 import { TransformCompiler } from "./tran.js";
 import { DispCompiler } from "./disp.js";
+import { MarkerTracker, MarkerUse } from "./marker-tracker.js";
 export class VarsCompiler extends SectionCompiler {
   public get id() {
     return constants.section.vars;
@@ -20,7 +21,8 @@ export class VarsCompiler extends SectionCompiler {
   public get dependencies(): Set<SectionIdent> {
     const defaults = new Set(<SectionIdent[]>[
       constants.section.strs,
-      constants.section.elem
+      constants.section.elem,
+      constants.section.list,
     ]);
     defaults.delete(this.id);
     return defaults;
@@ -32,7 +34,6 @@ export class VarsCompiler extends SectionCompiler {
 
   public validate(): boolean {
     let valid = true;
-    // TODO-LDML scan for markers?
 
     // Check for duplicate ids
     const allIds = new Set();
@@ -139,51 +140,47 @@ export class VarsCompiler extends SectionCompiler {
     return valid;
   }
 
-  private collectMarkers(emitMarkers : Set<string>, matchMarkers : Set<string>) : boolean {
+  private collectMarkers(mt : MarkerTracker) : boolean {
     let valid = true;
 
     // call our friends to validate
-    valid = this.validateVarsMarkers(this.keyboard, emitMarkers, matchMarkers) && valid; // accumulate validity
-    valid = KeysCompiler.validateMarkers(this.keyboard, emitMarkers, matchMarkers) && valid; // accumulate validity
-    valid = TransformCompiler.validateMarkers(this.keyboard, emitMarkers, matchMarkers) && valid; // accumulate validity
-    valid = DispCompiler.validateMarkers(this.keyboard, emitMarkers, matchMarkers) && valid; // accumulate validity
+    valid = this.validateVarsMarkers(this.keyboard, mt) && valid; // accumulate validity
+    valid = KeysCompiler.validateMarkers(this.keyboard, mt) && valid; // accumulate validity
+    valid = TransformCompiler.validateMarkers(this.keyboard, mt) && valid; // accumulate validity
+    valid = DispCompiler.validateMarkers(this.keyboard, mt) && valid; // accumulate validity
 
     return valid;
   }
 
   private validateMarkers(): boolean {
-    /** only the markers used in emitters */
-    const emitMarkers : Set<string> = new Set<string>();
-    /** only the markers used in matchers */
-    const matchMarkers : Set<string> = new Set<string>();
-
-
-    let valid = this.collectMarkers(emitMarkers, matchMarkers);
-
+    const mt = new MarkerTracker();
+    let valid = this.collectMarkers(mt);
     // see if there are any matched-but-not-emitted
-    const matchedNotEmitted : string[] = [];
-    for (const m of matchMarkers.values()) {
-      if (m === '.') continue; // match-all marker
-      if (!emitMarkers.has(m)) {
-        matchedNotEmitted.push(m);
+    const matchedNotEmitted : Set<string> = new Set<string>();
+    for (const m of mt.matched.values()) {
+      if (m === MarkerParser.ANY_MARKER_ID) continue; // match-all marker
+      if (!mt.emitted.has(m)) {
+        matchedNotEmitted.add(m);
+      }
+    }
+    for (const m of mt.consumed.values()) {
+      if (m === MarkerParser.ANY_MARKER_ID) continue; // match-all marker
+      if (!mt.emitted.has(m)) {
+        matchedNotEmitted.add(m);
       }
     }
 
     // report once
-    if (matchedNotEmitted.length) {
-      matchedNotEmitted.sort();
-      this.callbacks.reportMessage(CompilerMessages.Error_MissingMarkers({ ids: matchedNotEmitted }));
+    if (matchedNotEmitted.size > 0) {
+      this.callbacks.reportMessage(CompilerMessages.Error_MissingMarkers({ ids: Array.from(matchedNotEmitted.values()).sort() }));
       valid = false;
     }
     return valid;
   }
 
-  validateVarsMarkers(keyboard: LDMLKeyboard.LKKeyboard, emitMarkers: Set<string>, matchMarkers: Set<string>) : boolean {
+  validateVarsMarkers(keyboard: LDMLKeyboard.LKKeyboard, mt : MarkerTracker) : boolean {
     keyboard?.variables?.string?.forEach(({value}) =>
-          MarkerParser.allReferences(value).forEach(marker => {
-            emitMarkers.add(marker);
-            matchMarkers.add(marker);
-          }));
+          mt.add(MarkerUse.variable, MarkerParser.allReferences(value)));
     return true;
   }
 
@@ -207,6 +204,14 @@ export class VarsCompiler extends SectionCompiler {
     variables?.unicodeSet?.forEach((e) =>
       this.addUnicodeSet(result, e, sections));
 
+    // reload markers - TODO-LDML: double work!
+    const mt = new MarkerTracker();
+    this.collectMarkers(mt);
+
+    // collect all markers, excluding the match-all
+    const allMarkers : string[] = Array.from(mt.all).filter(m => m !== MarkerParser.ANY_MARKER_ID).sort();
+    result.markers = sections.list.allocList(sections.strs, allMarkers);
+
     return result.valid() ? result : null;
   }
 
diff --git a/developer/src/kmc-ldml/test/fixtures/sections/vars/markers-maximal.xml b/developer/src/kmc-ldml/test/fixtures/sections/vars/markers-maximal.xml
index baa9383075..cbe36d4db1 100644
--- a/developer/src/kmc-ldml/test/fixtures/sections/vars/markers-maximal.xml
+++ b/developer/src/kmc-ldml/test/fixtures/sections/vars/markers-maximal.xml
@@ -15,30 +15,30 @@
   </names>
 
   <displays>
-    <display to="\{m}" display="Ⓜ️" />
+    <display to="\m{m}" display="Ⓜ️" />
   </displays>
 
   <keys>
-    <key id="m" to="\{m}" />
+    <key id="m" to="\m{m}" />
   </keys>
 
   <!-- from spec -->
   <variables>
-    <string id="x" value="\{x}" />
+    <string id="x" value="\m{x}" />
   </variables>
 
   <transforms type="simple">
     <transformGroup>
-      <transform from="m" to="\{m}"/>
-      <transform from="x" to="\{x}"/>
+      <transform from="m" to="\m{m}"/>
+      <transform from="x" to="\m{x}"/>
     </transformGroup>
     <transformGroup>
-      <transform from="e\{x}" to="é" />
+      <transform from="e\m{x}" to="é" />
     </transformGroup>
   </transforms>
   <transforms type="backspace">
     <transformGroup>
-      <transform from="A\{.}B" />
+      <transform from="A\m{.}B" />
     </transformGroup>
   </transforms>
 </keyboard>
diff --git a/developer/src/kmc-ldml/test/test-vars.ts b/developer/src/kmc-ldml/test/test-vars.ts
index d9e9d4df29..6acaef35da 100644
--- a/developer/src/kmc-ldml/test/test-vars.ts
+++ b/developer/src/kmc-ldml/test/test-vars.ts
@@ -189,6 +189,12 @@ describe('vars', function () {
     testCompilationCases(VarsCompiler, [
       {
         subpath: 'sections/vars/markers-maximal.xml',
+        callback(sect) {
+          const vars = <Vars> sect;
+          assert.ok(vars.markers);
+          assert.sameDeepOrderedMembers(vars.markers.toStringArray(),
+            ['m','x']);
+        },
       },
       {
         subpath: 'sections/vars/fail-markers-badref-0.xml',

From 99886623a5022f01c094dde63ecbd56b632b0423 Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Sat, 29 Jul 2023 14:48:08 -0500
Subject: [PATCH 43/83] =?UTF-8?q?feat(developer):=20marker=20accounting=20?=
 =?UTF-8?q?=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- .. and the basic.txt to prove it

#9119
---
 .../src/kmc-ldml/test/fixtures/basic.txt      | 30 +++++++++++++------
 .../src/kmc-ldml/test/fixtures/basic.xml      |  1 +
 2 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/developer/src/kmc-ldml/test/fixtures/basic.txt b/developer/src/kmc-ldml/test/fixtures/basic.txt
index 0fe6d67a30..8713f93ae1 100644
--- a/developer/src/kmc-ldml/test/fixtures/basic.txt
+++ b/developer/src/kmc-ldml/test/fixtures/basic.txt
@@ -257,7 +257,7 @@ block(keys)                         # struct COMP_KMXPLUS_KEYS {
   index(strNull,strHmaqtugha,2)     #   KMXPLUS_STR 'hmaqtugha'
   00 00 00 00                       #   KMXPLUS_STR switch
   0A 00 00 00                       #   KMX_DWORD width*10
-  01 00 00 00 # TODO: index(listNull,indexAe,4)         #   LIST longPress 'a e'
+  02 00 00 00 # TODO: index(listNull,indexAe,4)         #   LIST longPress 'a e'
   00 00 00 00                       #   STR longPressDefault
   00 00 00 00 # TODO: index(listNull,listNull,4)        #   LIST multiTap
   00 00 00 00                       #   flicks 0
@@ -324,21 +324,25 @@ block(layr)                         # struct COMP_KMXPLUS_LAYR {
 block(list)                         # struct COMP_KMXPLUS_LAYR_LIST {
   6c 69 73 74                       #   KMX_DWORD header.ident;   // 0000 Section name - list
   diff(list,endList)                #   KMX_DWORD header.size;    // 0004 Section length
-  02 00 00 00                       #   KMX_DWORD listCount (should be 2)
-  02 00 00 00                       #   KMX_DWORD indexCount (should be 2)
+  03 00 00 00                       #   KMX_DWORD listCount (should be 2)
+  03 00 00 00                       #   KMX_DWORD indexCount (should be 2)
   # list #0 the null list
   block(listNull)
   00 00 00 00 #index(indexNull,indexNull,2)      #   KMX_DWORD list index (0)
   00 00 00 00                       #   KMX_DWORD lists[0].count
-  # list #1 the ae list
+  block(listA)
+  00 00 00 00 # first index
+  01 00 00 00 #count
   block(listAe)
-  00 00 00 00 # index(indexAe,indexNull,2)        #   KMX_DWORD list index (also 0)
+  01 00 00 00 # index(indexAe,indexNull,2)        #   KMX_DWORD list index (also 0)
   02 00 00 00                       #   KMX_DWORD count
   block(endLists)
   # indices
   #block(indexNull)
   # No null index
   # index(strNull,strNull,2)         #   KMXPLUS_STR string index
+  block(indexA)
+  index(strNull,strA,2) # a
   block(indexAe)
   index(strNull,strA,2) #   KMXPLUS_STR a
   index(strNull,strElemBkspFrom2,2) #   KMXPLUS_STR e
@@ -401,6 +405,7 @@ block(strs)                         #  struct COMP_KMXPLUS_STRS {
   diff(strs,strName)       sizeof(strName,2)
   diff(strs,strFromSet)     sizeof(strFromSet,2)
   diff(strs,strUSet)       sizeof(strUSet,2)
+  diff(strs,strAmarker)     sizeof(strAmarker,2)
   diff(strs,strElemTranFrom1) sizeof(strElemTranFrom1,2)
   diff(strs,strElemTranFrom1a) sizeof(strElemTranFrom1a,2)
   diff(strs,strElemTranFrom1b) sizeof(strElemTranFrom1b,2)
@@ -433,6 +438,7 @@ block(strs)                         #  struct COMP_KMXPLUS_STRS {
   block(strName)             54 00 65 00 73 00 74 00 4b 00 62 00 64 00   block(x) 00 00 # 'TestKbd'
   block(strFromSet)          5B 00 5C 00 75 00 31 00 41 00 37 00 35 00 2D 00 5C 00 75 00 31 00 41 00 37 00 39 00 5D 00 block(x) 00 00 # [\u1a75-\u1a79]
   block(strUSet)             5b 00 61 00 62 00 63 00 5d 00 block(x) 00 00               # '[abc]'
+  block(strAmarker)          5C 00 6D 00 7B 00 61 00 7D 00 block(x) 00 00               # '\m{a}'
   block(strElemTranFrom1)    5E 00 block(x) 00 00                                       # '^'
   block(strElemTranFrom1a)   5E 00 61 00 block(x) 00 00                                       # '^a'
   block(strElemTranFrom1b)   5E 00 65 00 block(x) 00 00                                       # '^e'
@@ -520,23 +526,29 @@ block(uset)
 block(vars)                         # struct COMP_KMXPLUS_VARS {
   76 61 72 73                       #   KMX_DWORD header.ident;   // 0000 Section name - vars
   diff(vars,varsEnd)                #   KMX_DWORD header.size;    // 0004 Section length
-  00 00 00 00                       #   KMX_DWORD markers - list
+  01 00 00 00                       #   KMX_DWORD markers - list 1 ['a']
   diff(varsBegin,varsEnd,16)        #   KMX_DWORD varCount
 
-  # var 0
   block(varsBegin)
+  # var 0
+  00 00 00 00                       #   KMX_DWORD type = str
+  index(strNull,strA,2)             #   KMXPLUS_STR id  'a'
+  index(strNull,strAmarker,2)       #   KMXPLUS_STR value '\m{a}'
+  00 00 00 00                       #   KMXPLUS_ELEM 
+
+  # var 1
   01 00 00 00                       #   KMX_DWORD type = set
   index(strNull,strVse,2)           #   KMXPLUS_STR id  'vse'
   index(strNull,strSet,2)           #   KMXPLUS_STR value 'a b c'
   01 00 00 00                       #   KMXPLUS_ELEM elem 'a b c' see 'elemSet'
 
-  # var 1
+  # var 2
   00 00 00 00                       #   KMX_DWORD type = string
   index(strNull,strVst,2)           #   KMXPLUS_STR id  'vst'
   index(strNull,strSet2,2)          #   KMXPLUS_STR value 'abc'
   00 00 00 00                       #   KMXPLUS_ELEM elem
 
-  # var 2
+  # var 3
   02 00 00 00                       #   KMX_DWORD type = string
   index(strNull,strVus,2)           #   KMXPLUS_STR id  'vus'
   index(strNull,strUSet,2)          #   KMXPLUS_STR value '[abc]'
diff --git a/developer/src/kmc-ldml/test/fixtures/basic.xml b/developer/src/kmc-ldml/test/fixtures/basic.xml
index 0eb6676490..cb0c56d269 100644
--- a/developer/src/kmc-ldml/test/fixtures/basic.xml
+++ b/developer/src/kmc-ldml/test/fixtures/basic.xml
@@ -37,6 +37,7 @@
   </layers>
 
   <variables>
+    <string id="a" value="\m{a}"/>
     <string id="vst" value="abc"/>
     <set id="vse" value="a b c"/>
     <unicodeSet id="vus" value="[abc]"/>

From 581d65a4c7f62b6da93adb8f94952652bf1a9e8e Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Sat, 29 Jul 2023 14:55:21 -0500
Subject: [PATCH 44/83] =?UTF-8?q?fix(common):=20list=20fix=20=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- a falsy list should show up as list #0

#9119
---
 common/web/types/src/kmx/kmx-plus-builder/build-list.ts | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/common/web/types/src/kmx/kmx-plus-builder/build-list.ts b/common/web/types/src/kmx/kmx-plus-builder/build-list.ts
index e770e7b278..26659f8eb6 100644
--- a/common/web/types/src/kmx/kmx-plus-builder/build-list.ts
+++ b/common/web/types/src/kmx/kmx-plus-builder/build-list.ts
@@ -86,6 +86,9 @@ export function build_list(source_list: List, sect_strs: BUILDER_STRS): BUILDER_
  * @returns
  */
 export function build_list_index(sect_list: BUILDER_LIST, value: ListItem) : BUILDER_LIST_REF {
+  if (!value) { 
+    return 0; // empty list
+  }
   if(!(value instanceof ListItem)) {
     throw new Error('unexpected value '+ value);
   }

From d514dc9915e66ec6a1f515628ed8dec84c1302cb Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Sat, 29 Jul 2023 16:06:59 -0500
Subject: [PATCH 45/83] =?UTF-8?q?feat(common):=20marker=20processing=20sup?=
 =?UTF-8?q?port=20=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- common: emit the sentinel value, with range checking, from MarkerParser
---
 .../types/src/ldml-keyboard/pattern-parser.ts | 27 +++++++++++++++++++
 .../test/ldml-keyboard/test-pattern-parser.ts |  6 +++++
 2 files changed, 33 insertions(+)

diff --git a/common/web/types/src/ldml-keyboard/pattern-parser.ts b/common/web/types/src/ldml-keyboard/pattern-parser.ts
index 34c400b867..d9f8dc313b 100644
--- a/common/web/types/src/ldml-keyboard/pattern-parser.ts
+++ b/common/web/types/src/ldml-keyboard/pattern-parser.ts
@@ -40,6 +40,25 @@ export class MarkerParser {
    */
   public static readonly ANY_MARKER_ID = '.';
 
+  /**
+   * Marker sentinel, == U_SENTINEL
+   */
+  public static readonly SENTINEL = '\uFFFF';
+
+  /**
+   * Matches all markers.
+   */
+  public static readonly SENTINEL_ALL_MARKERS = this.SENTINEL + this.SENTINEL;
+
+  /** Minimum ID (trailing code unit) */
+  public static readonly MIN_MARKER_INDEX = 0x0001;
+  /** Index meaning 'any marker' == `\m{.}` */
+  public static readonly ANY_MARKER_INDEX = 0xFFFF;
+  /** Maximum usable marker index */
+  public static readonly MAX_MARKER_INDEX = this.ANY_MARKER_INDEX - 1;
+  /** Max count of markers */
+  public static readonly MAX_MARKER_COUNT = this.MAX_MARKER_INDEX - this.MIN_MARKER_INDEX;
+
   /**
    * Pattern for matching a marker reference, OR the special marker \m{.}
    */
@@ -56,6 +75,14 @@ export class MarkerParser {
     }
     return matchArray(str, this.REFERENCE);
   }
+
+  /** @returns string for marker #n */
+  public static markerOutput(n: number): string {
+    if (n < MarkerParser.MIN_MARKER_INDEX || n > MarkerParser.ANY_MARKER_INDEX) {
+      throw RangeError(`Internal Error: marker index out of range ${n}`);
+    }
+    return this.SENTINEL + String.fromCharCode(n);
+  }
 }
 
 /**
diff --git a/common/web/types/test/ldml-keyboard/test-pattern-parser.ts b/common/web/types/test/ldml-keyboard/test-pattern-parser.ts
index f8b881b644..afb1207cc4 100644
--- a/common/web/types/test/ldml-keyboard/test-pattern-parser.ts
+++ b/common/web/types/test/ldml-keyboard/test-pattern-parser.ts
@@ -51,6 +51,12 @@ describe('Test of Pattern Parsers', () => {
         assert.deepEqual(MarkerParser.allReferences(str), [], `expected no markers: ${str}`);
       }
     });
+    it('should be able to emit sentinel values', () => {
+      assert.equal(MarkerParser.markerOutput(295), '\uFFFF\u0127', 'Wrong sentinel value emitted');
+      assert.equal(MarkerParser.markerOutput(MarkerParser.ANY_MARKER_INDEX), MarkerParser.SENTINEL_ALL_MARKERS, 'Wrong sentinel value emitted for ffff');
+      assert.throws(() => MarkerParser.markerOutput(0)); // below MIN
+      assert.throws(() => MarkerParser.markerOutput(0x10000)); // above MAX
+    });
   });
   describe('should test VariableParser', () => {
     // same test as for markers

From 2211900e5f33ac8256613913fc036cb1cdbc1928 Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Sat, 29 Jul 2023 16:15:55 -0500
Subject: [PATCH 46/83] =?UTF-8?q?feat(common):=20marker=20-=20move=20value?=
 =?UTF-8?q?s=20into=20constants=20=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- constants need to be shared with C++

#9119
---
 .../web/types/src/ldml-keyboard/pattern-parser.ts   | 13 +++++++------
 core/include/ldml/keyboardprocessor_ldml.h          |  5 +++++
 core/include/ldml/keyboardprocessor_ldml.ts         | 13 +++++++++++++
 3 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/common/web/types/src/ldml-keyboard/pattern-parser.ts b/common/web/types/src/ldml-keyboard/pattern-parser.ts
index d9f8dc313b..63d099a1f6 100644
--- a/common/web/types/src/ldml-keyboard/pattern-parser.ts
+++ b/common/web/types/src/ldml-keyboard/pattern-parser.ts
@@ -2,6 +2,7 @@
  * Utilities for transform and marker processing
  */
 
+import { constants } from "@keymanapp/ldml-keyboard-constants";
 import { MATCH_QUAD_ESCAPE, isOneChar, unescapeOneQuadString, unescapeString } from "../util/util.js";
 
 
@@ -41,9 +42,9 @@ export class MarkerParser {
   public static readonly ANY_MARKER_ID = '.';
 
   /**
-   * Marker sentinel, == U_SENTINEL
+   * Marker sentinel as a string - U+FFFF
    */
-  public static readonly SENTINEL = '\uFFFF';
+  public static readonly SENTINEL = String.fromCodePoint(constants.marker_sentinel);
 
   /**
    * Matches all markers.
@@ -51,13 +52,13 @@ export class MarkerParser {
   public static readonly SENTINEL_ALL_MARKERS = this.SENTINEL + this.SENTINEL;
 
   /** Minimum ID (trailing code unit) */
-  public static readonly MIN_MARKER_INDEX = 0x0001;
+  public static readonly MIN_MARKER_INDEX = constants.marker_min_index;
   /** Index meaning 'any marker' == `\m{.}` */
-  public static readonly ANY_MARKER_INDEX = 0xFFFF;
+  public static readonly ANY_MARKER_INDEX = constants.marker_any_index;
   /** Maximum usable marker index */
-  public static readonly MAX_MARKER_INDEX = this.ANY_MARKER_INDEX - 1;
+  public static readonly MAX_MARKER_INDEX = constants.marker_max_index;
   /** Max count of markers */
-  public static readonly MAX_MARKER_COUNT = this.MAX_MARKER_INDEX - this.MIN_MARKER_INDEX;
+  public static readonly MAX_MARKER_COUNT = constants.marker_max_count;
 
   /**
    * Pattern for matching a marker reference, OR the special marker \m{.}
diff --git a/core/include/ldml/keyboardprocessor_ldml.h b/core/include/ldml/keyboardprocessor_ldml.h
index a92731bfa3..262a18e6f0 100644
--- a/core/include/ldml/keyboardprocessor_ldml.h
+++ b/core/include/ldml/keyboardprocessor_ldml.h
@@ -93,6 +93,11 @@
 #define LDML_LENGTH_VARS_ITEM 0x10
 #define LDML_LENGTH_VKEY 0xC
 #define LDML_LENGTH_VKEY_ITEM 0x8
+#define LDML_MARKER_ANY_INDEX 0xFFFF
+#define LDML_MARKER_MAX_COUNT 0xFFFD
+#define LDML_MARKER_MAX_INDEX 0xFFFE
+#define LDML_MARKER_MIN_INDEX 0x1
+#define LDML_MARKER_SENTINEL 0xFFFF
 #define LDML_META_SETTINGS_FALLBACK_OMIT 0x1
 #define LDML_META_SETTINGS_TRANSFORMFAILURE_OMIT 0x2
 #define LDML_META_SETTINGS_TRANSFORMPARTIAL_HIDE 0x4
diff --git a/core/include/ldml/keyboardprocessor_ldml.ts b/core/include/ldml/keyboardprocessor_ldml.ts
index 4d59700895..fdfc63da18 100644
--- a/core/include/ldml/keyboardprocessor_ldml.ts
+++ b/core/include/ldml/keyboardprocessor_ldml.ts
@@ -613,6 +613,19 @@ class Constants {
     }
     return chars.join('');
   }
+
+  // ---- marker stuff ----
+  /** sentinel value indicating a marker follows */
+  readonly marker_sentinel = 0xFFFF;
+  /** minimum usable marker index */
+  readonly marker_min_index = 0x0001;
+  /** index value referring to the 'any' marker match */
+  readonly marker_any_index = 0xFFFF;
+  /** maximum marker index prior to the 'any' value */
+  readonly marker_max_index = this.marker_any_index - 1;
+  /** maximum count of markers (not including 'any') */
+  readonly marker_max_count = this.marker_max_index - this.marker_min_index;
+
 };
 
 export const constants = new Constants();

From 97c8380f7c8c075d878f10f55584d70475ce2801 Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Sat, 29 Jul 2023 16:48:33 -0500
Subject: [PATCH 47/83] =?UTF-8?q?feat(common):=20marker=20-=20emit=20senti?=
 =?UTF-8?q?nel=20values=20=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

#9119
---
 common/web/types/src/kmx/string-list.ts       |  6 ++-
 .../types/src/ldml-keyboard/pattern-parser.ts | 23 +++++++++++
 .../test/ldml-keyboard/test-pattern-parser.ts | 38 ++++++++++++++++++-
 3 files changed, 65 insertions(+), 2 deletions(-)

diff --git a/common/web/types/src/kmx/string-list.ts b/common/web/types/src/kmx/string-list.ts
index b683aae411..f9f4b46e91 100644
--- a/common/web/types/src/kmx/string-list.ts
+++ b/common/web/types/src/kmx/string-list.ts
@@ -1,3 +1,4 @@
+import { OrderedStringList } from 'src/ldml-keyboard/pattern-parser.js';
 import { Strs, StrsItem } from './kmx-plus.js';
 
 /**
@@ -22,7 +23,7 @@ export class ListIndex {
  * A string list in memory. This will be replaced with an index
  * into the string table at finalization.
  */
-export class ListItem extends Array<ListIndex> {
+export class ListItem extends Array<ListIndex> implements OrderedStringList {
   /**
    * Construct a new list from an array of strings.
    * Use List. This is meant to be called by the List.allocString*() functions.
@@ -41,6 +42,9 @@ export class ListItem extends Array<ListIndex> {
         this.push(index);
     }
   }
+  getItemOrder(item: string): number {
+    return this.findIndex(({value}) => value.value === item);
+  }
   isEqual(a: ListItem | string[]): boolean {
     if (a.length != this.length) {
       return false;
diff --git a/common/web/types/src/ldml-keyboard/pattern-parser.ts b/common/web/types/src/ldml-keyboard/pattern-parser.ts
index 63d099a1f6..6dd9c751db 100644
--- a/common/web/types/src/ldml-keyboard/pattern-parser.ts
+++ b/common/web/types/src/ldml-keyboard/pattern-parser.ts
@@ -22,6 +22,12 @@ function matchArray(str: string, match: RegExp) : string[] {
  */
 const COMMON_ID = /^[0-9A-Za-z_]{1,32}$/;
 
+/** for use with markers, means an ordering can be determined */
+export interface OrderedStringList {
+  /** @returns the ordering of an item (0..), or -1 if not found */
+  getItemOrder(item : string) : number;
+}
+
 /**
  * Class for helping with markers
  */
@@ -84,6 +90,23 @@ export class MarkerParser {
     }
     return this.SENTINEL + String.fromCharCode(n);
   }
+
+  /** @returns all marker strings as sentinel values */
+  public static toSentinelString(s: string, markers: OrderedStringList) : string {
+    return s.replaceAll(this.REFERENCE, (sub, arg) => {
+      if (arg === MarkerParser.ANY_MARKER_ID) {
+        return MarkerParser.SENTINEL_ALL_MARKERS;
+      }
+      const order = markers.getItemOrder(arg);
+      if (order === -1) {
+        throw RangeError(`Internal Error: Could not find marker \\m{${arg}}`);
+      } else if(order >= MarkerParser.MAX_MARKER_INDEX) {
+        throw RangeError(`Internal Error: marker \\m{${arg}} has out of range index ${order}`);
+      } else {
+        return MarkerParser.markerOutput(order+1);
+      }
+    });
+  }
 }
 
 /**
diff --git a/common/web/types/test/ldml-keyboard/test-pattern-parser.ts b/common/web/types/test/ldml-keyboard/test-pattern-parser.ts
index afb1207cc4..f12c09aad6 100644
--- a/common/web/types/test/ldml-keyboard/test-pattern-parser.ts
+++ b/common/web/types/test/ldml-keyboard/test-pattern-parser.ts
@@ -1,6 +1,6 @@
 import 'mocha';
 import { assert } from 'chai';
-import { ElementParser, ElementSegment, ElementType, MarkerParser, VariableParser } from '../../src/ldml-keyboard/pattern-parser.js';
+import { ElementParser, ElementSegment, ElementType, MarkerParser, OrderedStringList, VariableParser } from '../../src/ldml-keyboard/pattern-parser.js';
 
 describe('Test of Pattern Parsers', () => {
   describe('should test MarkerParser', () => {
@@ -57,6 +57,42 @@ describe('Test of Pattern Parsers', () => {
       assert.throws(() => MarkerParser.markerOutput(0)); // below MIN
       assert.throws(() => MarkerParser.markerOutput(0x10000)); // above MAX
     });
+    it('should be able to output sentinel strings', () => {
+      class MyMarkers implements OrderedStringList {
+        getItemOrder(item: string): number {
+          const m : any = {
+            'a': 0,
+            'b': 1,
+            'c': 2,
+            'zzz': 0x2FFFFF,
+          };
+          const o = m[item];
+          if (o === undefined) return -1;
+          return o;
+        }
+      };
+      const markers = new MyMarkers();
+      assert.equal(MarkerParser.toSentinelString(
+        `No markers here!`, markers),
+        `No markers here!`
+      );
+      assert.equal(MarkerParser.toSentinelString(
+        `Give me \\m{a} and \\m{c}, or \\m{.}.`, markers),
+        `Give me \uFFFF\u0001 and \uFFFF\u0003, or \uFFFF\uFFFF.`
+      );
+      assert.throws(() =>
+        MarkerParser.toSentinelString(
+          `Want to see something funny? \\m{zzz}`, // out of range
+          markers
+        )
+      );
+      assert.throws(() =>
+        MarkerParser.toSentinelString(
+          `Want to see something sad? \\m{nothing}`, // non existent
+          markers
+        )
+      );
+    });
   });
   describe('should test VariableParser', () => {
     // same test as for markers

From a6bce2d6c0c386f6e402534d94390e4099ec7e8c Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Sat, 29 Jul 2023 16:59:54 -0500
Subject: [PATCH 48/83] =?UTF-8?q?feat(common):=20marker=20-=20utilities=20?=
 =?UTF-8?q?to=20emit=20sentinel=20values=20=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

#9119
---
 common/web/types/src/kmx/kmx-plus.ts                     | 4 ++++
 common/web/types/src/ldml-keyboard/pattern-parser.ts     | 5 ++++-
 .../web/types/test/ldml-keyboard/test-pattern-parser.ts  | 9 +++++++++
 3 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/common/web/types/src/kmx/kmx-plus.ts b/common/web/types/src/kmx/kmx-plus.ts
index 9ca0a13ad7..795cebc696 100644
--- a/common/web/types/src/kmx/kmx-plus.ts
+++ b/common/web/types/src/kmx/kmx-plus.ts
@@ -6,6 +6,7 @@ import { isOneChar, toOneChar, unescapeString } from '../util/util.js';
 import { KMXFile } from './kmx.js';
 import { UnicodeSetParser, UnicodeSet } from '@keymanapp/common-types';
 import { VariableParser } from '../ldml-keyboard/pattern-parser.js';
+import { MarkerParser } from '../ldml-keyboard/pattern-parser.js';
 
 // Implementation of file structures from /core/src/ldml/C7043_ldml.md
 // Writer in kmx-builder.ts
@@ -292,6 +293,9 @@ export class Vars extends Section {
       return v[0];
     }
   }
+  substituteMarkerString(s : string) : string {
+    return MarkerParser.toSentinelString(s, this.markers);
+  }
 };
 
 /**
diff --git a/common/web/types/src/ldml-keyboard/pattern-parser.ts b/common/web/types/src/ldml-keyboard/pattern-parser.ts
index 6dd9c751db..97d7c49cab 100644
--- a/common/web/types/src/ldml-keyboard/pattern-parser.ts
+++ b/common/web/types/src/ldml-keyboard/pattern-parser.ts
@@ -92,11 +92,14 @@ export class MarkerParser {
   }
 
   /** @returns all marker strings as sentinel values */
-  public static toSentinelString(s: string, markers: OrderedStringList) : string {
+  public static toSentinelString(s: string, markers?: OrderedStringList) : string {
     return s.replaceAll(this.REFERENCE, (sub, arg) => {
       if (arg === MarkerParser.ANY_MARKER_ID) {
         return MarkerParser.SENTINEL_ALL_MARKERS;
       }
+      if (!markers) {
+        throw RangeError(`Internal Error: Could not find marker \\m{${arg}} (no markers defined)`);
+      }
       const order = markers.getItemOrder(arg);
       if (order === -1) {
         throw RangeError(`Internal Error: Could not find marker \\m{${arg}}`);
diff --git a/common/web/types/test/ldml-keyboard/test-pattern-parser.ts b/common/web/types/test/ldml-keyboard/test-pattern-parser.ts
index f12c09aad6..9d00ab82bf 100644
--- a/common/web/types/test/ldml-keyboard/test-pattern-parser.ts
+++ b/common/web/types/test/ldml-keyboard/test-pattern-parser.ts
@@ -58,6 +58,15 @@ describe('Test of Pattern Parsers', () => {
       assert.throws(() => MarkerParser.markerOutput(0x10000)); // above MAX
     });
     it('should be able to output sentinel strings', () => {
+      // with nothing (no markers)
+      assert.equal(
+        MarkerParser.toSentinelString(`No markers here!`),
+        `No markers here!`
+      );
+      assert.throws(() =>
+        MarkerParser.toSentinelString(`Marker \\m{sorryNoMarkers}`)
+      );
+      // with a custom class
       class MyMarkers implements OrderedStringList {
         getItemOrder(item: string): number {
           const m : any = {

From 8c79f1415623738e6b3adfd0589fda99df988bae Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Sat, 29 Jul 2023 17:26:36 -0500
Subject: [PATCH 49/83] =?UTF-8?q?feat(developer):=20marker=20-=20use=20mar?=
 =?UTF-8?q?ker=20in=20the=20sections=20=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- bunch of rework to make 'vars' a dependent of lots of sections
- should be emitting marker ids in the output stream now.

#9119
---
 common/web/types/src/ldml-keyboard/pattern-parser.ts    | 1 +
 developer/src/kmc-ldml/src/compiler/disp.ts             | 4 +++-
 developer/src/kmc-ldml/src/compiler/empty-compiler.ts   | 5 +++--
 developer/src/kmc-ldml/src/compiler/keys.ts             | 9 ++++++++-
 developer/src/kmc-ldml/src/compiler/section-compiler.ts | 3 ++-
 developer/src/kmc-ldml/src/compiler/tran.ts             | 4 ++++
 developer/src/kmc-ldml/src/compiler/vars.ts             | 2 +-
 developer/src/kmc-ldml/test/helpers/index.ts            | 3 ++-
 developer/src/kmc-ldml/test/test-tran.ts                | 3 +--
 developer/src/kmc-ldml/test/test-vars.ts                | 7 +++++--
 10 files changed, 30 insertions(+), 11 deletions(-)

diff --git a/common/web/types/src/ldml-keyboard/pattern-parser.ts b/common/web/types/src/ldml-keyboard/pattern-parser.ts
index 97d7c49cab..faee3b1594 100644
--- a/common/web/types/src/ldml-keyboard/pattern-parser.ts
+++ b/common/web/types/src/ldml-keyboard/pattern-parser.ts
@@ -93,6 +93,7 @@ export class MarkerParser {
 
   /** @returns all marker strings as sentinel values */
   public static toSentinelString(s: string, markers?: OrderedStringList) : string {
+    if (!s) return s;
     return s.replaceAll(this.REFERENCE, (sub, arg) => {
       if (arg === MarkerParser.ANY_MARKER_ID) {
         return MarkerParser.SENTINEL_ALL_MARKERS;
diff --git a/developer/src/kmc-ldml/src/compiler/disp.ts b/developer/src/kmc-ldml/src/compiler/disp.ts
index d644f56b88..241c5af685 100644
--- a/developer/src/kmc-ldml/src/compiler/disp.ts
+++ b/developer/src/kmc-ldml/src/compiler/disp.ts
@@ -44,9 +44,11 @@ export class DispCompiler extends SectionCompiler {
     // displayOptions
     result.baseCharacter = sections.strs.allocAndUnescapeString(this.keyboard.displays?.displayOptions?.baseCharacter);
 
+    // TODO-LDML: substitute variables!
+
     // displays
     result.disps = this.keyboard.displays?.display.map(display => ({
-      to: sections.strs.allocAndUnescapeString(display.to),
+      to: sections.strs.allocAndUnescapeString(sections.vars.substituteMarkerString(display.to)),
       display: sections.strs.allocAndUnescapeString(display.display),
     })) || []; // TODO-LDML: need coverage for the []
 
diff --git a/developer/src/kmc-ldml/src/compiler/empty-compiler.ts b/developer/src/kmc-ldml/src/compiler/empty-compiler.ts
index e167c784b2..93476d4497 100644
--- a/developer/src/kmc-ldml/src/compiler/empty-compiler.ts
+++ b/developer/src/kmc-ldml/src/compiler/empty-compiler.ts
@@ -1,6 +1,7 @@
 import { SectionIdent, constants } from '@keymanapp/ldml-keyboard-constants';
 import { SectionCompiler } from "./section-compiler.js";
 import { LDMLKeyboard, KMXPlus, CompilerCallbacks } from "@keymanapp/common-types";
+import { VarsCompiler } from './vars.js';
 
 /**
  * Compiler for typrs that don't actually consume input XML
@@ -69,6 +70,6 @@ export class UsetCompiler extends EmptyCompiler {
 }
 
 /**
- * For test use. The top three compilers.
+ * For test use. The top compilers.
  */
-export const BASIC_DEPENDENCIES = [ StrsCompiler, ListCompiler, ElemCompiler ];
+export const BASIC_DEPENDENCIES = [ StrsCompiler, ListCompiler, ElemCompiler, VarsCompiler ];
diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts
index 28cc1536b8..89d93178d3 100644
--- a/developer/src/kmc-ldml/src/compiler/keys.ts
+++ b/developer/src/kmc-ldml/src/compiler/keys.ts
@@ -132,6 +132,7 @@ export class KeysCompiler extends SectionCompiler {
         flicks.flicks.push({
           directions,
           flags,
+          // TODO-LDML: markers,variables
           to,
         });
       }
@@ -160,17 +161,23 @@ export class KeysCompiler extends SectionCompiler {
       const id = sections.strs.allocString(key.id);
       const longPress: ListItem = sections.list.allocListFromEscapedSpaces(
         sections.strs,
+        // TODO-LDML: markers,variables
         key.longPress
       );
       const longPressDefault = sections.strs.allocAndUnescapeString(
+        // TODO-LDML: markers,variables
         key.longPressDefault
       );
       const multiTap: ListItem = sections.list.allocListFromEscapedSpaces(
         sections.strs,
+        // TODO-LDML: markers,variables
         key.multiTap
       );
       const keySwitch = sections.strs.allocString(key.switch); // 'switch' is a reserved word
-      const to = sections.strs.allocAndUnescapeString(key.to, true);
+      const toRaw = key.to;
+      // TODO-LDML: variables
+      let toCooked = sections.vars.substituteMarkerString(toRaw);
+      const to = sections.strs.allocAndUnescapeString(toCooked, true);
       if (!to.isOneChar) {
         flags |= constants.keys_key_flags_extend;
       }
diff --git a/developer/src/kmc-ldml/src/compiler/section-compiler.ts b/developer/src/kmc-ldml/src/compiler/section-compiler.ts
index 5150108e6c..9e187fafe6 100644
--- a/developer/src/kmc-ldml/src/compiler/section-compiler.ts
+++ b/developer/src/kmc-ldml/src/compiler/section-compiler.ts
@@ -32,7 +32,8 @@ export class SectionCompiler {
     const defaults = new Set(<SectionIdent[]>[
       constants.section.strs,
       constants.section.list,
-      constants.section.elem
+      constants.section.elem,
+      constants.section.vars,
     ]);
     return defaults;
   }
diff --git a/developer/src/kmc-ldml/src/compiler/tran.ts b/developer/src/kmc-ldml/src/compiler/tran.ts
index 15db1b2220..fa1f828622 100644
--- a/developer/src/kmc-ldml/src/compiler/tran.ts
+++ b/developer/src/kmc-ldml/src/compiler/tran.ts
@@ -141,6 +141,10 @@ export class TransformCompiler<T extends TransformCompilerType, TranBase extends
       cookedTo = sections.vars.substituteStrings(cookedTo, sections);
     }
 
+    // add in markers. idempotent if no markers.
+    cookedFrom = sections.vars.substituteMarkerString(cookedFrom); // TODO-LDML: need to support \m{.} here, maybe other edge cases
+    cookedTo = sections.vars.substituteMarkerString(cookedTo);
+
     result.from = sections.strs.allocAndUnescapeString(cookedFrom); // TODO-LDML: not unescaped here, done previously
     result.to = sections.strs.allocAndUnescapeString(cookedTo); // TODO-LDML: not unescaped here, done previously
     return result;
diff --git a/developer/src/kmc-ldml/src/compiler/vars.ts b/developer/src/kmc-ldml/src/compiler/vars.ts
index 822f6e7d42..b6e6136750 100644
--- a/developer/src/kmc-ldml/src/compiler/vars.ts
+++ b/developer/src/kmc-ldml/src/compiler/vars.ts
@@ -188,7 +188,7 @@ export class VarsCompiler extends SectionCompiler {
     const result =  new Vars();
 
     const variables = this.keyboard?.variables;
-
+    // we always have vars, it's depended on by other sections 
     if (!variables) return result; // Empty vars, to simplify other sections
 
     // we already know the variables do not conflict with each other
diff --git a/developer/src/kmc-ldml/test/helpers/index.ts b/developer/src/kmc-ldml/test/helpers/index.ts
index 30dde83e26..a6e05ebf39 100644
--- a/developer/src/kmc-ldml/test/helpers/index.ts
+++ b/developer/src/kmc-ldml/test/helpers/index.ts
@@ -19,6 +19,7 @@ import DependencySections = KMXPlus.DependencySections;
 import Section = KMXPlus.Section;
 import { ElemCompiler, ListCompiler, StrsCompiler } from '../../src/compiler/empty-compiler.js';
 import { KmnCompiler } from '@keymanapp/kmc-kmn';
+import { VarsCompiler } from '../../src/compiler/vars.js';
 // import Vars = KMXPlus.Vars;
 
 /**
@@ -92,7 +93,7 @@ async function loadDepsFor(sections: DependencySections, parentCompiler: Section
   const parentId = parentCompiler.id;
   if (!dependencies) {
     // default dependencies
-    dependencies = [ StrsCompiler, ListCompiler, ElemCompiler ];
+    dependencies = [ StrsCompiler, ListCompiler, ElemCompiler, VarsCompiler ];
   }
   for (const dep of dependencies) {
     const compiler = new dep(source, callbacks);
diff --git a/developer/src/kmc-ldml/test/test-tran.ts b/developer/src/kmc-ldml/test/test-tran.ts
index 6e25cb4ac8..ec77ab02f2 100644
--- a/developer/src/kmc-ldml/test/test-tran.ts
+++ b/developer/src/kmc-ldml/test/test-tran.ts
@@ -1,7 +1,6 @@
 import 'mocha';
 import { assert } from 'chai';
 import { TranCompiler, BkspCompiler } from '../src/compiler/tran.js';
-import { VarsCompiler } from '../src/compiler/vars.js';
 import { BASIC_DEPENDENCIES, UsetCompiler } from '../src/compiler/empty-compiler.js';
 import { CompilerMessages } from '../src/compiler/messages.js';
 import { compilerTestCallbacks, testCompilationCases } from './helpers/index.js';
@@ -10,7 +9,7 @@ import { KMXPlus } from '@keymanapp/common-types';
 import Tran = KMXPlus.Tran;// for tests…
 import Bksp = KMXPlus.Bksp;// for tests…
 import { constants } from '@keymanapp/ldml-keyboard-constants';
-const tranDependencies = [ ...BASIC_DEPENDENCIES, UsetCompiler, VarsCompiler ];
+const tranDependencies = [ ...BASIC_DEPENDENCIES, UsetCompiler ];
 const bkspDependencies = tranDependencies;
 
 describe('tran', function () {
diff --git a/developer/src/kmc-ldml/test/test-vars.ts b/developer/src/kmc-ldml/test/test-vars.ts
index 6acaef35da..b18f546f27 100644
--- a/developer/src/kmc-ldml/test/test-vars.ts
+++ b/developer/src/kmc-ldml/test/test-vars.ts
@@ -5,6 +5,9 @@ import { CompilerMessages } from '../src/compiler/messages.js';
 import { CompilerMessages as KmnCompilerMessages } from '@keymanapp/kmc-kmn';
 import { testCompilationCases } from './helpers/index.js';
 import { KMXPlus } from '@keymanapp/common-types';
+import { BASIC_DEPENDENCIES } from '../src/compiler/empty-compiler.js';
+// now that 'everything' depends on vars, we need an explicit dependency here
+const varsDependencies = BASIC_DEPENDENCIES.filter(c => c !== VarsCompiler);
 
 import Vars = KMXPlus.Vars;
 
@@ -182,7 +185,7 @@ describe('vars', function () {
         CompilerMessages.Error_MissingStringVariable({id: 'missingStringInSet'})
       ],
     },
-  ]);
+  ], varsDependencies);
   describe('markers', function () {
     this.slow(500); // 0.5 sec -- json schema validation takes a while
 
@@ -208,6 +211,6 @@ describe('vars', function () {
           }),
         ],
       },
-    ]);
+    ], varsDependencies);
   });
 });

From 1c9543557fa99d65492c56cb2fa15da14da2ccac Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Sat, 29 Jul 2023 17:56:10 -0500
Subject: [PATCH 50/83] =?UTF-8?q?feat(developer):=20marker=20-=20basic.xml?=
 =?UTF-8?q?=20update=20=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- verified sentinel in the binary output

#9119
---
 .../src/kmc-ldml/test/fixtures/basic.txt      | 22 +++++++++++++++++--
 .../src/kmc-ldml/test/fixtures/basic.xml      |  5 +++++
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/developer/src/kmc-ldml/test/fixtures/basic.txt b/developer/src/kmc-ldml/test/fixtures/basic.txt
index 8713f93ae1..cbbb403891 100644
--- a/developer/src/kmc-ldml/test/fixtures/basic.txt
+++ b/developer/src/kmc-ldml/test/fixtures/basic.txt
@@ -426,6 +426,7 @@ block(strs)                         #  struct COMP_KMXPLUS_STRS {
   diff(strs,strTranTo)          sizeof(strTranTo,2)
   diff(strs,strKeys)       sizeof(strKeys,2)
   diff(strs,strIndicator)  sizeof(strIndicator,2)
+  diff(strs,strSentinel0001)  sizeof(strSentinel0001,2)
 
 
   # String table -- block(x) is used to store the null u16char at end of each string
@@ -464,7 +465,7 @@ block(strs)                         #  struct COMP_KMXPLUS_STRS {
   block(strKeys)             90 17 b6 17 block(x) 00 00                               # 'ថា'
   # <reorder before="ᩫ" from="᩠᩵ᩅ" order="10 55 10" />
   block(strIndicator)        3d d8 40 de block(x) 00 00                               # '🙀'
-
+  block(strSentinel0001)     FF FF 01 00 block(x) 00 00                   # U+FFFF U+0001
 
 
 
@@ -484,10 +485,15 @@ block(tran)                         # struct COMP_KMXPLUS_TRAN {
   block(tranGroupStart)             # COMP_KMXPLUS_TRAN_GROUP
     # group 0
     00 00 00 00                       # KMX_DWORD type = transform
-    01 00 00 00                       # KMX_DWORD count
+    02 00 00 00                       # KMX_DWORD count
     diff(tranTransformStart,tranTransform0,16) # KMX_DWORD index
 
     # group 1
+    00 00 00 00                       # KMX_DWORD type = transform
+    01 00 00 00                       # KMX_DWORD count
+    diff(tranTransformStart,tranTransform2,16) # KMX_DWORD index
+
+    # group 2
     01 00 00 00                       # KMX_DWORD type = reorder
     01 00 00 00                       # KMX_DWORD count
     diff(tranReorderStart,tranReorder0,8) # KMX_DWORD index
@@ -500,6 +506,18 @@ block(tran)                         # struct COMP_KMXPLUS_TRAN {
     index(strNull,strNull,2)          # mapFrom
     index(strNull,strNull,2)          # mapTo
 
+  block(tranTransform1)
+    index(strNull,strA,2)  #    KMXPLUS_STR from; 'a'
+    index(strNull,strSentinel0001,2)        #    KMXPLUS_STR to; \m{a}
+    index(strNull,strNull,2)          # mapFrom
+    index(strNull,strNull,2)          # mapTo
+
+  block(tranTransform2)  # Next group
+    index(strNull,strSentinel0001,2)  #    KMXPLUS_STR from; (\m{a})
+    index(strNull,strNull,2)        #    KMXPLUS_STR to; (none)
+    index(strNull,strNull,2)          # mapFrom
+    index(strNull,strNull,2)          # mapTo
+
   # reorders
   block(tranReorderStart)           # COMP_KMXPLUS_TRAN_REORDER
     block(tranReorder0)
diff --git a/developer/src/kmc-ldml/test/fixtures/basic.xml b/developer/src/kmc-ldml/test/fixtures/basic.xml
index cb0c56d269..2d345324d9 100644
--- a/developer/src/kmc-ldml/test/fixtures/basic.xml
+++ b/developer/src/kmc-ldml/test/fixtures/basic.xml
@@ -46,6 +46,11 @@
   <transforms type="simple">
     <transformGroup>
       <transform from="^a" to="â" />
+      <transform from="a" to="\m{a}" />
+    </transformGroup>
+    <transformGroup>
+      <!-- delete that marker -->
+      <transform from="\m{a}" />
     </transformGroup>
 
     <transformGroup>

From 0d3ffbbe57315b23b84a4378cdf508401b6ead5e Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Sat, 29 Jul 2023 20:30:59 -0500
Subject: [PATCH 51/83] =?UTF-8?q?chore(core):=20marker=20-=20pt-abnt2=20br?=
 =?UTF-8?q?oken=20=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- upstream in CLDR is incomplete in marker/transform implementation, disabling for now

#9119
---
 core/tests/unit/ldml/keyboards/meson.build | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/tests/unit/ldml/keyboards/meson.build b/core/tests/unit/ldml/keyboards/meson.build
index 93ccfaad63..3d1bfdddcc 100644
--- a/core/tests/unit/ldml/keyboards/meson.build
+++ b/core/tests/unit/ldml/keyboards/meson.build
@@ -9,7 +9,7 @@
 # tests     in resources/standards-data/ldml-keyboards/techpreview/test/
 tests_from_cldr = [
   'ja-Latn',
-  'pt-k0-abnt2',
+  # 'pt-k0-abnt2', #TODO-LDML: marker syntax fail!
   'fr-t-k0-azerty',
 ]
 

From 0e5fccac00cced1c44e977422c7a0a16673c6163 Mon Sep 17 00:00:00 2001
From: Keyman Build Agent <buildagent@keyman.com>
Date: Sun, 30 Jul 2023 14:02:33 -0400
Subject: [PATCH 52/83] auto: increment master version to 17.0.150

---
 HISTORY.md | 5 +++++
 VERSION.md | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/HISTORY.md b/HISTORY.md
index b4d57e1c3e..4b5d8c3b27 100644
--- a/HISTORY.md
+++ b/HISTORY.md
@@ -1,5 +1,10 @@
 # Keyman Version History
 
+## 17.0.149 alpha 2023-07-30
+
+* fix(core): Better range check for Uni_IsValid() (#9346)
+* chore(core): update documentation in transform logic and processor (#9352)
+
 ## 17.0.148 alpha 2023-07-27
 
 * feat(core): merge transform/reorder processing w/ u32 (#9293)
diff --git a/VERSION.md b/VERSION.md
index c826535c22..9addf7a986 100644
--- a/VERSION.md
+++ b/VERSION.md
@@ -1 +1 @@
-17.0.149
\ No newline at end of file
+17.0.150
\ No newline at end of file

From c4efba04d9af8e5b3618ccbfd7a75c17c42a25d4 Mon Sep 17 00:00:00 2001
From: Eberhard Beilharz <eb1@sil.org>
Date: Mon, 31 Jul 2023 09:54:09 +0200
Subject: [PATCH 53/83] chore(linux): Don't fail on parallel builds

If another build runs on the same build agent at the same time and
installs dependencies at the same time, previously the build failed
because apt/dpkg was already running. This change checks for the
existence of the apt/dpkg lock file and waits until the other process
is finished before starting the installation of dependencies.
---
 linux/scripts/package-build.inc.sh | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/linux/scripts/package-build.inc.sh b/linux/scripts/package-build.inc.sh
index 6c249891c1..fcfe492b43 100644
--- a/linux/scripts/package-build.inc.sh
+++ b/linux/scripts/package-build.inc.sh
@@ -49,6 +49,14 @@ function downloadSource() {
     sha256sum -c --ignore-missing SHA256SUMS |grep "${proj}"
 }
 
+function wait_for_apt_deb {
+    # from https://gist.github.com/hrpatel/117419dcc3a75e46f79a9f1dce99ef52
+    while sudo fuser /var/{lib/{dpkg,apt/lists},cache/apt/archives}/lock &>/dev/null 2>&1; do
+       echo "Waiting for apt/dpkg lock to release, sleeping 10s"
+       sleep 10
+    done
+}
+
 function checkAndInstallRequirements()
 {
 	local TOINSTALL=""
@@ -63,12 +71,12 @@ function checkAndInstallRequirements()
 	export DEBIAN_FRONTEND=noninteractive
 
 	if [ -n "$TOINSTALL" ]; then
-		sudo apt-get update
+		wait_for_apt_deb && sudo apt-get update
 		# shellcheck disable=SC2086
-		sudo apt-get -qy install $TOINSTALL
+		wait_for_apt_deb && sudo apt-get -qy install $TOINSTALL
 	fi
 
 	sudo mk-build-deps debian/control
-	sudo apt-get -qy --allow-downgrades install ./keyman-build-deps_*.deb
+	wait_for_apt_deb && sudo apt-get -qy --allow-downgrades install ./keyman-build-deps_*.deb
 	sudo rm -f keyman-buid-deps_*
 }

From 1d0db5945a78d49f834179ecfd97c63592b0df1e Mon Sep 17 00:00:00 2001
From: Eberhard Beilharz <eb1@sil.org>
Date: Thu, 27 Jul 2023 16:58:16 +0200
Subject: [PATCH 54/83] docs(core): Don't put emscripten on the path

This change addresses the code review comments. We don't want
emscripten to be in `PATH` and instead set the `EMSCRIPTEN_BASE`
environment variable.
---
 docs/build/linux-ubuntu.md | 10 ++++------
 linux/Dockerfile           |  9 +++++----
 2 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/docs/build/linux-ubuntu.md b/docs/build/linux-ubuntu.md
index 44c4056862..f8c29dc0fe 100644
--- a/docs/build/linux-ubuntu.md
+++ b/docs/build/linux-ubuntu.md
@@ -80,12 +80,10 @@ git clone https://github.com/emscripten-core/emsdk.git
 cd emsdk
 ./emsdk install latest
 ./emsdk activate latest
-source ./emsdk_env.sh
-export EMSDK_NODE=/usr/bin/node
+export EMSCRIPTEN_BASE=$(pwd)/upstream/emscripten
 ```
 
-*emscripten* comes with an older version of node, so it's important to set
-the `EMSDK_NODE` environment variable to the node version we need.
+**NOTE:** Don't put EMSDK on the path, i.e. don't source `emsdk_env.sh`.
 
 ### Building Keyman Core
 
@@ -114,7 +112,7 @@ mkdir -p build/linux
 docker run -it --rm -v $(pwd)/..:/home/build/build \
   -v $(pwd)/build/linux:/home/build/build/core/build \
   keymanapp/keyman-linux-builder:latest \
-  bash -c 'core/build.sh --debug'
+  wrapper core/build.sh --debug
 ```
 
 - linux
@@ -124,7 +122,7 @@ docker run -it --rm -v $(pwd)/..:/home/build/build \
 cd $(git rev-parse --show-toplevel)
 docker run -it --rm -v $(pwd):/home/build/build \
   keymanapp/keyman-linux-builder:latest \
-  bash -c 'DESTDIR=/home/build linux/build.sh --debug build install'
+  wrapper 'DESTDIR=/home/build linux/build.sh --debug build install'
 ```
 
 ## Keyman for Android
diff --git a/linux/Dockerfile b/linux/Dockerfile
index 49c696cc25..9e8339440b 100644
--- a/linux/Dockerfile
+++ b/linux/Dockerfile
@@ -20,7 +20,7 @@ ENV DEBCONF_NOWARNINGS yes
 
 # Update to the latest
 RUN apt-get -q -y update &&  \
-  apt-get -q -y install devscripts equivs meson python3 python3-setuptools software-properties-common && \
+  apt-get -q -y install devscripts equivs meson python3 python3-setuptools software-properties-common curl && \
   add-apt-repository ppa:keymanapp/keyman && \
   add-apt-repository ppa:keymanapp/keyman-alpha
 RUN apt-get -q -y update &&  \
@@ -41,9 +41,10 @@ RUN cd /usr/share && \
   cd emsdk && \
   ./emsdk install latest && \
   ./emsdk activate latest && \
-  echo 'source "/usr/share/emsdk/emsdk_env.sh"' >> $HOME/.bashrc && \
-  echo "export EMSDK_NODE=/usr/bin/node" >> $HOME/.bashrc && \
-  echo 'echo "node $(node --version)"' >> $HOME/.bashrc
+  echo "#!/bin/bash" > /usr/bin/wrapper && \
+  echo "export EMSCRIPTEN_BASE=/usr/share/emsdk/upstream/emscripten" >> /usr/bin/wrapper && \
+  echo "bash -c \"\${@:-bash}\"">> /usr/bin/wrapper && \
+  chmod +x /usr/bin/wrapper
 
 # now, switch to build user
 USER build

From 8c2cd8916ae24ed9db68539d02fa0ae78d8a7ce7 Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Mon, 31 Jul 2023 11:16:49 -0500
Subject: [PATCH 55/83] fix(developer): fix breakage from emscripten 3.1.44

- use wasmExports.malloc if available otherwise asm.malloc
---
 developer/src/kmc-kmn/src/compiler/compiler.ts | 8 +++++++-
 developer/src/kmcmplib/src/meson.build         | 2 +-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/developer/src/kmc-kmn/src/compiler/compiler.ts b/developer/src/kmc-kmn/src/compiler/compiler.ts
index 499c74abe9..27c7bf3138 100644
--- a/developer/src/kmc-kmn/src/compiler/compiler.ts
+++ b/developer/src/kmc-kmn/src/compiler/compiler.ts
@@ -237,7 +237,13 @@ export class KmnCompiler implements UnicodeSetParser {
       return null;
     }
 
-    const buf = this.Module.asm.malloc(rangeCount * 2 * this.Module.HEAPU32.BYTES_PER_ELEMENT);
+    let malloc = this.Module?.wasmExports?.malloc || this.Module?.asm?.malloc;
+
+    if (!malloc) {
+      throw new Error(`Internal Error: missing wasmExports.malloc() / asm.malloc()`);
+    }
+
+    const buf = malloc(rangeCount * 2 * this.Module.HEAPU32.BYTES_PER_ELEMENT);
     // TODO-LDML: Catch OOM
     /** return code, if positive: range count */
     const rc = this.Module.kmcmp_parseUnicodeSet(pattern, buf, rangeCount * 2);
diff --git a/developer/src/kmcmplib/src/meson.build b/developer/src/kmcmplib/src/meson.build
index 605292583d..21e8f2227b 100644
--- a/developer/src/kmcmplib/src/meson.build
+++ b/developer/src/kmcmplib/src/meson.build
@@ -28,7 +28,7 @@ if cpp_compiler.get_id() == 'emscripten'
   # wasm-exceptions supported in Node 18+, Chrome 95+, Firefox 100+, Safari 15.2+
   flags += ['-fwasm-exceptions']
   lib_links = ['--whole-archive', '-sMODULARIZE', '-sEXPORT_ES6']
-  links += ['-fwasm-exceptions', '--bind', '-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\']']
+  links += ['-fwasm-exceptions', '--bind', '-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\',\'wasmExports\']']
 endif
 
 icu = subproject('icu-for-uset', default_options: [ 'default_library=static', 'cpp_std=c++17', 'warning_level=0', 'werror=false'])

From ea14dae0cd2bccf20320b1a3acdb79b3127d0bae Mon Sep 17 00:00:00 2001
From: Keyman Build Agent <buildagent@keyman.com>
Date: Mon, 31 Jul 2023 14:02:07 -0400
Subject: [PATCH 56/83] auto: increment master version to 17.0.151

---
 HISTORY.md | 5 +++++
 VERSION.md | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/HISTORY.md b/HISTORY.md
index 4b5d8c3b27..04f84fef16 100644
--- a/HISTORY.md
+++ b/HISTORY.md
@@ -1,5 +1,10 @@
 # Keyman Version History
 
+## 17.0.150 alpha 2023-07-31
+
+* chore(linux): Update debian changelog (#9358)
+* chore(linux): Fix creation of PRs after uploading to Debian (#9360)
+
 ## 17.0.149 alpha 2023-07-30
 
 * fix(core): Better range check for Uni_IsValid() (#9346)
diff --git a/VERSION.md b/VERSION.md
index 9addf7a986..d1b3623053 100644
--- a/VERSION.md
+++ b/VERSION.md
@@ -1 +1 @@
-17.0.150
\ No newline at end of file
+17.0.151
\ No newline at end of file

From 1bb3ae548a25194f83a72448d6cac6ec6f51a74b Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Mon, 31 Jul 2023 13:19:08 -0500
Subject: [PATCH 57/83] =?UTF-8?q?fix(developer):=20more=20wasm=20uset=20fi?=
 =?UTF-8?q?xes=20=F0=9F=99=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- While i'm in the neighborhood..
- use 'free' besides just 'malloc' (sure enough..)
- move wasmExports into this
- add code coverage for failed sizes
- cleanup
---
 .../src/kmc-kmn/src/compiler/compiler.ts      | 30 +++++++++++--------
 developer/src/kmc-kmn/test/test-wasm-uset.ts  | 25 ++++++++++++----
 2 files changed, 36 insertions(+), 19 deletions(-)

diff --git a/developer/src/kmc-kmn/src/compiler/compiler.ts b/developer/src/kmc-kmn/src/compiler/compiler.ts
index 6994498522..987aea874f 100644
--- a/developer/src/kmc-kmn/src/compiler/compiler.ts
+++ b/developer/src/kmc-kmn/src/compiler/compiler.ts
@@ -46,10 +46,17 @@ let callbackProcIdentifier = 0;
 const
   callbackPrefix = 'kmnCompilerCallbacks_';
 
+interface MallocAndFree {
+  malloc(sz: number) : number;
+  free(p: number) : null;
+};
+
+
 export class KmnCompiler implements UnicodeSetParser {
   private Module: any;
   callbackID: string; // a unique numeric id added to globals with prefixed names
   callbacks: CompilerCallbacks;
+  wasmExports: MallocAndFree;
 
   constructor() {
     this.callbackID = callbackPrefix + callbackProcIdentifier.toString();
@@ -61,6 +68,7 @@ export class KmnCompiler implements UnicodeSetParser {
     if(!this.Module) {
       try {
         this.Module = await loadWasmHost();
+        this.wasmExports = (this.Module.wasmExports ?? this.Module.asm);
       } catch(e: any) {
         /* c8 ignore next 3 */
         this.callbacks.reportMessage(CompilerMessages.Fatal_MissingWasmModule({e}));
@@ -237,28 +245,23 @@ export class KmnCompiler implements UnicodeSetParser {
       return null;
     }
 
-    const malloc = (this.Module.wasmExports ?? this.Module.asm)?.malloc;
-
-
-    const buf = malloc(rangeCount * 2 * this.Module.HEAPU32.BYTES_PER_ELEMENT);
     // TODO-LDML: Catch OOM
-    /** return code, if positive: range count */
+    const buf = this.wasmExports.malloc(rangeCount * 2 * this.Module.HEAPU32.BYTES_PER_ELEMENT);
+    /** If <= 0: return code. If positive: range count */
     const rc = this.Module.kmcmp_parseUnicodeSet(pattern, buf, rangeCount * 2);
     if (rc >= 0) {
       const ranges = [];
       const startu = (buf / this.Module.HEAPU32.BYTES_PER_ELEMENT);
       for (let i = 0; i < rc; i++) {
-        const low = this.Module.HEAPU32[startu + (i * 2) + 0];
-        const high = this.Module.HEAPU32[startu + (i * 2) + 1];
-        ranges.push([low, high]);
+        const start  = this.Module.HEAPU32[startu + (i * 2) + 0];
+        const end    = this.Module.HEAPU32[startu + (i * 2) + 1];
+        ranges.push([start, end]);
       }
-      // TODO-LDML: no free??
-      // Module.asm.free(buf);
+      this.wasmExports.free(buf);
       return new UnicodeSet(pattern, ranges);
     } else {
-      // translate error
-      // TODO-LDML: no free??
-      // Module.asm.free(buf);
+      this.wasmExports.free(buf);
+      // translate error code into callback
       this.callbacks.reportMessage(getUnicodeSetError(rc));
       return null;
     }
@@ -268,6 +271,7 @@ export class KmnCompiler implements UnicodeSetParser {
       /* c8 ignore next 2 */
       return null;
     }
+    // call with rangeCount = 0 to invoke in 'preflight' mode.
     const rc = this.Module.kmcmp_parseUnicodeSet(pattern, 0, 0);
     if (rc >= 0) {
       return rc;
diff --git a/developer/src/kmc-kmn/test/test-wasm-uset.ts b/developer/src/kmc-kmn/test/test-wasm-uset.ts
index 2a05c24f26..a924f9fe05 100644
--- a/developer/src/kmc-kmn/test/test-wasm-uset.ts
+++ b/developer/src/kmc-kmn/test/test-wasm-uset.ts
@@ -65,12 +65,25 @@ describe('Compiler UnicodeSet function', function() {
       '[[]': CompilerMessages.ERROR_UnicodeSetSyntaxError,
     };
     for(const [pat, expected] of Object.entries(failures)) {
-      callbacks.clear();
-      assert.notOk(compiler.parseUnicodeSet(pat, 1));
-      assert.equal(callbacks.messages.length, 1);
-      const firstMessage = callbacks.messages[0];
-      const code = firstMessage.code;
-      assert.equal(code, expected, `${compilerErrorFormatCode(code)}≠${compilerErrorFormatCode(expected)} got ${firstMessage.message} for ${pat}`);
+      {
+        // verify fails parse
+        callbacks.clear();
+        assert.notOk(compiler.parseUnicodeSet(pat, 1));
+        assert.equal(callbacks.messages.length, 1);
+        const firstMessage = callbacks.messages[0];
+        const code = firstMessage.code;
+        assert.equal(code, expected, `${compilerErrorFormatCode(code)}≠${compilerErrorFormatCode(expected)} got ${firstMessage.message} for parsing ${pat}`);
+      }
+      // skip 'out of range' because that one won't fail during sizing.
+      if (expected !== CompilerMessages.FATAL_UnicodeSetOutOfRange) {
+        // verify fails size
+        callbacks.clear();
+        assert.equal(compiler.sizeUnicodeSet(pat), -1, `sizing ${pat}`);
+        assert.equal(callbacks.messages.length, 1);
+        const firstMessage = callbacks.messages[0];
+        const code = firstMessage.code;
+        assert.equal(code, expected, `${compilerErrorFormatCode(code)}≠${compilerErrorFormatCode(expected)} got ${firstMessage.message} for sizing ${pat}`);
+      }
     }
   });
 });

From f2fea17b66effbf3e857e43a1d98937a702e9be9 Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Mon, 31 Jul 2023 15:57:44 -0500
Subject: [PATCH 58/83] Update
 developer/src/kmc-ldml/test/fixtures/sections/vars/fail-markers-badref-0.xml

Co-authored-by: Marc Durdin <marc@durdin.net>
---
 .../test/fixtures/sections/vars/fail-markers-badref-0.xml       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/developer/src/kmc-ldml/test/fixtures/sections/vars/fail-markers-badref-0.xml b/developer/src/kmc-ldml/test/fixtures/sections/vars/fail-markers-badref-0.xml
index 063b5a5d11..fd63c36b02 100644
--- a/developer/src/kmc-ldml/test/fixtures/sections/vars/fail-markers-badref-0.xml
+++ b/developer/src/kmc-ldml/test/fixtures/sections/vars/fail-markers-badref-0.xml
@@ -13,7 +13,7 @@ This will fail because the two markers given don't exist anywhere.
   <displays>
     <!-- Not ok, nobody emits this, so this display doesn't make sense -->
     <display to="\m{doesnt_exist_1}" display="¬" />
-    <!-- OK - it si in the strings list.  Could warn that it matches no keys -->
+    <!-- OK - it is in the strings list.  Could warn that it matches no keys -->
     <display to="\m{maybe_emitted}" display="‽" />
   </displays>
 

From e1d5513952980e928d7f489192a275a84e808ed3 Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Mon, 31 Jul 2023 22:09:34 -0500
Subject: [PATCH 59/83] Update developer/src/kmcmplib/src/meson.build

Co-authored-by: Marc Durdin <marc@durdin.net>
---
 developer/src/kmcmplib/src/meson.build | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/developer/src/kmcmplib/src/meson.build b/developer/src/kmcmplib/src/meson.build
index 21e8f2227b..8d68c54979 100644
--- a/developer/src/kmcmplib/src/meson.build
+++ b/developer/src/kmcmplib/src/meson.build
@@ -28,7 +28,15 @@ if cpp_compiler.get_id() == 'emscripten'
   # wasm-exceptions supported in Node 18+, Chrome 95+, Firefox 100+, Safari 15.2+
   flags += ['-fwasm-exceptions']
   lib_links = ['--whole-archive', '-sMODULARIZE', '-sEXPORT_ES6']
-  links += ['-fwasm-exceptions', '--bind', '-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\',\'wasmExports\']']
+  links += ['-fwasm-exceptions', '--bind']
+  if cpp_compiler.version().version_compare('>=3.1.44')
+    # emscripten 3.1.44 removes .asm object and so we need to export `wasmExports`
+    # #9375; https://github.com/emscripten-core/emscripten/blob/main/ChangeLog.md#3144---072523
+    links += ['-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\',\'wasmExports\']']
+  else
+    # emscripten < 3.1.44 does not include `wasmExports`
+    links += ['-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\']']
+  endif
 endif
 
 icu = subproject('icu-for-uset', default_options: [ 'default_library=static', 'cpp_std=c++17', 'warning_level=0', 'werror=false'])

From 51774b637be89044843ef16abbb78cd954b7d265 Mon Sep 17 00:00:00 2001
From: "Steven R. Loomis" <srl295@gmail.com>
Date: Mon, 31 Jul 2023 22:23:37 -0500
Subject: [PATCH 60/83] Apply suggestions from code review

Co-authored-by: Marc Durdin <marc@durdin.net>
---
 developer/src/kmc-kmn/src/compiler/compiler.ts | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/developer/src/kmc-kmn/src/compiler/compiler.ts b/developer/src/kmc-kmn/src/compiler/compiler.ts
index 27c7bf3138..6994498522 100644
--- a/developer/src/kmc-kmn/src/compiler/compiler.ts
+++ b/developer/src/kmc-kmn/src/compiler/compiler.ts
@@ -237,11 +237,8 @@ export class KmnCompiler implements UnicodeSetParser {
       return null;
     }
 
-    let malloc = this.Module?.wasmExports?.malloc || this.Module?.asm?.malloc;
+    const malloc = (this.Module.wasmExports ?? this.Module.asm)?.malloc;
 
-    if (!malloc) {
-      throw new Error(`Internal Error: missing wasmExports.malloc() / asm.malloc()`);
-    }
 
     const buf = malloc(rangeCount * 2 * this.Module.HEAPU32.BYTES_PER_ELEMENT);
     // TODO-LDML: Catch OOM

From f6a64bc223491e35cea88ced6936be029a81e3ce Mon Sep 17 00:00:00 2001
From: Ross <writetoross@gmail.com>
Date: Tue, 1 Aug 2023 14:59:41 +1000
Subject: [PATCH 61/83] docs(windows): corrected nmake cmd for certificates

---
 windows/src/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/windows/src/README.md b/windows/src/README.md
index 9277b9c603..d8ca3b3901 100644
--- a/windows/src/README.md
+++ b/windows/src/README.md
@@ -59,13 +59,13 @@ will need to uninstall and reinstall.
 
 ## Certificates
 
-In order to create a release build, you will need a code signing certiicate.
+In order to create a release build, you will need a code signing certificate.
 You can use your own certificate, or you can use test certificates which are not
 globally trusted. The environment variables `SC_PFX_SHA1` and `SC_PFX_SHA256`
 can be set to custom certificate paths.
 
 The Keyman repo no longer includes test certificates.  To build your own, run
-`nmake test-certificates` from **common/windows/delphi/tools/certificates** to
+`nmake test-certificate` from **common/windows/delphi/tools/certificates** to
 build and install your own local root CA "**KeymanTestCA**" certificates. If you
 specify a password for the certificate, you'll need to set that in the
 environment variable `SC_PWD`.

From b8d0777800188144f6eee086ab82cf56962c3dd4 Mon Sep 17 00:00:00 2001
From: Eberhard Beilharz <eb1@sil.org>
Date: Tue, 1 Aug 2023 15:03:47 +0200
Subject: [PATCH 62/83] chore(linux): Remove package build on Jenkins for
 Keyman 17

---
 docs/linux/packaging.md                     | 210 ++++----------------
 linux/Jenkinsfile                           |  11 -
 linux/scripts/dist.sh                       |   2 +-
 linux/scripts/jenkins.sh                    |  65 ------
 resources/build/build-utils.sh              |   2 +-
 resources/build/increment-version.sh        |   2 +-
 resources/build/run-required-test-builds.sh |   6 +-
 resources/build/trigger-builds.inc.sh       |  62 +-----
 resources/build/trigger-definitions.inc.sh  |  15 +-
 9 files changed, 52 insertions(+), 323 deletions(-)
 delete mode 100644 linux/Jenkinsfile
 delete mode 100755 linux/scripts/jenkins.sh

diff --git a/docs/linux/packaging.md b/docs/linux/packaging.md
index bfb6a9d13d..c133575d04 100644
--- a/docs/linux/packaging.md
+++ b/docs/linux/packaging.md
@@ -10,7 +10,7 @@ We use different channels to build and distribute the Linux packages:
   [alpha](https://launchpad.net/~keymanapp/+archive/ubuntu/keyman-alpha) versions
 - [pso](http://packages.sil.org/) and [llso](http://linux.lsdev.sil.org/ubuntu/)
   for stable, beta, and alpha versions
-- artifacts on [Jenkins](https://jenkins.lsdev.sil.org/view/Keyman/view/Pipeline/job/pipeline-keyman-packaging/view/change-requests/)
+- artifacts on [GitHub](https://github.com/keymanapp/keyman/actions/workflows/deb-packaging.yml)
   for pull requests
 
 Packages on [llso](http://linux.lsdev.sil.org/ubuntu/) are uploaded automatically and are
@@ -21,191 +21,63 @@ pso enabled.
 ## Package builds
 
 Package builds happen on [Launchpad](#package-builds-on-launchpad) and
-[Jenkins](#package-builds-on-jenkins). Package builds for the official Ubuntu/Debian
+[GitHub](#github-actions-package-builds). Package builds for the official Ubuntu/Debian
 repos happen outside of our control. However, we
 [upload source packages](#uploading-debian-source-packages) to the Debian community.
 
-## Package builds on Jenkins
+## GitHub Actions package builds
 
 ### Build jobs
 
-The definition of the packaging jobs, the triggering of the jobs and the necessary build scripts
-are scattered over several source repos:
+The [Keyman GitHub repo](https://github.com/keymanapp/keyman) contains various
+scripts that are used to trigger a build and as part of the package build,
+and of course the source code for the packages:
 
-- [ci-builder-scripts](https://github.com/sillsdev/ci-builder-scripts) contains the definition of
-  a meta job (multi-branch pipeline job) that gets triggered when a change gets pushed to the
-  [Keyman GitHub repo](https://github.com/keymanapp/keyman). The meta job creates a new build
-  configuration/job for each branch/pull request on GitHub. The new job gets triggered to initialize
-  itself, but then exits immediately. We use the Jenkins
-  [Job DSL plugin](https://github.com/jenkinsci/job-dsl-plugin/wiki) to define the meta job.
+- [.github/workflows/deb-packaging.yml](https://github.com/keymanapp/keyman/blob/master/.github/workflows/deb-packaging.yml)
+  contains the definition of the packaging GHA
+- [resources/build/run-required-test-builds.sh](https://github.com/keymanapp/keyman/blob/master/resources/build/run-required-test-builds.sh)
+  runs on [TeamCity](https://build.palaso.org/buildConfiguration/Keyman_Test?)
+  to trigger the builds for the various platforms, among them the GHA package build.
+- [resources/build/increment-version.sh](https://github.com/keymanapp/keyman/blob/master/resources/build/increment-version.sh)
+  runs on [TeamCity](https://build.palaso.org/buildConfiguration/Keyman_TriggerReleaseBuildsMaster?)
+  and increments the version number before triggering the builds for the
+  various platforms.
+- The [linux/scripts](https://github.com/keymanapp/keyman/tree/master/linux/scripts)
+  subdirectory contains `bash` scripts that are used during the package build.
+  Some are only needed for Launchpad builds.
 
-  ci-builder-scripts also contains several generic scripts to set up a package build environment
-  (using `sbuilder`) and for building source and binary packages. These scripts are shared with
-  other projects.
+  - [deb-packaging.sh](https://github.com/keymanapp/keyman/blob/master/linux/scripts/deb-packaging.sh)
+    gets called by the packaging GHA to install dependencies, create the source
+    package and to verify the API.
 
-  The Keyman GitHub repo defines a webhook that triggers the meta job on Jenkins.
-
-  Changes to ci-builder-scripts go through [Gerrit](https://gerrit.lsdev.sil.org). See
-  [CONTRIBUTING.md](https://github.com/sillsdev/ci-builder-scripts/blob/master/CONTRIBUTING.md)
-  for details.
-
-  File structure:
-
-  - [groovy/KeymanPackagingJobs.groovy](https://github.com/sillsdev/ci-builder-scripts/blob/master/groovy/KeymanPackagingJobs.groovy)
-    contains the meta job definition
-  - The [bash/](https://github.com/sillsdev/ci-builder-scripts/tree/master/bash) subdirectory
-    contains `bash` scripts:
-
-    - [setup.sh](https://github.com/sillsdev/ci-builder-scripts/blob/master/bash/setup.sh) -
-      setup sbuild chroot environment
-    - [update](https://github.com/sillsdev/ci-builder-scripts/blob/master/bash/update) -
-      update the sbuild chroot environment
-    - [build-package](https://github.com/sillsdev/ci-builder-scripts/blob/master/bash/build-package) -
-      create a binary package
-
-- [lsdev-pipeline-library](https://github.com/sillsdev/lsdev-pipeline-library) contains a reusable
-  Jenkins pipeline library. The
-  [vars/keymanPackaging.groovy](https://github.com/sillsdev/lsdev-pipeline-library/blob/master/vars/keymanPackaging.groovy)
-  file contains the bulk of the logic of the Keyman packaging job.
-
-- The [Keyman GitHub repo](https://github.com/keymanapp/keyman) contains various scripts that are
-  used to trigger a build and as part of the package build, and of course the source code for the
-  packages:
-
-  - [resources/build/run-required-test-builds.sh](https://github.com/keymanapp/keyman/blob/master/resources/build/run-required-test-builds.sh)
-    runs on [TeamCity](https://build.palaso.org/buildConfiguration/Keyman_Test?) to trigger the
-    builds for the various platforms, among them the Jenkins package build.
-  - [resources/build/increment-version.sh](https://github.com/keymanapp/keyman/blob/master/resources/build/increment-version.sh)
-    runs on [TeamCity](https://build.palaso.org/buildConfiguration/Keyman_TriggerReleaseBuildsMaster?)
-    and increments the version number before triggering the builds for the various platforms.
-  - [linux/Jenkinsfile](https://github.com/keymanapp/keyman/blob/master/linux/Jenkinsfile) is a flag
-    for the meta job. If the meta job finds this file, it will create a new build configuration. This
-    file simply calls the packaging functionality defined in `lsdev-pipeline-library` and passes the
-    distributions and architectures to build as parameters.
-  - [linux/build/agent/install-deps](https://github.com/keymanapp/keyman/blob/master/linux/build/agent/install-deps)
-    installs dependencies on the current build agent.
-  - The [linux/scripts](https://github.com/keymanapp/keyman/tree/master/linux/scripts) subdirectory
-    contains `bash` scripts that are used during the package build. Some are only needed for
-    Launchpad builds.
-
-    - [jenkins.sh](https://github.com/keymanapp/keyman/blob/master/linux/scripts/jenkins.sh)
-      gets called from `lsdev-pipeline-library` to create a source package.
-
-  - [linux/debian](https://github.com/keymanapp/keyman/tree/master/linux/debian) - this is the `debian`
-    subdirectory for Keyman for Linux with the meta data for the Linux package.
-    See [Debian New Maintainers' Guide](https://www.debian.org/doc/manuals/maint-guide/) for
-    details to the various files.
+- [linux/debian](https://github.com/keymanapp/keyman/tree/master/linux/debian) -
+  this is the `debian` subdirectory for Keyman for Linux with the meta data
+  for the Linux package.
+  See [Debian New Maintainers' Guide](https://www.debian.org/doc/manuals/maint-guide/)
+  for details to the various files in the `debian` directory.
 
 ### Flow of a Linux package build
 
-- TeamCity jobs [Keyman_Test](https://build.palaso.org/buildConfiguration/Keyman_Test) or
-  [Keyman_TriggerReleaseBuilds*](https://build.palaso.org/buildConfiguration/Keyman_TriggerReleaseBuildsBeta)
-  trigger a build on [Jenkins](https://jenkins.lsdev.sil.org/view/Keyman/view/Pipeline/job/pipeline-keyman-packaging/)
-- Jenkins verifies the build parameters and starts the matching build configuration for the PR or
-  branch
-- The [build job](https://github.com/sillsdev/lsdev-pipeline-library/blob/master/vars/keymanPackaging.groovy) runs several checks:
-
-  - it exits immediately if the build is not manually triggered and no parameters are passed in
-    (i.e. it got triggered by the GitHub webhook)
-  - it doesn't build if this is a PR, didn't get triggered manually and the PR is not from a trusted
-    user
-  - it doesn't build if no Linux-relevant files changed unless the parameter `force` was passed
-  - manually triggered builds will always build
-
-- build job installs
-  [dependencies](https://github.com/keymanapp/keyman/blob/master/linux/build/agent/install-deps)
-  on the current build agent
-- build job creates a source package for the linux packages (keyman, kmflcomp,
-  libkmfl, and ibus-kmfl). This is done by calling
-  [scripts/jenkins.sh](https://github.com/keymanapp/keyman/blob/master/linux/scripts/jenkins.sh).
-- build job creates the binary package for each linux package on each distribution (currently
-  bionic, focal, and groovy) and each architecture (amd64, i386 only for bionic)
-- at the end of the build if it is not a build of a PR, the `.deb` file gets uploaded to llso
-  (alpha packages to e.g. `bionic-experimental`, beta packages to `bionic-proposed` and
-  packages build from the stable branch to the main section `bionic`)
+- TeamCity jobs [Keyman_Test](https://build.palaso.org/buildConfiguration/Keyman_Test)
+  or [Keyman_TriggerReleaseBuilds*](https://build.palaso.org/buildConfiguration/Keyman_TriggerReleaseBuildsBeta)
+  trigger a packaging GHA build
+- packaging GHA calls [deb-packaging.sh](https://github.com/keymanapp/keyman/blob/master/linux/scripts/deb-packaging.sh)
+  which installs dependencies and creates the source package
+- packaging GHA creates the binary package for each linux package on each
+  distribution
+- packaging GHA verifies that the API didn't change with the help of
+  [deb-packaging.sh](https://github.com/keymanapp/keyman/blob/master/linux/scripts/deb-packaging.sh)
+- at the end of the build if it is not a build of a PR, the `.deb` files get
+  uploaded to llso (alpha packages to e.g. `jammy-experimental`, beta
+  packages to `jammy-proposed` and packages build from the stable branch
+  to the main section `jammy`)
 - if the build is successful the job archives the artifacts
 
-The Jenkins build progress is visible in two ways:
-
-- [traditional view](https://jenkins.lsdev.sil.org/view/Keyman/view/Pipeline/job/pipeline-keyman-packaging/)
-- [blue ocean view](https://jenkins.lsdev.sil.org/blue/organizations/jenkins/pipeline-keyman-packaging/activity)
-
-**Note:** TC release builds pass the git tag to build to the Jenkins job. The same tag
-gets passed twice as parameters `tag` and `tag2`. The first parameter gets persisted between
-builds, allowing to retrigger a tag-build. The second parameter is necessary to distinguish
-if this is a retriggered build of a tag-build.
-
-### Local package builds
-
-It is possible to use the usual Debian/Ubuntu tools to create the package locally. For someone who
-only occasionally deals with packaging it might be easier to use the scripts that Jenkins runs:
-
-#### Prerequisites for local package builds
-
-Install `sbuild` (and probably some other packages that I forgot).
-
-You’ll need a chroot image before you can use sbuild. The scripts in
-[ci-builder-scripts](https://github.com/sillsdev/ci-builder-scripts) will help
-with that. [`setup.sh`](https://github.com/sillsdev/ci-builder-scripts/blob/master/bash/setup.sh)
-can setup such chroots:
-
-```bash
-bash/setup.sh --dists "focal bionic" --arches "amd64 i386"
-```
-
-[`update`](https://github.com/sillsdev/ci-builder-scripts/blob/master/bash/update) is used to
-later update those chroots:
-
-```bash
-bash/update --dists "focal bionic" --arches "amd64 i386"
-```
-
-Set the `DEBSIGNKEY` environment variable to your public GPG key that will be used to sign
-the packages.
-
-#### Building packages
-
-Building packages happen in the [Keyman source tree](https://github.com/keymanapp/keyman).
-
-The Keyman
-[`linux/scripts/jenkins.sh`](https://github.com/keymanapp/keyman/blob/master/linux/scripts/jenkins.sh)
-script can be used to create a source package.
-
-```bash
-cd linux
-./scripts/jenkins.sh keyman ${DEBSIGNKEY}
-```
-
-This creates a source package (`keyman_<version>-1.dsc`) and some `*.tar.?z`
-files in the source root directory for `keyman`.
-
-ci-builder-script's [`build-package`](https://github.com/sillsdev/ci-builder-scripts/blob/master/bash/build-package)
-script creates the binary packages:
-
-```bash
-cd $KEYMAN_ROOT
-~/ci-builder-scripts/bash/build-package \
-    --dists "focal bionic" --arches "amd64 i386" \
-    --debkeyid ${DEBSIGNKEY} --build-in-place --no-upload
-```
-
-This will create the binary package `keyman_<version>-1+<dist>1_<arch>.deb`.
-
-To speed up package building you might want to limit the build to a single dist
-(e.g. `--dists "bionic"`) and arch (e.g. `--arches "amd64"`).
-
-After building packages it might be a good idea to clean up the source tree
-before doing further work:
-
-```bash
-git clean -dxf
-```
-
-### Local package builds (Docker)
+### Local package builds with Docker
 
 It is possible to use the usual Debian/Ubuntu tools to create the package locally.
 For someone who only occasionally deals with packaging it might be easier to use
-the scripts that run on GitHub actions:
+Docker and the scripts that run on GitHub actions:
 
 #### Prerequisites for local package builds with Docker
 
diff --git a/linux/Jenkinsfile b/linux/Jenkinsfile
deleted file mode 100644
index 1607d74d44..0000000000
--- a/linux/Jenkinsfile
+++ /dev/null
@@ -1,11 +0,0 @@
-#!groovy
-// Copyright (c) 2019-2023 SIL International
-// This software is licensed under the MIT license (http://opensource.org/licenses/MIT)
-
-@Library('lsdev-pipeline-library') _
-
-keymanPackaging {
-  distributionsToPackage = 'focal jammy lunar mantic'
-  arches = 'amd64 i386'
-  packagesToBuild = ['keyman']
-}
diff --git a/linux/scripts/dist.sh b/linux/scripts/dist.sh
index f8d63b63e3..d19abf7f78 100755
--- a/linux/scripts/dist.sh
+++ b/linux/scripts/dist.sh
@@ -49,7 +49,7 @@ dpkg-source --tar-ignore=*~ --tar-ignore=.git --tar-ignore=.gitattributes \
     --tar-ignore=core/build \
     --tar-ignore=developer --tar-ignore=docs --tar-ignore=ios \
     --tar-ignore=linux/keyman-config/buildtools/build-langtags.py --tar-ignore=__pycache__ \
-    --tar-ignore=linux/help --tar-ignore=linux/Jenkinsfile \
+    --tar-ignore=linux/help \
     --tar-ignore=mac --tar-ignore=node_modules --tar-ignore=oem \
     --tar-ignore=linux/build \
     --tar-ignore=linux/builddebs \
diff --git a/linux/scripts/jenkins.sh b/linux/scripts/jenkins.sh
deleted file mode 100755
index 5c79625955..0000000000
--- a/linux/scripts/jenkins.sh
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/bin/bash
-# $1 - project name with appended tier, e.g. keyman-alpha
-# $2 - GPG key used for signing the source package
-
-set -e
-set -u
-
-## START STANDARD BUILD SCRIPT INCLUDE
-# adjust relative paths as necessary
-THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")"
-. "${THIS_SCRIPT%/*}/../../resources/build/build-utils.sh"
-## END STANDARD BUILD SCRIPT INCLUDE
-
-. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh"
-
-. "$THIS_SCRIPT_PATH/package-build.inc.sh"
-
-keyman_projects="keyman"
-
-tier="stable"
-
-if [[ "$1" =~ "-alpha" ]]; then
-    tier="alpha"
-elif [[ "$1" =~ "-beta" ]]; then
-    tier="beta"
-fi
-
-proj="$1"
-proj=${proj%"-alpha"}
-proj=${proj%"-beta"}
-
-fullsourcename="keyman"
-sourcedir="$KEYMAN_ROOT"
-sourcename=${fullsourcename%"-alpha"}
-sourcename=${sourcename%"-beta"}
-
-# set Debian/changelog environment
-export DEBFULLNAME="${fullsourcename} Package Signing Key"
-export DEBEMAIL='jenkins@sil.org'
-
-checkAndInstallRequirements
-
-# clean up prev deb builds
-builder_heading "cleaning previous builds of $1"
-
-rm -rf builddebs
-rm -rf "$sourcedir/${1}"_*.{dsc,build,buildinfo,changes,tar.?z,log}
-rm -rf "$sourcedir/../${1}"_*.{dsc,build,buildinfo,changes,tar.?z,log}
-
-builder_heading "Make source package for $fullsourcename"
-builder_heading "reconfigure"
-TIER="$tier" ./scripts/reconf.sh
-
-builder_heading "Make origdist"
-./scripts/dist.sh origdist
-builder_heading "Make deb source"
-./scripts/deb.sh sourcepackage
-
-#sign source package
-for file in builddebs/*.dsc; do
-	builder_heading "Signing source package $file"
-	debsign -k"$2" "$file"
-done
-
-mv builddebs/* ..
diff --git a/resources/build/build-utils.sh b/resources/build/build-utils.sh
index 7ae8db2ae1..34015c9034 100755
--- a/resources/build/build-utils.sh
+++ b/resources/build/build-utils.sh
@@ -74,7 +74,7 @@ function findVersion() {
         VERSION_TAG=
     fi
 
-    if [ -z "${TEAMCITY_VERSION-}" -a -z "${JENKINS_HOME-}" ]; then
+    if [ -z "${TEAMCITY_VERSION-}" ]; then
         # Local dev machine, not TeamCity
         VERSION_TAG="$VERSION_TAG-local"
         VERSION_ENVIRONMENT=local
diff --git a/resources/build/increment-version.sh b/resources/build/increment-version.sh
index d54990194c..445459da16 100755
--- a/resources/build/increment-version.sh
+++ b/resources/build/increment-version.sh
@@ -143,7 +143,7 @@ if [ "$action" == "commit" ]; then
   popd > /dev/null
 
   #
-  # Trigger builds for the previous version on TeamCity, Jenkins and GitHub
+  # Trigger builds for the previous version on TeamCity and GitHub
   #
 
   triggerBuilds
diff --git a/resources/build/run-required-test-builds.sh b/resources/build/run-required-test-builds.sh
index 218383d159..300bd83156 100755
--- a/resources/build/run-required-test-builds.sh
+++ b/resources/build/run-required-test-builds.sh
@@ -54,11 +54,7 @@ function triggerTestBuilds() {
     eval test_builds='(${'bc_test_$platform'[@]})'
     for test_build in "${test_builds[@]}"; do
       if [[ $test_build == "" ]]; then continue; fi
-      if [ "${test_build:(-8)}" == "_Jenkins" ]; then
-        local job=${test_build%_Jenkins}
-        echo "  -- Triggering build configuration $job/$branch on Jenkins"
-        triggerJenkinsBuild "$job" "$branch" "$force"
-      elif [ "${test_build:(-7)}" == "_GitHub" ]; then
+      if [ "${test_build:(-7)}" == "_GitHub" ]; then
         local job=${test_build%_GitHub}
         echo "  -- Triggering GitHub action build $job/$branch"
         triggerGitHubActionsBuild true "$job" "$branch"
diff --git a/resources/build/trigger-builds.inc.sh b/resources/build/trigger-builds.inc.sh
index ac8d519d4a..8d6297b9e1 100644
--- a/resources/build/trigger-builds.inc.sh
+++ b/resources/build/trigger-builds.inc.sh
@@ -17,11 +17,7 @@ function triggerBuilds() {
     eval builds='(${'bc_${bcbase}_${platform}'[@]})'
     for build in "${builds[@]}"; do
       if [[ $build == "" ]]; then continue; fi
-      if [ "${build:(-8)}" == "_Jenkins" ]; then
-        local job=${build%_Jenkins}
-        echo Triggering Jenkins build "$job" "$base" "true"
-        triggerJenkinsBuild "$job" "$base" "true"
-      elif [ "${build:(-7)}" == "_GitHub" ]; then
+      if [ "${build:(-7)}" == "_GitHub" ]; then
         local job=${build%_GitHub}
         echo Triggering GitHub action build "$job" "$base"
         triggerGitHubActionsBuild false "$job" "$base"
@@ -69,62 +65,6 @@ function triggerTeamCityBuild() {
     -d "$command"
 }
 
-function triggerJenkinsBuild() {
-  local JENKINS_JOB="$1"
-  local JENKINS_BRANCH="${2:-master}"
-
-  local JENKINS_SERVER=https://jenkins.lsdev.sil.org
-
-  local FORCE=""
-  if [ "${3:-false}" == "true" ]; then
-    FORCE=", \"force\": true"
-  fi
-
-  local TAG=""
-  # This will only be true if we created and pushed a tag
-  if [ "${action:-""}" == "commit" ]; then
-    TAG=", \"tag\": \"$VERSION_GIT_TAG\", \"tag2\": \"$VERSION_GIT_TAG\""
-  fi
-
-  if [[ $JENKINS_BRANCH != stable-* ]] && [[ $JENKINS_BRANCH =~ [0-9]+ ]]; then
-    JENKINS_BRANCH="PR-${JENKINS_BRANCH}"
-  fi
-
-  local OUTPUT=$(curl --silent --write-out '\n' \
-    -X POST \
-    --header "token: $JENKINS_TOKEN" \
-    --header "Content-Type: application/json" \
-    $JENKINS_SERVER/generic-webhook-trigger/invoke \
-    --data "{ \"project\": \"$JENKINS_JOB/$JENKINS_BRANCH\", \"branch\": \"$JENKINS_BRANCH\" $TAG $FORCE }")
-
-  if echo "$OUTPUT" | grep -q "\"triggered\":true"; then
-    echo -n "     job triggered: "
-  else
-    echo "##teamcity[buildProblem description='Triggering Jenkins build failed']"
-    echo -n "     triggering failed: "
-  fi
-
-  # Strip {"jobs":{ from the beginning of OUTPUT
-  OUTPUT=${OUTPUT#\{\"jobs\":\{}
-  # Split json string to lines with one job each
-  local jobs count
-  count=0
-  IFS='|' jobs=(${OUTPUT//\},\"pipeline/\},|\"pipeline})
-  # Find job that actually got triggered (or that we should have triggered)
-  for line in "${jobs[@]}"; do
-    if [[ $line == \"$JENKINS_JOB/$JENKINS_BRANCH* ]]; then
-      echo "$line"
-      count=$((++count))
-    fi
-  done
-  if [[ $count < 1 ]]; then
-    # DEBUG
-    echo -n $OUTPUT
-
-    echo
-  fi
-}
-
 function triggerGitHubActionsBuild() {
   local IS_TEST_BUILD="$1"
   local GITHUB_ACTION="$2"
diff --git a/resources/build/trigger-definitions.inc.sh b/resources/build/trigger-definitions.inc.sh
index 9370dadd82..4698719d4d 100644
--- a/resources/build/trigger-definitions.inc.sh
+++ b/resources/build/trigger-definitions.inc.sh
@@ -34,9 +34,6 @@ watch_common_linux='common/linux|common/web'
 # These bc_x_y variables ARE used in trigger-builds.inc.sh by pattern so the names are important,
 # and you won't find them directly in a grep search.
 #
-# _Jenkins should be appended to any build configuration (pipeline) name that is from Jenkins,
-# not TeamCity.
-#
 # _GitHub should be appended to any build configuration name that is from GitHub, not TeamCity.
 
 # Test Build Configurations
@@ -46,7 +43,7 @@ bc_test_all=()
 
 bc_test_android=(KeymanAndroid_TestPullRequests KeymanAndroid_TestSamplesAndTestProjects)
 bc_test_ios=(Keyman_iOS_TestPullRequests Keyman_iOS_TestSamplesAndTestProjects)
-bc_test_linux=(KeymanLinux_TestPullRequests Keyman_Linux_Test_Integration Keyman_Common_KPAPI_TestPullRequests_Linux pipeline-keyman-packaging_Jenkins deb-pr-packaging_GitHub)
+bc_test_linux=(KeymanLinux_TestPullRequests Keyman_Linux_Test_Integration Keyman_Common_KPAPI_TestPullRequests_Linux deb-pr-packaging_GitHub)
 bc_test_mac=(Keyman_KeymanMac_PullRequests Keyman_Common_KPAPI_TestPullRequests_macOS)
 bc_test_windows=(KeymanDesktop_TestPullRequests KeymanDesktop_TestPrRenderOnScreenKeyboards Keyman_Common_KPAPI_TestPullRequests_Windows)
 bc_test_web=(Keymanweb_TestPullRequests Keyman_Common_LMLayer_TestPullRequests Keyman_Common_KPAPI_TestPullRequests_WASM)
@@ -66,7 +63,7 @@ vcs_test=HttpsGithubComKeymanappKeymanPRs
 
 bc_master_android=(KeymanAndroid_Build)
 bc_master_ios=(Keyman_iOS_Master)
-bc_master_linux=(KeymanLinux_Master pipeline-keyman-packaging_Jenkins deb-release-packaging_GitHub)
+bc_master_linux=(KeymanLinux_Master deb-release-packaging_GitHub)
 bc_master_mac=(KeymanMac_Master)
 bc_master_windows=(Keyman_Build)
 bc_master_web=(Keymanweb_Build)
@@ -78,7 +75,7 @@ vcs_master=HttpsGithubComKeymanappKeyman
 
 bc_beta_android=(KeymanAndroid_Build)
 bc_beta_ios=(Keyman_iOS_Master)
-bc_beta_linux=(KeymanLinux_Master pipeline-keyman-packaging_Jenkins deb-release-packaging_GitHub)
+bc_beta_linux=(KeymanLinux_Master deb-release-packaging_GitHub)
 bc_beta_mac=(KeymanMac_Master)
 bc_beta_windows=(Keyman_Build)
 bc_beta_web=(Keymanweb_Build)
@@ -90,7 +87,7 @@ vcs_beta=HttpsGithubComKeymanappKeyman
 
 bc_stable_14_0_android=(KeymanAndroid_Build)
 bc_stable_14_0_ios=(Keyman_iOS_Master)
-bc_stable_14_0_linux=(KeymanLinux_Master pipeline-keyman-packaging_Jenkins deb-release-packaging_GitHub)
+bc_stable_14_0_linux=(KeymanLinux_Master deb-release-packaging_GitHub)
 bc_stable_14_0_mac=(KeymanMac_Master)
 bc_stable_14_0_windows=(Keyman_Build)
 bc_stable_14_0_web=(Keymanweb_Build)
@@ -105,7 +102,7 @@ vcs_stable_14_0=HttpsGithubComKeymanappKeyman
 
 bc_stable_15_0_android=(KeymanAndroid_Build)
 bc_stable_15_0_ios=(Keyman_iOS_Master)
-bc_stable_15_0_linux=(KeymanLinux_Master pipeline-keyman-packaging_Jenkins deb-release-packaging_GitHub)
+bc_stable_15_0_linux=(KeymanLinux_Master deb-release-packaging_GitHub)
 bc_stable_15_0_mac=(KeymanMac_Master)
 bc_stable_15_0_windows=(Keyman_Build)
 bc_stable_15_0_web=(Keymanweb_Build)
@@ -116,7 +113,7 @@ vcs_stable_15_0=HttpsGithubComKeymanappKeyman
 
 bc_stable_16_0_android=(KeymanAndroid_Build)
 bc_stable_16_0_ios=(Keyman_iOS_Master)
-bc_stable_16_0_linux=(KeymanLinux_Master pipeline-keyman-packaging_Jenkins)
+bc_stable_16_0_linux=(KeymanLinux_Master)
 bc_stable_16_0_mac=(KeymanMac_Master)
 bc_stable_16_0_windows=(Keyman_Build)
 bc_stable_16_0_web=(Keymanweb_Build)

From ecf35b399c96fff8caaa4f0d33be3a7b38ec2d53 Mon Sep 17 00:00:00 2001
From: Darcy Wong <darcy_wong@sil.org>
Date: Tue, 1 Aug 2023 20:36:38 +0700
Subject: [PATCH 63/83] docs(windows): Update OS requirement to Windows 10

---
 .../src/desktop/help/about/requirements.md    |  9 ++-----
 .../help/basic/make-taskbar-icon-visible.md   | 27 -------------------
 windows/src/desktop/help/basic/uninstall.md   | 15 -----------
 windows/src/desktop/help/common/os.md         | 12 +--------
 .../src/desktop/help/common/requirements.md   |  2 +-
 5 files changed, 4 insertions(+), 61 deletions(-)

diff --git a/windows/src/desktop/help/about/requirements.md b/windows/src/desktop/help/about/requirements.md
index b9dd275555..2a82eb89de 100644
--- a/windows/src/desktop/help/about/requirements.md
+++ b/windows/src/desktop/help/about/requirements.md
@@ -4,20 +4,15 @@ title: System Requirements
 
 ## Supported Windows Operating Systems
 
-Keyman fully supports 32-bit *and* 64-bit versions of the following
+Keyman fully supports 64-bit versions of the following
 Windows operating systems:
 
--   Windows 7
--   Windows 8
--   Windows 8.1
 -   Windows 10
 -   Windows 11
--   Windows Server 2008 and 2008 R2
--   Windows Server 2012 and 2012 R2
 
 **Note:** Keyman works slightly differently in different versions of Windows. Older versions of Windows have more language limitations and need extra configuration.
 
 ## Resource Requirements
 
 Keyman has minimal resource requirements. Any computer that can run
-Windows 7 should be able to run Keyman without trouble.
+Windows 10 should be able to run Keyman without trouble.
diff --git a/windows/src/desktop/help/basic/make-taskbar-icon-visible.md b/windows/src/desktop/help/basic/make-taskbar-icon-visible.md
index 9c5bbd66a4..62b90be5d4 100644
--- a/windows/src/desktop/help/basic/make-taskbar-icon-visible.md
+++ b/windows/src/desktop/help/basic/make-taskbar-icon-visible.md
@@ -37,30 +37,3 @@ Here's how to make the change with Windows Settings:
     5.  The Keyman icon will now always appear in the Windows
         Taskbar near the clock, if Keyman is on.
     6.  Continue on to [Step 5](../start/tutorial#step-5-) of this guide.
-
--   On Windows 8:
-
-    1.  Right-click on the Windows Taskbar.
-        ![](../desktop_images/win8-taskbar1.png)
-    2.  Select 'Properties'.
-    3.  Next to 'Notification area', click Customize….
-        ![](../desktop_images/win8-taskbar2.png)
-    4.  From the dropdown menu beside 'Keyman
-        Engine x86', select 'Show icon and notifications'.
-        ![](../desktop_images/win8-taskbar3.png)
-    5.  Click OK to apply changes. The Keyman icon will now always
-        appear in the Windows Taskbar near the clock, if Keyman is
-        on.
-    6.  Continue on to [Step 5](../start/tutorial#step-5-) of this guide.
-
--   On Windows 7:
-
-    1.  Open the Windows Start menu.
-    2.  In the search field, type and enter: Notification Area Icons
-    3.  From the dropdown menu beside Keyman
-        Engine x86, select 'Show icon and notifications'.
-        ![](../desktop_images/7-taskbar.png)
-    4.  Click OK to apply changes. The Keyman icon will now always
-        appear in the Windows Taskbar near the clock, if Keyman is
-        on.
-    5.  Continue on to [Step 5](../start/tutorial#step-5-) of this guide.
diff --git a/windows/src/desktop/help/basic/uninstall.md b/windows/src/desktop/help/basic/uninstall.md
index d9952a4279..7e7cb2274c 100644
--- a/windows/src/desktop/help/basic/uninstall.md
+++ b/windows/src/desktop/help/basic/uninstall.md
@@ -15,18 +15,3 @@ title: Software Task - Uninstall Keyman
 5.  Click Uninstall.
 
 6.  Follow the prompts to complete the uninstall.
-
-## Uninstall Keyman from Windows 7 or 8
-
-1.  Exit Keyman.
-
-2.  Open Windows Control Panel.
-
-3.  Select \'Add or Remove Programs\' or \'Programs and Features\' or
-    \'Uninstall a program\'.
-
-4.  Click Keyman in the list.
-
-5.  Click Remove or Uninstall or right-click \'Uninstall\'.
-
-6.  Follow the prompts to complete the uninstall.
diff --git a/windows/src/desktop/help/common/os.md b/windows/src/desktop/help/common/os.md
index ca1ead7b27..40f47723b7 100644
--- a/windows/src/desktop/help/common/os.md
+++ b/windows/src/desktop/help/common/os.md
@@ -2,23 +2,13 @@
 title: What operating systems does Keyman support?
 ---
 
-Keyman fully supports 32-bit *and* 64-bit versions of the following
+Keyman fully supports 64-bit versions of the following
 Windows operating systems:
 
--   Windows Server 2008
-
--   Windows 7
-
--   Windows 8
-
--   Windows 8.1
-
 -   Windows 10
 
 -   Windows 11
 
--   Windows Server 2012 and 2012 R2
-
 **Note:**
 Keyman works slightly differently in different versions of Windows.
 Older versions of Windows have more language limitations and need extra
diff --git a/windows/src/desktop/help/common/requirements.md b/windows/src/desktop/help/common/requirements.md
index cdd3d81c23..11fefc6dfd 100644
--- a/windows/src/desktop/help/common/requirements.md
+++ b/windows/src/desktop/help/common/requirements.md
@@ -3,4 +3,4 @@ title: What are Keyman's hardware requirements?
 ---
 
 Keyman has minimal resource requirements. Any computer that can run
-Windows 7 or later should be able to run Keyman without trouble.
+Windows 10 or later should be able to run Keyman without trouble.

From f154c493e3b070b2c9242042334e1af69f1e0344 Mon Sep 17 00:00:00 2001
From: Eberhard Beilharz <eb1@sil.org>
Date: Mon, 31 Jul 2023 19:30:47 +0200
Subject: [PATCH 64/83] docs(linux): Add build doc for Keyman Web and Android

This adds the documentation how to build Keyman Web as well as
Keyman for Android on Linux. and adds the dependencies to the
`Dockerfile` so that it's now also possible to build Keyman Web
and Keyman for Android with Docker.
---
 docs/build/linux-ubuntu.md | 201 +++++++++++++++++++++++++------------
 docs/build/windows.md      |   1 +
 linux/Dockerfile           |  52 ++++++++--
 3 files changed, 181 insertions(+), 73 deletions(-)

diff --git a/docs/build/linux-ubuntu.md b/docs/build/linux-ubuntu.md
index f8c29dc0fe..5240d13e20 100644
--- a/docs/build/linux-ubuntu.md
+++ b/docs/build/linux-ubuntu.md
@@ -4,14 +4,13 @@
 
 On Linux, you can build the following projects:
 
+- [Keyman Core](#keyman-core) (aka core)
 - [Keyman for Linux](#keyman-for-linux)
-- [Keyman Core](#keyman-core) (Linux only) (aka core)
+- [Keyman Web](#keyman-web)
 - [Keyman for Android](#keyman-for-android)
-<!-- TODO: document how to build for Web, Core-Wasm and Common/Web on Linux.
+<!-- TODO: document how to build for Common/Web on Linux.
      See TC build agent for details. -->
-- Keyman Core (wasm targets)
 - Common/Web
-- KeymanWeb
 
 The following projects **cannot** be built on Linux:
 
@@ -20,13 +19,15 @@ The following projects **cannot** be built on Linux:
 - Keyman for macOS
 - Keyman for iOS
 
-## System Requirements
+## Requirements
+
+### System Requirements
 
 - Minimum Ubuntu version: Ubuntu 20.04
 
 Other Linux distributions will also work if appropriate dependencies are installed.
 
-## Repository Paths
+### Repository Paths
 
 Recommended filesystem layout:
 
@@ -39,7 +40,7 @@ $HOME/keyman/
     ...
 ```
 
-## Prerequisites
+### Prerequisites
 
 The current list of dependencies can be found in the `Build-Depends` section of `linux/debian/control`.
 They are most easily installed with the `mk-build-deps` tool:
@@ -50,7 +51,7 @@ sudo apt install devscripts equivs
 sudo mk-build-deps --install linux/debian/control
 ```
 
-### Node.js
+#### Node.js
 
 Node.js v18 is required for Core build, Web tests, and Developer command line tools.
 
@@ -61,19 +62,9 @@ curl -sL https://deb.nodesource.com/setup_18.x | bash
 apt-get -q -y install nodejs
 ```
 
-## Keyman for Linux
+#### Emscripten
 
-All dependencies are already installed if you followed the instructions
-under [Prerequisites](#prerequisites).
-
-### Building Keyman for Linux
-
-- [Building Keyman for Linux](../../linux/README.md)
-
-## Keyman Core
-
-Most dependencies are already installed if you followed the instructions under
-[Prerequisites](#prerequisites). You'll still have to install `emscripten`:
+You'll also have to install `emscripten` (version 3.1.44 is known to work):
 
 ```shell
 git clone https://github.com/emscripten-core/emsdk.git
@@ -85,65 +76,97 @@ export EMSCRIPTEN_BASE=$(pwd)/upstream/emscripten
 
 **NOTE:** Don't put EMSDK on the path, i.e. don't source `emsdk_env.sh`.
 
+## Keyman Core
+
+All dependencies are already installed if you followed the instructions under
+[Prerequisites](#prerequisites).
+
 ### Building Keyman Core
 
+Keyman Core can be built with the `core/build.sh` script.
+
 - [Building Keyman Core](../../core/doc/BUILDING.md)
 
-## Docker Builder
+## Keyman for Linux
 
-The Docker builder allows you to perform a linux build from anywhere Docker is supported.
-To build the docker image:
+All dependencies are already installed if you followed the instructions
+under [Prerequisites](#prerequisites).
 
-```shell
-cd linux
-docker pull ubuntu:latest
-docker build . -t keymanapp/keyman-linux-builder:latest
+### Building Keyman for Linux
+
+Keyman for Linux can be built with the `linux/build.sh` script.
+
+- [Building Keyman for Linux](../../linux/README.md)
+
+## Keyman Web
+
+Most dependencies are already installed if you followed the instructions under
+[Prerequisites](#prerequisites). You'll still have to install Chrome:
+
+```bash
+wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
+sudo apt install ./google-chrome-stable_current_amd64.deb
 ```
 
-Once the image is built, it may be used to build parts of Keyman.
+And add the `CHROME_BIN` environment variable to `.bashrc:
 
-- core
-
-```shell
-# build 'core' in docker
-cd $(git rev-parse --show-toplevel)/core
-# keep linux build artifacts separate
-mkdir -p build/linux
-docker run -it --rm -v $(pwd)/..:/home/build/build \
-  -v $(pwd)/build/linux:/home/build/build/core/build \
-  keymanapp/keyman-linux-builder:latest \
-  wrapper core/build.sh --debug
+```bash
+export CHROME_BIN=/opt/google/chrome/chrome
 ```
 
-- linux
+### Environment variables for Keyman Web
 
-```shell
-# build 'linux' installation in docker
-cd $(git rev-parse --show-toplevel)
-docker run -it --rm -v $(pwd):/home/build/build \
-  keymanapp/keyman-linux-builder:latest \
-  wrapper 'DESTDIR=/home/build linux/build.sh --debug build install'
-```
+`CHROME_BIN` pointing to the Google Chrome binary.
+
+### Building Keyman Web
+
+Keyman Web can be built with the `web/build.sh` script.
+
+- [Building Keyman Web](../../web/README.md)
 
 ## Keyman for Android
 
 **Dependencies:**
 
-- [Base](#base-dependencies)
-- [Web](./windows#web-dependencies)
+Most dependencies are already installed if you followed the instructions
+under [Prerequisites](#prerequisites).
 
 **Additional requirements:**
 
-- Android SDK
 - [Android Studio](https://developer.android.com/studio/install#linux)
-- Gradle
+  or sdkmanager
 - Maven
-- OpenJDK 11 (for Keyman 17.0+)
 - pandoc
+- Android SDK
+- Gradle
+- jq
+
+If you only use the command line you don't need Android Studio, however
+to do development it's recommended to install it.
 
 Run Android Studio once after installation to install additional components
 such as emulator images and SDK updates.
 
+Maven, jq and pandoc can be installed with:
+
+```shell
+sudo apt update
+sudo apt install maven pandoc jq
+```
+
+If necessary, Android SDK and Gradle will be installed by the build script.
+In order for that to work, run the following command once. You won't need
+this if you install Android SDK through Android Studio.
+
+```shell
+sudo apt install sdkmanager
+sudo sdkmanager platform-tools
+sudo chown -R $USER:$USER /opt/android-sdk/
+sdkmanager --licenses
+```
+
+### Environment variables for Keyman for Android
+
 **Required environment variable:**
 
 - `ANDROID_HOME` pointing to Android SDK (`$HOME/Android/Sdk`)
@@ -152,23 +175,15 @@ such as emulator images and SDK updates.
 
 - [`JAVA_HOME`](#java_home)
 
-Building:
+### Building Keyman for Android
+
+Keyman for Android can be built with the `android/build.sh` script.
 
 - [Building Keyman for Android](../../android/README.md)
 
-## Prerequisites
+### Notes on Environment Variables
 
-Many dependencies are only required for specific projects.
-
-### Base Dependencies
-
-**Environment variables:**
-
-- --
-
-## Notes on Environment Variables
-
-### JAVA_HOME
+#### JAVA_HOME
 
 This environment variable tells Gradle what version of Java to use for building
 Keyman for Android. OpenJDK 11 is used for master.
@@ -191,3 +206,59 @@ older versions, you can set `JAVA_HOME_11` to the OpenJDK 11 path and
 from command line. But note that you do need to update your `JAVA_HOME` env
 var to the associated version before opening Android Studio and loading any
 Android projects. `JAVA_HOME_11` is mostly used by CI.
+
+## Docker Builder
+
+The Docker builder allows you to perform a build from anywhere Docker is supported.
+
+To build the docker image:
+
+```shell
+cd linux
+docker pull ubuntu:latest
+docker build . -t keymanapp/keyman-linux-builder:latest
+```
+
+Once the image is built, it may be used to build parts of Keyman.
+
+**Note** that it's not yet possible to run tests in the Docker container.
+
+- core
+
+  ```shell
+  # build 'Keyman Core' in docker
+  # keep linux build artifacts separate
+  mkdir -p $(git rev-parse --show-toplevel)/core/build/linux
+  docker run -it --rm -v $(git rev-parse --show-toplevel):/home/build/build \
+    -v $(git rev-parse --show-toplevel)/core/build/linux:/home/build/build/core/build \
+    keymanapp/keyman-linux-builder:latest \
+    core/build.sh --debug
+  ```
+
+- linux
+
+  ```shell
+  # build 'Keyman for Linux' installation in docker
+  docker run -it --rm -v $(git rev-parse --show-toplevel):/home/build/build \
+    --entrypoint /bin/bash keymanapp/keyman-linux-builder:latest \
+    -c 'DESTDIR=/home/build /usr/bin/bashwrapper linux/build.sh --debug build install'
+  ```
+
+- Keyman Web
+
+  ```shell
+  # build 'Keyman Web' in docker
+  docker run --privileged -it --rm \
+    -v $(git rev-parse --show-toplevel):/home/build/build \
+    keymanapp/keyman-linux-builder:latest \
+    web/build.sh --debug
+  ```
+
+- Keyman for Android
+
+  ```shell
+  # build 'Keyman for Android' in docker
+  docker run -it --rm -v $(git rev-parse --show-toplevel):/home/build/build \
+    keymanapp/keyman-linux-builder:latest \
+    android/build.sh --debug
+  ```
diff --git a/docs/build/windows.md b/docs/build/windows.md
index fd66445b81..9faebc3d63 100644
--- a/docs/build/windows.md
+++ b/docs/build/windows.md
@@ -153,6 +153,7 @@ choco install git jq python ninja pandoc
 refreshenv
 # choco meson (0.55) is too old, 1.0 required:
 python -m pip install meson
+```
 
 **Environment variables**:
 * [`KEYMAN_ROOT`](#keyman_root)
diff --git a/linux/Dockerfile b/linux/Dockerfile
index 9e8339440b..6852b54238 100644
--- a/linux/Dockerfile
+++ b/linux/Dockerfile
@@ -11,7 +11,7 @@ LABEL org.opencontainers.image.title="Keyman Linux Build Image"
 # We will switch to a build user after some installation
 USER root
 ENV HOME /home/build
-RUN useradd -c "Build user" --home-dir $HOME --create-home --shell /usr/bin/bash build
+RUN useradd -c "Build user" --home-dir $HOME --create-home --shell /usr/bin/bashwrapper build
 VOLUME /home/build/build
 WORKDIR /home/build/build
 ENV DEBIAN_FRONTEND noninteractive
@@ -29,11 +29,12 @@ RUN apt-get -q -y update &&  \
 # Install dependencies
 ADD debian/control /tmp/control
 # Answer 'yes' to install questions
-RUN (yes | mk-build-deps --install /tmp/control) || true
+RUN (yes | mk-build-deps --install /tmp/control) || true && \
+  rm /tmp/control
 
 # Install Node
-RUN curl -sL https://deb.nodesource.com/setup_18.x | bash
-RUN apt-get -q -y install nodejs
+RUN curl -sL https://deb.nodesource.com/setup_18.x | bash && \
+  apt-get -q -y install nodejs
 
 # Install emscripten
 RUN cd /usr/share && \
@@ -41,10 +42,45 @@ RUN cd /usr/share && \
   cd emsdk && \
   ./emsdk install latest && \
   ./emsdk activate latest && \
-  echo "#!/bin/bash" > /usr/bin/wrapper && \
-  echo "export EMSCRIPTEN_BASE=/usr/share/emsdk/upstream/emscripten" >> /usr/bin/wrapper && \
-  echo "bash -c \"\${@:-bash}\"">> /usr/bin/wrapper && \
-  chmod +x /usr/bin/wrapper
+  echo "#!/bin/bash" > /usr/bin/bashwrapper && \
+  echo "export EMSCRIPTEN_BASE=/usr/share/emsdk/upstream/emscripten" >> /usr/bin/bashwrapper
+
+# Keyman Web
+RUN curl --output google-chrome-stable_current_amd64.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \
+  apt-get -q -y install ./google-chrome-stable_current_amd64.deb && \
+  rm google-chrome-stable_current_amd64.deb && \
+  echo "export CHROME_BIN=/opt/google/chrome/chrome" >> /usr/bin/bashwrapper
+
+# Keyman for Android
+RUN apt-get -q -y install gradle maven pandoc sdkmanager jq && \
+  sdkmanager platform-tools && \
+  yes | sdkmanager --licenses && \
+  chown -R build:build /opt/android-sdk/ && \
+  echo "export ANDROID_HOME=/opt/android-sdk" >> /usr/bin/bashwrapper && \
+  echo "export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64" >> /usr/bin/bashwrapper
+
+# Finish bashwrapper script and adjust permissions
+RUN echo "\${@:-bash}" >> /usr/bin/bashwrapper && \
+  chmod +x /usr/bin/bashwrapper && \
+  chown -R build:build $HOME
 
 # now, switch to build user
 USER build
+
+# Pre-install gradle. This will put files in ~/.gradle which will speed up builds.
+RUN mkdir -p $HOME/tmp/gradle/wrapper && \
+  # KMEA uses gradle-7.5.1-bin
+  curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.jar https://raw.githubusercontent.com/keymanapp/keyman/master/android/KMEA/gradle/wrapper/gradle-wrapper.jar && \
+  curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.properties https://raw.githubusercontent.com/keymanapp/keyman/master/android/KMEA/gradle/wrapper/gradle-wrapper.properties && \
+  curl --location --output $HOME/tmp/gradlew https://raw.githubusercontent.com/keymanapp/keyman/master/android/KMEA/gradlew && \
+  chmod +x $HOME/tmp/gradlew && \
+  $HOME/tmp/gradlew --quiet && \
+  # Some projects use gradle-7.5.1-all, so we pre-install that as well
+  curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.jar https://raw.githubusercontent.com/keymanapp/keyman/master/android/Samples/KMSample1/gradle/wrapper/gradle-wrapper.jar && \
+  curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.properties https://raw.githubusercontent.com/keymanapp/keyman/master/android/Samples/KMSample1/gradle/wrapper/gradle-wrapper.properties && \
+  curl --location --output $HOME/tmp/gradlew https://raw.githubusercontent.com/keymanapp/keyman/master/android/Samples/KMSample1/gradlew && \
+  chmod +x $HOME/tmp/gradlew && \
+  $HOME/tmp/gradlew --quiet && \
+  rm -rf $HOME/tmp
+
+ENTRYPOINT [ "/usr/bin/bashwrapper" ]

From 0ac8b2a5c6c666d6ee38628e2619b0ffb0ab0c58 Mon Sep 17 00:00:00 2001
From: Keyman Build Agent <buildagent@keyman.com>
Date: Tue, 1 Aug 2023 14:04:11 -0400
Subject: [PATCH 65/83] auto: increment master version to 17.0.152

---
 HISTORY.md | 8 ++++++++
 VERSION.md | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/HISTORY.md b/HISTORY.md
index 04f84fef16..9140446de9 100644
--- a/HISTORY.md
+++ b/HISTORY.md
@@ -1,5 +1,13 @@
 # Keyman Version History
 
+## 17.0.151 alpha 2023-08-01
+
+* feat(developer) marker steps (#9364)
+* feat(common): marker processing (#9365)
+* chore(linux): Don't fail on parallel builds (#9368)
+* fix(developer): fix breakage from emscripten 3.1.44 (#9375)
+* docs(core): Document how to build Core on Linux (#9328)
+
 ## 17.0.150 alpha 2023-07-31
 
 * chore(linux): Update debian changelog (#9358)
diff --git a/VERSION.md b/VERSION.md
index d1b3623053..aaa2187da5 100644
--- a/VERSION.md
+++ b/VERSION.md
@@ -1 +1 @@
-17.0.151
\ No newline at end of file
+17.0.152
\ No newline at end of file

From 669198add6ee81f563a4b11ba84d5f27db701161 Mon Sep 17 00:00:00 2001
From: Marc Durdin <marc@durdin.net>
Date: Tue, 1 Aug 2023 16:37:29 +0700
Subject: [PATCH 66/83] chore(linux): touch linux so we get a build

---
 linux/README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/linux/README.md b/linux/README.md
index 3dfb340477..e4b50cdc90 100644
--- a/linux/README.md
+++ b/linux/README.md
@@ -1,3 +1,5 @@
 # Keyman for Linux
 
 See [/docs/linux/README.md](../docs/linux/README.md) for documentation.
+
+.
\ No newline at end of file

From a010507a3fc9c69a0fc59aee7a07b5be60f3b1fb Mon Sep 17 00:00:00 2001
From: Marc Durdin <marc@durdin.net>
Date: Tue, 1 Aug 2023 16:00:12 +0700
Subject: [PATCH 67/83] chore: add run-name to deb-packaging

---
 .github/workflows/deb-packaging.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/deb-packaging.yml b/.github/workflows/deb-packaging.yml
index ddbdc66dc0..ee0b8a5bd8 100644
--- a/.github/workflows/deb-packaging.yml
+++ b/.github/workflows/deb-packaging.yml
@@ -1,4 +1,5 @@
 name: "Ubuntu packaging"
+run-name: "Ubuntu packaging - ${{ github.ref_name }} by @${{ github.actor }}"
 on:
   repository_dispatch:
     types: ['deb-release-packaging:*', 'deb-pr-packaging:*']

From 1566c14881fad234d9479ed9290a317b1208c41a Mon Sep 17 00:00:00 2001
From: Marc Durdin <marc@durdin.net>
Date: Wed, 2 Aug 2023 09:57:42 +0700
Subject: [PATCH 68/83] chore: try another variable for reporting

---
 .github/workflows/deb-packaging.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/deb-packaging.yml b/.github/workflows/deb-packaging.yml
index ee0b8a5bd8..a923c1ab69 100644
--- a/.github/workflows/deb-packaging.yml
+++ b/.github/workflows/deb-packaging.yml
@@ -1,5 +1,5 @@
 name: "Ubuntu packaging"
-run-name: "Ubuntu packaging - ${{ github.ref_name }} by @${{ github.actor }}"
+run-name: "Ubuntu packaging - ${{ github.event.client_payload.branch }} (branch ${{ github.head_ref }}), by @${{ github.actor }}"
 on:
   repository_dispatch:
     types: ['deb-release-packaging:*', 'deb-pr-packaging:*']

From bf5b5f6e2296bb40904d92c2ae91853bf36373eb Mon Sep 17 00:00:00 2001
From: Darcy Wong <darcy_wong@sil.org>
Date: Wed, 2 Aug 2023 11:06:38 +0700
Subject: [PATCH 69/83] docs(windows): Clarify OS versions per review comments

---
 windows/src/desktop/help/about/requirements.md |  9 ++++++---
 windows/src/desktop/help/common/os.md          | 14 ++++++++------
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/windows/src/desktop/help/about/requirements.md b/windows/src/desktop/help/about/requirements.md
index 2a82eb89de..f83cda2686 100644
--- a/windows/src/desktop/help/about/requirements.md
+++ b/windows/src/desktop/help/about/requirements.md
@@ -4,13 +4,16 @@ title: System Requirements
 
 ## Supported Windows Operating Systems
 
-Keyman fully supports 64-bit versions of the following
+Keyman fully supports versions of the following
 Windows operating systems:
 
--   Windows 10
+-   Windows 10 (32-bit *and* 64-bit)
 -   Windows 11
+-   Windows Server 2019 and 2022
 
-**Note:** Keyman works slightly differently in different versions of Windows. Older versions of Windows have more language limitations and need extra configuration.
+**Note:** Keyman doesn't work on Windows 11 for ARM64. 
+Keyman works slightly differently in different versions of Windows. 
+Versions prior to Windows 10 have more language limitations and need extra configuration.
 
 ## Resource Requirements
 
diff --git a/windows/src/desktop/help/common/os.md b/windows/src/desktop/help/common/os.md
index 40f47723b7..f64c565567 100644
--- a/windows/src/desktop/help/common/os.md
+++ b/windows/src/desktop/help/common/os.md
@@ -2,16 +2,18 @@
 title: What operating systems does Keyman support?
 ---
 
-Keyman fully supports 64-bit versions of the following
+Keyman fully supports versions of the following
 Windows operating systems:
 
--   Windows 10
+-   Windows 10 (32-bit *and* 64-bit)
 
 -   Windows 11
 
-**Note:**
-Keyman works slightly differently in different versions of Windows.
-Older versions of Windows have more language limitations and need extra
-configuration. Windows Vista and later versions of Windows include added
+-   Windows Server 2019 and 2022
+
+**Note:** Keyman doesn't work on Windows 11 for ARM64. 
+Keyman works slightly differently in different versions of Windows. 
+Versions prior to Windows 10 have more language limitations and need extra configuration. 
+Windows 10 and later versions of Windows include added
 security measures and some configuration actions will require you to
 confirm security prompts.

From ee12ad4d57f88738803d3d658736e7644b17822b Mon Sep 17 00:00:00 2001
From: Darcy Wong <darcy_wong@sil.org>
Date: Wed, 2 Aug 2023 11:09:19 +0700
Subject: [PATCH 70/83] docs(windows): Remove troubleshooting hidden page

Page was only applicable to Win 7
---
 .../src/desktop/help/basic/select-keyboard.md |  1 -
 .../desktop/help/troubleshooting/hidden.md    | 20 -------------------
 .../src/desktop/help/troubleshooting/index.md |  2 --
 3 files changed, 23 deletions(-)
 delete mode 100644 windows/src/desktop/help/troubleshooting/hidden.md

diff --git a/windows/src/desktop/help/basic/select-keyboard.md b/windows/src/desktop/help/basic/select-keyboard.md
index 203866d545..13e6bbc016 100644
--- a/windows/src/desktop/help/basic/select-keyboard.md
+++ b/windows/src/desktop/help/basic/select-keyboard.md
@@ -120,4 +120,3 @@ reach the desired keyboard, then release <kbd>spacebar</kbd>.
 -   [Keyman Configuration - Keyboard Layouts Tab](config/keyboards)
 -   [How To - Download and Install a Keyman Keyboard](../start/download-and-install-keyboard)
 -   [Keyboard Task - Enable or Disable a Keyboard](enable-or-disable-keyboard)
--   [How To - Fix A Problem with an Active Keyman Keyboard Not Typing](../troubleshooting/hidden)
diff --git a/windows/src/desktop/help/troubleshooting/hidden.md b/windows/src/desktop/help/troubleshooting/hidden.md
deleted file mode 100644
index 11d35a151f..0000000000
--- a/windows/src/desktop/help/troubleshooting/hidden.md
+++ /dev/null
@@ -1,20 +0,0 @@
----
-title: How To - Fix A Problem with an Active Keyman Keyboard Not Typing
----
-
-## Symptoms
-You may experience an issue in Windows 7 where:
-* Keyman is on; and
-* your Keyman keyboard is active, your Keyman keyboard is active, with its icon showing in the Keyman menu; but
-* your Keyman keyboard does not work.
-
-## Resolution
-Unhide the Keyman menu as described in [Step 2](../start/tutorial#step-2-) of the Getting Started tutorial.
-
-This can also be resolved by switching language using the Windows Language Bar or the Language Icon
-
-## Background
-When the Keyman icon is in the hidden notification area of the Windows taskbar, opening the notification area and selecting the icon causes it to change the keyboard for the taskbar, not for your active application. Clicking back to the active application results in the keyboard turning off again.
-
-## Related Topics
--   [Keyboard Task - Turn on a Keyboard](../basic/select-keyboard)
diff --git a/windows/src/desktop/help/troubleshooting/index.md b/windows/src/desktop/help/troubleshooting/index.md
index 03e9745fec..54416ad9ba 100644
--- a/windows/src/desktop/help/troubleshooting/index.md
+++ b/windows/src/desktop/help/troubleshooting/index.md
@@ -10,6 +10,4 @@ title: Troubleshooting
 * [How To - Resolve Security Software Conflicts with keyman32.dll](securitysoftware)
 * [How To - Use the Keyman Setup Bootstrapper](bootstrapper)
 
-* [How To - Fix A Problem with an Active Keyman Keyboard Not Typing](hidden)
-
 * [How To - Allow Windows 10 to Install Apps From Anywhere](install-app-from-anywhere)

From 08f33bbb179b465bfb2a159373a328b5b9dda230 Mon Sep 17 00:00:00 2001
From: Darcy Wong <darcy_wong@sil.org>
Date: Wed, 2 Aug 2023 11:09:41 +0700
Subject: [PATCH 71/83] docs(windows): Remove other references to Win 7

---
 .../desktop/help/advanced/text_services_framework.md  |  8 --------
 windows/src/desktop/help/basic/config/options.md      | 11 ++++-------
 2 files changed, 4 insertions(+), 15 deletions(-)

diff --git a/windows/src/desktop/help/advanced/text_services_framework.md b/windows/src/desktop/help/advanced/text_services_framework.md
index 7b3ed4f994..e7a977a062 100644
--- a/windows/src/desktop/help/advanced/text_services_framework.md
+++ b/windows/src/desktop/help/advanced/text_services_framework.md
@@ -10,14 +10,6 @@ supports features such as keyboard drivers, handwriting recognition,
 speech recognition, as well as spell checking and other text processing
 functions.
 
-In Windows 7, the Language Bar is the core user interface for TSF. From
-the Language Bar, you can select the input language, and control
-keyboard input, handwriting recognition and speech recognition. In
-Windows 8, the Language Bar is no longer used, and the interface is
-tightly integrated into the taskbar.
-
-![](../desktop_images/language-bar.png)
-
 With Keyman Desktop 9 and later versions, all keyboards are registered
 through the Windows interfaces, and the key advantage is that Keyman now
 automatically detects applications that have support for TSF and
diff --git a/windows/src/desktop/help/basic/config/options.md b/windows/src/desktop/help/basic/config/options.md
index e19d51c24d..201c825961 100644
--- a/windows/src/desktop/help/basic/config/options.md
+++ b/windows/src/desktop/help/basic/config/options.md
@@ -60,16 +60,13 @@ To open the Options tab of Keyman Configuration:
     Click the Reset Hints button to switch all hint messages on again,
     even those that you have switched off on a case-by-case basis.
 
--   Select keyboard layout for all applications (Windows 7)
+-   Select keyboard layout for all applications
 
-    Unlike Windows 7 default behaviour, Keyman allows you to select one
+    Keyman allows you to select one
     Windows language and Keyman keyboard for all open applications and
-    text fields across your entire system. Tick this option to select
-    one Windows language and Keyman keyboard across your entire system.
-    Untick this option to select Windows language and Keyman keyboards
-    independently for different programs.
+    text fields across your entire system.
 
-    On Windows 8, 8.1, 10 and later versions, this checkbox is disabled
+    On Windows 10 and later versions, this checkbox is disabled
     because Windows has this functionality built in. The setting can be
     changed in Windows, with the following steps:
 

From 4ea880bc2790f0cdfb3a56bdcf3ea73ffd30ab89 Mon Sep 17 00:00:00 2001
From: Darcy Wong <darcy_wong@sil.org>
Date: Wed, 2 Aug 2023 12:49:15 +0700
Subject: [PATCH 72/83] Apply suggestions from code review

Co-authored-by: Marc Durdin <marc@durdin.net>
---
 windows/src/desktop/help/about/requirements.md |  7 +++----
 windows/src/desktop/help/common/os.md          | 10 +++-------
 2 files changed, 6 insertions(+), 11 deletions(-)

diff --git a/windows/src/desktop/help/about/requirements.md b/windows/src/desktop/help/about/requirements.md
index f83cda2686..db36b6a512 100644
--- a/windows/src/desktop/help/about/requirements.md
+++ b/windows/src/desktop/help/about/requirements.md
@@ -4,16 +4,15 @@ title: System Requirements
 
 ## Supported Windows Operating Systems
 
-Keyman fully supports versions of the following
+Keyman fully supports the following versions of the
 Windows operating systems:
 
 -   Windows 10 (32-bit *and* 64-bit)
 -   Windows 11
 -   Windows Server 2019 and 2022
 
-**Note:** Keyman doesn't work on Windows 11 for ARM64. 
-Keyman works slightly differently in different versions of Windows. 
-Versions prior to Windows 10 have more language limitations and need extra configuration.
+* **Note:** Keyman does not yet support Windows Insider Preview ARM64. 
+* Keyman works slightly differently in different versions of Windows. 
 
 ## Resource Requirements
 
diff --git a/windows/src/desktop/help/common/os.md b/windows/src/desktop/help/common/os.md
index f64c565567..037e314e98 100644
--- a/windows/src/desktop/help/common/os.md
+++ b/windows/src/desktop/help/common/os.md
@@ -2,7 +2,7 @@
 title: What operating systems does Keyman support?
 ---
 
-Keyman fully supports versions of the following
+Keyman fully supports the following versions of the
 Windows operating systems:
 
 -   Windows 10 (32-bit *and* 64-bit)
@@ -11,9 +11,5 @@ Windows operating systems:
 
 -   Windows Server 2019 and 2022
 
-**Note:** Keyman doesn't work on Windows 11 for ARM64. 
-Keyman works slightly differently in different versions of Windows. 
-Versions prior to Windows 10 have more language limitations and need extra configuration. 
-Windows 10 and later versions of Windows include added
-security measures and some configuration actions will require you to
-confirm security prompts.
+* **Note:** Keyman does not yet support Windows Insider Preview ARM64. 
+* Keyman works slightly differently in different versions of Windows. 

From b5fd39418607eaba11e3a81f77b9c17abc43295d Mon Sep 17 00:00:00 2001
From: Darcy Wong <darcy_wong@sil.org>
Date: Wed, 2 Aug 2023 12:57:34 +0700
Subject: [PATCH 73/83] docs(windows): Address more comments

---
 windows/src/desktop/help/about/requirements.md   | 12 +-----------
 windows/src/desktop/help/basic/config/options.md | 14 --------------
 2 files changed, 1 insertion(+), 25 deletions(-)

diff --git a/windows/src/desktop/help/about/requirements.md b/windows/src/desktop/help/about/requirements.md
index db36b6a512..e7d80dda5d 100644
--- a/windows/src/desktop/help/about/requirements.md
+++ b/windows/src/desktop/help/about/requirements.md
@@ -2,17 +2,7 @@
 title: System Requirements
 ---
 
-## Supported Windows Operating Systems
-
-Keyman fully supports the following versions of the
-Windows operating systems:
-
--   Windows 10 (32-bit *and* 64-bit)
--   Windows 11
--   Windows Server 2019 and 2022
-
-* **Note:** Keyman does not yet support Windows Insider Preview ARM64. 
-* Keyman works slightly differently in different versions of Windows. 
+[Supported Windows Operating Systems](../common/os)
 
 ## Resource Requirements
 
diff --git a/windows/src/desktop/help/basic/config/options.md b/windows/src/desktop/help/basic/config/options.md
index 201c825961..3b4bbd86b0 100644
--- a/windows/src/desktop/help/basic/config/options.md
+++ b/windows/src/desktop/help/basic/config/options.md
@@ -60,20 +60,6 @@ To open the Options tab of Keyman Configuration:
     Click the Reset Hints button to switch all hint messages on again,
     even those that you have switched off on a case-by-case basis.
 
--   Select keyboard layout for all applications
-
-    Keyman allows you to select one
-    Windows language and Keyman keyboard for all open applications and
-    text fields across your entire system.
-
-    On Windows 10 and later versions, this checkbox is disabled
-    because Windows has this functionality built in. The setting can be
-    changed in Windows, with the following steps:
-
-    -   Search for "Advanced Keyboard Settings"
-    -   Check the \"Let me set a different input method for each app
-        window\" checkbox.
-
 -   Automatically report errors to keyman.com
 
     If Keyman crashes, then it can automatically send a report to the

From f8d4f8eb85d99e8656d0f5081f3bb08ff78e5f78 Mon Sep 17 00:00:00 2001
From: "Joshua A. Horton" <joshua_horton@sil.org>
Date: Wed, 2 Aug 2023 14:50:24 +0700
Subject: [PATCH 74/83] fix(web): restoring original outputTarget only active
 due to focus-maintenance

---
 web/src/app/browser/src/contextManager.ts | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/web/src/app/browser/src/contextManager.ts b/web/src/app/browser/src/contextManager.ts
index 42e7edd729..64bdb1a8c0 100644
--- a/web/src/app/browser/src/contextManager.ts
+++ b/web/src/app/browser/src/contextManager.ts
@@ -211,6 +211,12 @@ export default class ContextManager extends ContextManagerBase<BrowserConfigurat
     const originalTarget = this.activeTarget; // may differ, depending on focus state.
 
     if(target == originalTarget) {
+      // A focus state may have .currentTarget as null at this stage; if the func
+      // is being called with a non-null parameter, we want this SET.
+      if(target) {
+        this.currentTarget = originalTarget;
+      }
+
       /**
        * If it's already active, we should cancel early.
        *

From e37d5388af3c37ba63876005626795300b5424fb Mon Sep 17 00:00:00 2001
From: Eberhard Beilharz <eb1@sil.org>
Date: Wed, 2 Aug 2023 10:23:05 +0200
Subject: [PATCH 75/83] chore(linux): Remove Kinetic from GHA

Don't build Ubuntu 22.10 Kinetic with packaging GHA.

Also prepare for Ubuntu 23.10 Mantic - we can't currently
build that because they are trying to get ibus 1.5.29-beta
to work on so there are often new packages. We wait until that
stabilized before we're providing a new patched ibus version
at which point we can enable building for mantic.
See also #9398.
---
 .github/workflows/deb-packaging.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/deb-packaging.yml b/.github/workflows/deb-packaging.yml
index a923c1ab69..4b4ff1e2cb 100644
--- a/.github/workflows/deb-packaging.yml
+++ b/.github/workflows/deb-packaging.yml
@@ -103,7 +103,10 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        dist: [focal, jammy, kinetic, lunar]
+        # Currently not building mantic until ibus version on mantic stabilizied
+        # and we can provide a patched version
+        # dist: [focal, jammy, lunar, mantic]
+        dist: [focal, jammy, lunar]
         arch: [amd64]
 
     runs-on: ubuntu-latest

From 1289a54691d0221692e65c8aa6012e714ed95ba5 Mon Sep 17 00:00:00 2001
From: Eberhard Beilharz <eb1@sil.org>
Date: Wed, 2 Aug 2023 12:12:26 +0200
Subject: [PATCH 76/83] chore(linux): Properly treat test builds with packaging
 GHA

Previously GHA builds always used the "local" environment. This
change now detects test builds as well as release builds and properly
sets the environment.
---
 .github/workflows/deb-packaging.yml |  2 ++
 resources/build/build-utils.sh      | 33 +++++++++++++++++------------
 2 files changed, 22 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/deb-packaging.yml b/.github/workflows/deb-packaging.yml
index a923c1ab69..08e0ff1dec 100644
--- a/.github/workflows/deb-packaging.yml
+++ b/.github/workflows/deb-packaging.yml
@@ -18,6 +18,8 @@ jobs:
       VERSION: ${{ steps.version_step.outputs.VERSION }}
       PRERELEASE_TAG: ${{ steps.prerelease_tag.outputs.PRERELEASE_TAG }}
       GIT_SHA: ${{ steps.set_status.outputs.GIT_SHA }}
+      GHA_TEST_BUILD: ${{ github.event.client_payload.isTestBuild }}
+      GHA_BRANCH: ${{ github.event.client_payload.branch }}
     steps:
     - name: Checkout
       uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c #v3.3.0
diff --git a/resources/build/build-utils.sh b/resources/build/build-utils.sh
index 34015c9034..cdb1944246 100755
--- a/resources/build/build-utils.sh
+++ b/resources/build/build-utils.sh
@@ -74,24 +74,31 @@ function findVersion() {
         VERSION_TAG=
     fi
 
-    if [ -z "${TEAMCITY_VERSION-}" ]; then
-        # Local dev machine, not TeamCity
+    if [ -z "${TEAMCITY_VERSION-}" ] && [ -z "${GITHUB_ACTIONS-}" ]; then
+        # Local dev machine, not TeamCity or GitHub Action
         VERSION_TAG="$VERSION_TAG-local"
         VERSION_ENVIRONMENT=local
-    else
+    elif [ -n "${TEAMCITY_PR_NUMBER-}" ]; then
         # On TeamCity: are we running a pull request build or a master/beta/stable build?
-        if [ ! -z "${TEAMCITY_PR_NUMBER-}" ]; then
-            VERSION_ENVIRONMENT=test
-            # Note TEAMCITY_PR_NUMBER can also be 'master', 'beta', or 'stable-x.y'
-            # This indicates we are running a Test build.
-            if [[ $TEAMCITY_PR_NUMBER =~ ^(master|beta|stable(-[0-9]+\.[0-9]+)?)$ ]]; then
-                VERSION_TAG="$VERSION_TAG-test"
-            else
-                VERSION_TAG="$VERSION_TAG-test-$TEAMCITY_PR_NUMBER"
-            fi
+        VERSION_ENVIRONMENT="test"
+        # Note TEAMCITY_PR_NUMBER can also be 'master', 'beta', or 'stable-x.y'
+        # This indicates we are running a Test build.
+        if [[ $TEAMCITY_PR_NUMBER =~ ^(master|beta|stable(-[0-9]+\.[0-9]+)?)$ ]]; then
+            VERSION_TAG="$VERSION_TAG-test"
         else
-            VERSION_ENVIRONMENT="$TIER"
+            VERSION_TAG="$VERSION_TAG-test-$TEAMCITY_PR_NUMBER"
         fi
+    elif [ -n "${GITHUB_ACTIONS-}" ] && ${GHA_TEST_BUILD-}; then
+        VERSION_ENVIRONMENT="test"
+        # Note GHA_BRANCH can be 'master', 'beta', or 'stable-x.y'
+        # This indicates we are running a Test build.
+        if [[ ${GHA_BRANCH-} =~ ^(master|beta|stable(-[0-9]+\.[0-9]+)?)$ ]]; then
+            VERSION_TAG="${VERSION_TAG}-test"
+        else
+            VERSION_TAG="${VERSION_TAG}-test-${GHA_BRANCH-unset}"
+        fi
+    else
+        VERSION_ENVIRONMENT="$TIER"
     fi
 
     VERSION_WITH_TAG="$VERSION$VERSION_TAG"

From f5819adb866afa5940ae06622ab0728ce6c162a4 Mon Sep 17 00:00:00 2001
From: Eberhard Beilharz <ermshiperete@users.noreply.github.com>
Date: Wed, 2 Aug 2023 12:23:09 +0200
Subject: [PATCH 77/83] Update docs/build/linux-ubuntu.md

Co-authored-by: Joshua Horton <joshua_horton@sil.org>
---
 docs/build/linux-ubuntu.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/build/linux-ubuntu.md b/docs/build/linux-ubuntu.md
index 5240d13e20..3c4f4c57c1 100644
--- a/docs/build/linux-ubuntu.md
+++ b/docs/build/linux-ubuntu.md
@@ -53,7 +53,7 @@ sudo mk-build-deps --install linux/debian/control
 
 #### Node.js
 
-Node.js v18 is required for Core build, Web tests, and Developer command line tools.
+Node.js v18 is required for Core builds, Web builds, and Developer command line tool builds and usage.
 
 You can install it with:
 

From 323d74142f96cda505eda22305f9b5bd65fecbf0 Mon Sep 17 00:00:00 2001
From: Keyman Build Agent <buildagent@keyman.com>
Date: Wed, 2 Aug 2023 14:03:16 -0400
Subject: [PATCH 78/83] auto: increment master version to 17.0.153

---
 HISTORY.md | 9 +++++++++
 VERSION.md | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/HISTORY.md b/HISTORY.md
index 9140446de9..526035723d 100644
--- a/HISTORY.md
+++ b/HISTORY.md
@@ -1,5 +1,14 @@
 # Keyman Version History
 
+## 17.0.152 alpha 2023-08-02
+
+* fix(developer): more wasm uset fixes (#9382)
+* docs(windows): corrected nmake cmd for certificates (#9376)
+* chore: add run-name to deb-packaging (#9386)
+* chore: try another variable for reporting (#9388)
+* chore(linux): Remove package build on Jenkins for Keyman 17 (#9380)
+* docs(linux): Add build doc for Keyman Web and Android (#9383)
+
 ## 17.0.151 alpha 2023-08-01
 
 * feat(developer) marker steps (#9364)
diff --git a/VERSION.md b/VERSION.md
index aaa2187da5..e207402e7d 100644
--- a/VERSION.md
+++ b/VERSION.md
@@ -1 +1 @@
-17.0.152
\ No newline at end of file
+17.0.153
\ No newline at end of file

From 985c3eeb7eb81040463aa97e9a4720c80ce431f8 Mon Sep 17 00:00:00 2001
From: Darcy Wong <darcy_wong@sil.org>
Date: Thu, 3 Aug 2023 06:55:28 +0700
Subject: [PATCH 79/83] Apply suggestions from code review

Co-authored-by: Marc Durdin <marc@durdin.net>
---
 windows/src/desktop/help/about/requirements.md | 4 +++-
 windows/src/desktop/help/common/os.md          | 6 ++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/windows/src/desktop/help/about/requirements.md b/windows/src/desktop/help/about/requirements.md
index e7d80dda5d..3c241bc858 100644
--- a/windows/src/desktop/help/about/requirements.md
+++ b/windows/src/desktop/help/about/requirements.md
@@ -2,7 +2,9 @@
 title: System Requirements
 ---
 
-[Supported Windows Operating Systems](../common/os)
+## Supported Windows Operating Systems
+
+* See [What operating systems does Keyman support?](../common/os) topic.
 
 ## Resource Requirements
 
diff --git a/windows/src/desktop/help/common/os.md b/windows/src/desktop/help/common/os.md
index 037e314e98..32bf778813 100644
--- a/windows/src/desktop/help/common/os.md
+++ b/windows/src/desktop/help/common/os.md
@@ -2,14 +2,16 @@
 title: What operating systems does Keyman support?
 ---
 
-Keyman fully supports the following versions of the
+Keyman for Windows fully supports the following versions of the
 Windows operating systems:
 
--   Windows 10 (32-bit *and* 64-bit)
+-   Windows 10 (32-bit and 64-bit editions)
 
 -   Windows 11
 
 -   Windows Server 2019 and 2022
 
+Keyman also runs on Android, iOS, Linux, macOS, and in websites. Visit https://keyman.com for more downloads.
+
 * **Note:** Keyman does not yet support Windows Insider Preview ARM64. 
 * Keyman works slightly differently in different versions of Windows. 

From d69d67fa036382c11dca3616b2f3cec12887614d Mon Sep 17 00:00:00 2001
From: "Joshua A. Horton" <joshua_horton@sil.org>
Date: Thu, 3 Aug 2023 10:00:06 +0700
Subject: [PATCH 80/83] fix(web): forgot to target es5 when minifying
 polyfilled worker

---
 common/web/lm-worker/build-wrap-and-minify.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/common/web/lm-worker/build-wrap-and-minify.js b/common/web/lm-worker/build-wrap-and-minify.js
index 009a8f496c..1572238b7a 100644
--- a/common/web/lm-worker/build-wrap-and-minify.js
+++ b/common/web/lm-worker/build-wrap-and-minify.js
@@ -35,6 +35,7 @@ if(MINIFY) {
     sourcesContent: DEBUG,
     minify: true,
     keepNames: true,
+    target: 'es5',
     outfile: `build/lib/worker-main.polyfilled.min.js`
   });
 }

From bdff790eb98aaff7991d413965a27baf71d28a97 Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Thu, 3 Aug 2023 10:14:22 +0700
Subject: [PATCH 81/83] chore(web): Apply suggestions from code review

Co-authored-by: Marc Durdin <marc@durdin.net>
---
 web/src/app/browser/src/contextManager.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/web/src/app/browser/src/contextManager.ts b/web/src/app/browser/src/contextManager.ts
index 64bdb1a8c0..492c627146 100644
--- a/web/src/app/browser/src/contextManager.ts
+++ b/web/src/app/browser/src/contextManager.ts
@@ -212,8 +212,8 @@ export default class ContextManager extends ContextManagerBase<BrowserConfigurat
 
     if(target == originalTarget) {
       // A focus state may have .currentTarget as null at this stage; if the func
-      // is being called with a non-null parameter, we want this SET.
-      if(target) {
+      // is being called with a non-null parameter, we want this SET. #9404
+      if(originalTarget) {
         this.currentTarget = originalTarget;
       }
 

From ca1c15c8bd58cfba63368dfe98d3b602365f4278 Mon Sep 17 00:00:00 2001
From: Darcy Wong <darcy_wong@sil.org>
Date: Thu, 3 Aug 2023 14:19:33 +0700
Subject: [PATCH 82/83] chore(common): Update crowdin GHA to trigger daily

---
 .github/workflows/crowdin.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/crowdin.yml b/.github/workflows/crowdin.yml
index a7d1c1df19..27b4d705ba 100644
--- a/.github/workflows/crowdin.yml
+++ b/.github/workflows/crowdin.yml
@@ -2,8 +2,8 @@ name: Upload translation sources to Crowdin translate.keyman.com
 
 on:
   schedule:
-    # At 06:00 every two weeks
-    - cron: '0 6 1,15 * *'
+    # At 06:00 every day. https://crontab.cronhub.io/
+    - cron: '0 6 * * *'
 
 jobs:
   upload-sources-to-crowdin:

From 4ff375cba7a110a85a757e38c086b73d58fba035 Mon Sep 17 00:00:00 2001
From: Keyman Build Agent <buildagent@keyman.com>
Date: Thu, 3 Aug 2023 14:03:03 -0400
Subject: [PATCH 83/83] auto: increment master version to 17.0.154

---
 HISTORY.md | 7 +++++++
 VERSION.md | 2 +-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/HISTORY.md b/HISTORY.md
index 526035723d..3da35e3d94 100644
--- a/HISTORY.md
+++ b/HISTORY.md
@@ -1,5 +1,12 @@
 # Keyman Version History
 
+## 17.0.153 alpha 2023-08-03
+
+* docs(windows): Update OS requirement to Windows 10 (#9381)
+* fix(web): maintenance of focus when changing keyboard via Toolbar UI (#9397)
+* chore(linux): Remove Kinetic from GHA (#9399)
+* chore(linux): Properly treat test builds with packaging GHA (#9400)
+
 ## 17.0.152 alpha 2023-08-02
 
 * fix(developer): more wasm uset fixes (#9382)
diff --git a/VERSION.md b/VERSION.md
index e207402e7d..0f0cc2cceb 100644
--- a/VERSION.md
+++ b/VERSION.md
@@ -1 +1 @@
-17.0.153
\ No newline at end of file
+17.0.154
\ No newline at end of file