From ea466d7e5d80af200cd74554e53f56726dbc75c9 Mon Sep 17 00:00:00 2001
From: Joshua Horton <jahorton@sil.org>
Date: Mon, 13 Jan 2025 10:30:00 +0700
Subject: [PATCH 01/53] change(web): adds auto-correct filter, requiring letter
 in existing text

---
 .../wordbreakers/src/main/default/index.ts    | 12 ++-
 .../wordbreakers/src/main/index.ts            |  3 +-
 .../worker-thread/src/main/predict-helpers.ts | 41 ++++++++
 .../src/tests/mocha/cases/auto-correct.js     | 94 ++++++++++++++-----
 4 files changed, 126 insertions(+), 24 deletions(-)

diff --git a/web/src/engine/predictive-text/wordbreakers/src/main/default/index.ts b/web/src/engine/predictive-text/wordbreakers/src/main/default/index.ts
index 390ba9806a..580fe39a30 100644
--- a/web/src/engine/predictive-text/wordbreakers/src/main/default/index.ts
+++ b/web/src/engine/predictive-text/wordbreakers/src/main/default/index.ts
@@ -38,7 +38,7 @@ export interface DefaultWordBreakerOptions {
  * @see http://unicode.org/reports/tr29/#Word_Boundaries
  * @see https://github.com/eddieantonio/unicode-default-word-boundary/tree/v12.0.0
  */
-export default function default_(text: string, options?: DefaultWordBreakerOptions): LexicalModelTypes.Span[] {
+ function default_(text: string, options?: DefaultWordBreakerOptions): LexicalModelTypes.Span[] {
   let boundaries = findBoundaries(text, options);
   if (boundaries.length == 0) {
     return [];
@@ -64,6 +64,16 @@ export default function default_(text: string, options?: DefaultWordBreakerOptio
   return spans;
 }
 
+// Exposes `searchForProperty` for external use while associating it with this wordbreaker.
+const def = Object.assign(default_, { 
+  /**
+   * This method returns enum values corresponding to the character type as perceived by the wordbreaking algorithm.
+   */
+  searchForProperty: searchForProperty
+});
+
+export default def;
+
 /**
  * A span that does not cut out the substring until it absolutely has to!
  */
diff --git a/web/src/engine/predictive-text/wordbreakers/src/main/index.ts b/web/src/engine/predictive-text/wordbreakers/src/main/index.ts
index 7f1682fca2..92976d91fd 100644
--- a/web/src/engine/predictive-text/wordbreakers/src/main/index.ts
+++ b/web/src/engine/predictive-text/wordbreakers/src/main/index.ts
@@ -1,5 +1,6 @@
 import placeholder from "./placeholder.js";
 import ascii from "./ascii.js";
 import default_ from "./default/index.js";
+import { WordBreakProperty } from "./default/data.inc.js";
 
-export { placeholder, ascii, default_ as default, default_ as defaultWordbreaker };
\ No newline at end of file
+export { placeholder, ascii, default_ as default, default_ as defaultWordbreaker, WordBreakProperty };
\ No newline at end of file
diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts
index a0b687b9bf..3fdc83422c 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts
@@ -6,6 +6,9 @@ import { ContextTracker, TrackedContextState } from './correction/context-tracke
 import { ExecutionTimer } from './correction/execution-timer.js';
 import ModelCompositor from './model-compositor.js';
 import { LexicalModelTypes } from '@keymanapp/common-types';
+import { defaultWordbreaker, WordBreakProperty } from '@keymanapp/models-wordbreakers';
+const searchForProperty = defaultWordbreaker.searchForProperty;
+
 import Context = LexicalModelTypes.Context;
 import Distribution = LexicalModelTypes.Distribution;
 import Keep = LexicalModelTypes.Keep;
@@ -590,6 +593,35 @@ export function processSimilarity(
   });
 }
 
+/**
+ * This function may be used to prevent auto-selection/auto-correct from applying in
+ * unexpected ways.  For example, when typing numbers in English, we don't expect
+ * '5' to auto-correct to '5th' just because there are no pure-number entries in
+ * the lexicon rooted on '5'.
+ * @param correction 
+ * @returns 
+ */
+export function correctionValidForAutoSelect(correction: string) {
+  let chars = [...correction];
+  
+  // If the _correction_ - the actual, existing text - does not include any letters,
+  // then predictions built upon it should not be considered valid for auto-correction.
+  for(let c of chars) {
+    // Found even one letter?  We'll consider it valid.
+    switch(searchForProperty(c.codePointAt(0))) {
+      case WordBreakProperty.ALetter:
+      case WordBreakProperty.Hebrew_Letter:
+      case WordBreakProperty.Katakana:
+        return true;
+      default:
+    }
+  }
+
+  // Only reached when the correction has nothing that passes as a letter in-context.
+  // (MidLet and MidNumLet only count when there are adjacent letters.)
+  return false;
+}
+
 export function predictionAutoSelect(suggestionDistribution: CorrectionPredictionTuple[]) {
   if(suggestionDistribution.length == 0) {
     return;
@@ -608,6 +640,11 @@ export function predictionAutoSelect(suggestionDistribution: CorrectionPredictio
   suggestionDistribution = suggestionDistribution.slice(1);
 
   if(suggestionDistribution.length == 1) {
+    // Prevent auto-acceptance when the root doesn't meet validation criteria.
+    if(!correctionValidForAutoSelect(suggestionDistribution[0].correction.sample)) {
+      return;
+    }
+
     // Mark for auto-acceptance; there are no alternatives.
     suggestionDistribution[0].prediction.sample.autoAccept = true;
     return;
@@ -651,6 +688,10 @@ export function predictionAutoSelect(suggestionDistribution: CorrectionPredictio
     return;
   }
 
+  if(!correctionValidForAutoSelect(bestSuggestion.correction.sample)) {
+    return;
+  }
+
   // compare correction-cost aspects?  We disable if the base correction is lower than best,
   // but should we do other comparisons too?
 
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/auto-correct.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/auto-correct.js
index d1e4d377a1..84bb8b412f 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/auto-correct.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/auto-correct.js
@@ -26,7 +26,7 @@ describe('predictionAutoSelect', () => {
     const predictions = [
       {
         correction: {
-          sample: 'apple', // can be null / "mocked out"
+          sample: 'apple',
           p: 1
         },
         prediction: {
@@ -52,6 +52,56 @@ describe('predictionAutoSelect', () => {
     assert.isOk(autoselected);
   });
 
+  it(`does not select suggestions if the root correction has no letters`, () => {
+    /**
+     * @type {import('#./predict-helpers.js').CorrectionPredictionTuple[]}
+     */
+    const predictions = [
+      {
+        correction: {
+          sample: '5',
+          p: 1
+        },
+        prediction: {
+          sample: {
+            tag: 'keep',
+            transform: {
+              insert: '5',
+              deleteLeft: 0
+            },
+            matchesModel: false
+          },
+          p: 0.01
+        },
+        totalProb: 0.01
+      },
+      {
+        correction: {
+          sample: '5',
+          p: 1
+        },
+        prediction: {
+          sample: {
+            transform: {
+              insert: '5th',
+              deleteLeft: 0
+            },
+            matchesModel: true
+          },
+          p: 0.8
+        },
+        totalProb: 0.8
+      }
+    ];
+
+    const originalPredictions = [].concat(predictions);
+    assert.doesNotThrow(() => predictionAutoSelect(predictions));
+    assert.sameDeepOrderedMembers(predictions, originalPredictions);
+
+    const autoselected = predictions.find((entry) => entry.prediction.sample.autoAccept);
+    assert.isNotOk(autoselected);
+  });
+
   it(`does not select solitary 'keep' suggestion that doesn't match the model`, () => {
     /**
      * @type {import('#./predict-helpers.js').CorrectionPredictionTuple[]}
@@ -59,7 +109,7 @@ describe('predictionAutoSelect', () => {
     const predictions = [
       {
         correction: {
-          sample: 'appl', // can be null / "mocked out"
+          sample: 'appl',
           p: 1
         },
         prediction: {
@@ -91,7 +141,7 @@ describe('predictionAutoSelect', () => {
      */
     const keepSuggestion = {
       correction: {
-        sample: 'thin', // can be null / "mocked out"
+        sample: 'thin',
         p: .8
       },
       prediction: {
@@ -110,7 +160,7 @@ describe('predictionAutoSelect', () => {
 
     const highestNonKeepSuggestion = {
       correction: {
-        sample: 'thin', // can be null / "mocked out"
+        sample: 'thin',
         p: .8
       },
       prediction: {
@@ -133,7 +183,7 @@ describe('predictionAutoSelect', () => {
       highestNonKeepSuggestion,
       {
         correction: {
-          sample: 'thin', // can be null / "mocked out"
+          sample: 'thin',
           p: .8
         },
         prediction: {
@@ -149,7 +199,7 @@ describe('predictionAutoSelect', () => {
       },
       {
         correction: {
-          sample: 'thic', // can be null / "mocked out"
+          sample: 'thic',
           p: .2
         },
         prediction: {
@@ -181,7 +231,7 @@ describe('predictionAutoSelect', () => {
      */
     const keepSuggestion = {
       correction: {
-        sample: 'thin', // can be null / "mocked out"
+        sample: 'thin',
         p: .8
       },
       prediction: {
@@ -204,7 +254,7 @@ describe('predictionAutoSelect', () => {
     // Refer to AUTOSELECT_PROPORTION_THRESHOLD in predict-helpers.ts.
     const onlyNonKeepSuggestion = {
       correction: {
-        sample: 'thin', // can be null / "mocked out"
+        sample: 'thin',
         p: .8
       },
       prediction: {
@@ -246,7 +296,7 @@ describe('predictionAutoSelect', () => {
      */
     const keepSuggestion = {
       correction: {
-        sample: 'thin', // can be null / "mocked out"
+        sample: 'thin',
         p: .8
       },
       prediction: {
@@ -269,7 +319,7 @@ describe('predictionAutoSelect', () => {
     // Refer to AUTOSELECT_PROPORTION_THRESHOLD in predict-helpers.ts.
     const highestNonKeepSuggestion = {
       correction: {
-        sample: 'thin', // can be null / "mocked out"
+        sample: 'thin',
         p: .8
       },
       prediction: {
@@ -292,7 +342,7 @@ describe('predictionAutoSelect', () => {
       highestNonKeepSuggestion,
       {
         correction: {
-          sample: 'thin', // can be null / "mocked out"
+          sample: 'thin',
           p: .8
         },
         prediction: {
@@ -308,7 +358,7 @@ describe('predictionAutoSelect', () => {
       },
       {
         correction: {
-          sample: 'thic', // can be null / "mocked out"
+          sample: 'thic',
           p: .2
         },
         prediction: {
@@ -343,7 +393,7 @@ describe('predictionAutoSelect', () => {
      */
     const keepSuggestion = {
       correction: {
-        sample: 'thin', // can be null / "mocked out"
+        sample: 'thin',
         p: .8
       },
       prediction: {
@@ -362,7 +412,7 @@ describe('predictionAutoSelect', () => {
 
     const highestNonKeepSuggestion = {
       correction: {
-        sample: 'thin', // can be null / "mocked out"
+        sample: 'thin',
         p: .9
       },
       prediction: {
@@ -385,7 +435,7 @@ describe('predictionAutoSelect', () => {
       highestNonKeepSuggestion,
       {
         correction: {
-          sample: 'thin', // can be null / "mocked out"
+          sample: 'thin',
           p: .9
         },
         prediction: {
@@ -401,7 +451,7 @@ describe('predictionAutoSelect', () => {
       },
       {
         correction: {
-          sample: 'thic', // can be null / "mocked out"
+          sample: 'thic',
           p: .1
         },
         prediction: {
@@ -436,7 +486,7 @@ describe('predictionAutoSelect', () => {
      */
     const keepSuggestion = {
       correction: {
-        sample: 'cant', // can be null / "mocked out"
+        sample: 'cant',
         p: 1
       },
       prediction: {
@@ -456,7 +506,7 @@ describe('predictionAutoSelect', () => {
 
     const expectedSuggestion = {
       correction: {
-        sample: 'cant', // can be null / "mocked out"
+        sample: 'cant',
         p: 1
       },
       prediction: {
@@ -480,7 +530,7 @@ describe('predictionAutoSelect', () => {
       expectedSuggestion,
       {
         correction: {
-          sample: 'cant', // can be null / "mocked out"
+          sample: 'cant',
           p: 1
         },
         prediction: {
@@ -515,7 +565,7 @@ describe('predictionAutoSelect', () => {
      */
     const keepSuggestion = {
       correction: {
-        sample: 'thi', // can be null / "mocked out"
+        sample: 'thi',
         p: .7
       },
       prediction: {
@@ -534,7 +584,7 @@ describe('predictionAutoSelect', () => {
 
     const highestCorrectionSuggestion = {
       correction: {
-        sample: 'thi', // can be null / "mocked out"
+        sample: 'thi',
         p: .7
       },
       prediction: {
@@ -551,7 +601,7 @@ describe('predictionAutoSelect', () => {
 
     const highestNonKeepSuggestion = {
       correction: {
-        sample: 'the', // can be null / "mocked out"
+        sample: 'the',
         p: .3
       },
       prediction: {

From 8eb5fc3b68b5f0a53aa7a2916f0d4e22fcccd767 Mon Sep 17 00:00:00 2001
From: Marc Durdin <marc@durdin.net>
Date: Wed, 15 Jan 2025 08:30:55 +0700
Subject: [PATCH 02/53] chore: seed epic/autocorrect

---
 web/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/web/README.md b/web/README.md
index 7f7415c698..05f9432c20 100644
--- a/web/README.md
+++ b/web/README.md
@@ -1,5 +1,5 @@
 # Keyman Engine for Web
-The Original Code is (C) SIL International
+The Original Code is (C) SIL Global
 
 ## Prerequisites
 See [build configuration](../docs/build/index.md) for details on how to

From b299cecaafc5f8f9b0b35ad6bb36a081c333508c Mon Sep 17 00:00:00 2001
From: "Joshua A. Horton" <joshua_horton@sil.org>
Date: Wed, 15 Jan 2025 12:02:02 +0700
Subject: [PATCH 03/53] change(web): re-enable autocorrect in Web

---
 web/src/engine/main/src/headless/languageProcessor.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/web/src/engine/main/src/headless/languageProcessor.ts b/web/src/engine/main/src/headless/languageProcessor.ts
index 82c9a0392c..866d3607bf 100644
--- a/web/src/engine/main/src/headless/languageProcessor.ts
+++ b/web/src/engine/main/src/headless/languageProcessor.ts
@@ -21,7 +21,7 @@ export class LanguageProcessor extends EventEmitter<LanguageProcessorEventMap> {
 
   private _mayPredict: boolean = true;
   private _mayCorrect: boolean = true;
-  private _mayAutoCorrect: boolean = false; // initialized to false - #12767
+  private _mayAutoCorrect: boolean = true;
 
   private _state: StateChangeEnum = 'inactive';
 

From bd09450d6943f6d8356592039c3e0c530aa7e7a8 Mon Sep 17 00:00:00 2001
From: "Joshua A. Horton" <joshua_horton@sil.org>
Date: Wed, 15 Jan 2025 12:05:43 +0700
Subject: [PATCH 04/53] chore: Revert "Merge pull request #12791 from
 keymanapp/fix/android/disable-auto-correct"

This reverts commit 30ed4b8d7ff0ad03243b171a544470e88ced172f, reversing
changes made to a420e7c49e27e294b3fe2631666946d9993a2124.
---
 .../kmapro/LanguageSettingsActivity.java        |   4 ++--
 .../layout/language_settings_list_layout.xml    |   5 ++---
 .../help/android_images/disable-suggestions.png | Bin 15434 -> 0 bytes
 android/docs/help/basic/using-the-banner.md     |  12 ------------
 4 files changed, 4 insertions(+), 17 deletions(-)
 delete mode 100644 android/docs/help/android_images/disable-suggestions.png

diff --git a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/LanguageSettingsActivity.java b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/LanguageSettingsActivity.java
index 96abc83f8a..977489f40a 100644
--- a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/LanguageSettingsActivity.java
+++ b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/LanguageSettingsActivity.java
@@ -108,11 +108,11 @@ public final class LanguageSettingsActivity extends AppCompatActivity {
     RadioGroup radioGroup = (RadioGroup) findViewById(R.id.suggestion_radio_group);
     radioGroup.clearCheck();
 
-    // Auto-correct disabled for Keyman 18.0 #12767
     int[] RadioButtonArray = {
       R.id.suggestion_radio_0,
       R.id.suggestion_radio_1,
-      R.id.suggestion_radio_2};
+      R.id.suggestion_radio_2,
+      R.id.suggestion_radio_3};
     RadioButton radioButton = (RadioButton)radioGroup.findViewById(RadioButtonArray[maySuggest]);
     radioButton.setChecked(true);
 
diff --git a/android/KMEA/app/src/main/res/layout/language_settings_list_layout.xml b/android/KMEA/app/src/main/res/layout/language_settings_list_layout.xml
index 42ae9b1511..6399987b04 100644
--- a/android/KMEA/app/src/main/res/layout/language_settings_list_layout.xml
+++ b/android/KMEA/app/src/main/res/layout/language_settings_list_layout.xml
@@ -90,13 +90,12 @@
                 android:layout_gravity="center_vertical"
                 android:text="@string/suggestions_radio_2" />
 
-            <!-- Auto-correct disabled for Keyman 18.0 #12767 -->
-            <!--com.google.android.material.radiobutton.MaterialRadioButton
+            <com.google.android.material.radiobutton.MaterialRadioButton
                 android:id="@+id/suggestion_radio_3"
                 android:layout_width="wrap_content"
                 android:layout_height="wrap_content"
                 android:layout_gravity="center_vertical"
-                android:text="@string/suggestions_radio_3" /-->
+                android:text="@string/suggestions_radio_3" />
 
         </RadioGroup>
 
diff --git a/android/docs/help/android_images/disable-suggestions.png b/android/docs/help/android_images/disable-suggestions.png
deleted file mode 100644
index a4a21b98cb5d277509ebbb21bf92141ead7d7048..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 15434
zcmZ|02RN5){5MP}Qjr;DR5nE_A|on7A!Qd5l5Dc0%qWyqvPmh)UfFvmNs=uRWy{EV
zKiB<#o_8JZdmqPrcgOX+e%E=P-|-pWD^OikaraKvog^eAyOl1S(;y)sJ%j(xquh@F
zj*b4^i$BO6<dv>c;vWx6lZPZEoFq!;<gU8JPW3x`T=~1%FxRd0Lae`pLb=D(C!%cm
zs3-+#@qHK4%SG0i%!>^aN9@lZxNjgwOU}T!H&Qx+U!6}q@e|Yc&sEbGJD<K(`ZyE%
z!r`=r8O!IHPDPGSqoc7l-uIT*qz4_!IhssFn<%|Gugbp-=qHh*^$j>rp2$fV8AW+U
zj#l8v-uw8K-XRhSxwO1<Bs;y;Z%au1*-q&_+kLR5(77v?OKG{7oAE615ph%MviZ%t
zY`=IerN*3VBWsG0#66n%*zF61)4P-v<Ybud>R$W&`+{I2Z-e=n?NSHgW*wXkHn#bE
zj~^f}`fyN*P9SopQW$%W2m>h}rFXL9EaQu|YmLgcRH|K_cpB(qqC-BKR~kq1|I2(x
ze&IqI7nK?>@1*P6=X3?O_;<bc>&r>a4;)Z`vnf6{n?|0*W$i~#PO7}Lw9AjjiI;n}
z?#;~uAHw7x^mu5meR<1feuevCQ=9&=EY*9lCqvS@CKX>P@@-S6CFfodVxn$%|59{$
z)?m*y)mNV^GM;Rta|b$UrY|^^SUXa^_tVQbdR)qLA9h5Z+id^5UWV4<b}|md&8lR2
z#m3{z1J8ck?%E^O{f;~L``wz(p4Ft~?dSiy_V?fG%}r+YHu~QpBko78?Wdyq^hR&D
z%x~$3ZZe)UC&~Z2%H;Q|t`O5vGt=4DD2_nolBF}f>g_E7cW)fNJ<6Q8E6|GlDs9kd
zIr5~i933GpQ&Ulk&Se^@b&`?X<1G2B?#7=pZ)WRc>1B)=+;pjUNW$Q&T<>(r@O?++
zm+KX_Vuz=a^2d^o#=I%<<dzm@epHuyvHsh4*NQ;m!dfOzv)pK}t+~@Xj=q-q!(%s6
ze{!I&B3wR~<#$5vM;d|udBV%<At7wc!dZv@)z(;t8Q1tZybFD#<zGE(B(IKGxOyl@
zDEubh&|tulQJxOFlLG^j?$3>s9Ys&mh3$^cdeY-~BhSoH$5lL%sOfpUpQc2~lxxgn
zL35F+)oYI*NA2nv1m5g(LH0uwempX#?=H$1PB=T845jB@HenupMp?PW;*Fq8qRaZb
z>8Ym5rSxngE9T1k{j08)s#1m{InPH9>U7|As&Bq?nQ5}f&l#?kXtl5F8+4b|dD=tk
z^}dz+m>46QFYYmX>=rd$T+;os+5$^C&v-l^NbOaR3@WwxF&g~)T6@<s%Br<4Y9by(
z-;6xCk*_C_5dE>Mv~Q<$ec0yN)qhQw(=-{Gxh2CNq~+Z)<&n1(5T(Q+{`bWbj{oky
zW0dxd;QI15sLV!tyXv!@mH+#068g4WCby)+ax0d&Mon(!>$OZ#XfY7W-Y3AePb)q0
z_bTJ*^|FM|3k%k6Hn!&}lGCUT;UPYsZFuax?*85I_2fy{w>NU>Lc9LXS1Dzf-N?+M
ztN9(r+@5=6c>OwuV5hmd*?UcaNDb4EOdgE+*NBXEY3-4(@d$0{75_^;TBpUPEcHt3
z&6E4LkMursk?Pg4RU_5Wu~@mY_V(A^ZU3#Jen9<llxg3IkmbBV&08T0Xa7vlg<83~
z(HW@feaN^T)bp>x{n1Rp!?55OpP%LNYq@)gqxG%GZILb;Fwc(AJ(KjwqVAA>lPE)<
z15NHJ!we?A(~pG*sNX$gPkK(i*SFzG*}xS>=E8t4FHVnV-aUFHWyO=0*lh`YmlJby
zYbxuNDrasA8&+;w+MM@K+P95bo|{vNZuh-)TPIu28TR&eM&}g`QR|QpMpoqz0mk4m
z50zbar}}qm?4>_Mgh&{r{Q_rgXF%G{Yt*wD(!BzfhGGx&m?vHTMYvw;qZXp9X5;tl
zk5$NcWf&|;Q+&SgYl?+PNSy`6pxS@Yp;bCm#HF<BJ)^f)hG%1jKmu!ERn^@o|G^E)
zb2u$^r3+!_j;PuEBkywic8OnGzv2w5+gh^5n}|;Z_ZYP=-yli~yF)>i`kSk1R|Q^)
z?Kz+Bw&?bNRmoVRzy8#^#5&_yBRSe><q4APjq4d&ovp(6pXUFTefKIdUEApHcM7eu
z8CrvRJ9M5_{m-3J+sPzPof1jWy<<`QL3T81BQzm}UijqB^FM~vdDbnb6*GwplrsDC
zH_CE2N#l)rdKdYJ;sxvUkEfJrWVapQ3KZX;|MEl0j!EO1!I(HQ-=GO9nRRDj%IbTX
zk{7NDFz(s2$MGFgtA5+}L|2mW@69O+#`aSRi&Sp|YZA(2d%LH6hsx-P6XJJ2{yO!n
zt>p$!>9--O1MKpYZ|~o-_;OR@0{scr-4;ldEp171m5P9%L({}b>r5QXI<js_`fK!8
zV?_NCTa7m>C&YH1rFj3#{y4TnzU|+|+n)~bDmL^^ZuEx+(*92_l)L3jt$ildks{|k
zVXlz%==4l{$k2x-Tf^R;Q;Ea*+toe^r!j@F2Ge;K3=C$tJz!K4jlSp=Dar56+4o@N
z5#={6?E<|F9U&$Zb+cUERRJNPS`*Qy$t@!_$DjUDD)Am;TC2DskQnz&HdD+}&Pq-A
zgqMH9RQk$t!LaR|h!DFkBjdyWi3GP}m-SQg=jF4qvS?G9YB{v8=sFDxjW6~*pUIAj
zWexmr)LKgW_1^1bJ`o0nb6L553Fs4h?~m;c6bvGg?=hp{)jSvBPX?5FLQFfUsUt;B
zFfrLL{$ms~EWdEhjJ3y%zR=xYouVk6&3tOAz4Glz^KnB)tL(s#5jkxs$|=9WjmeD%
z#GU>`c2f(nc`v#770l%Brig7)9(UA|w3Fz$c5*aa*Zfd}<u;Z+wi7F-B=@G>ef1!Q
zx2@FU_Q=$u&~S(71@|#Oqlt0$+EkJMmZj}>d-ZeKncjrdv!9lhN90`7jmMUF^n<p_
zN=fpZKei9B9ZVL8<)n13HITlp&lkDpm^-Oiwob&XfXw5g!X%GoUb~P)byJtd4GEUd
z1aTqU1*qKR(vIG;s{Aac<MGWhbBB&cpqsmMJi`oW)Sjx(?Tj=ZN36;?{_CDb)Uxh3
zjAu6W)4TKzty|U%j+Du|*BH4=SAI5JlAkei_GH+~P5!SpjhG!w(|FTtd52ma7bi38
zxM`#={tl`rUzDhfxfdxue9(<RD-ZSkj9kwSj?nX_)?_ZS@fOIc>@b?(%v<)D>t-4p
z+&Dx$Vf5#N=4%5>rqrChgI03pF(-<{Nrc#Zm*g``*#|fF$dUvcnaf_@;PmE{da?Of
zBuMm6yR0e8YI}vCoc83gBG+hTrLZSKo8DXxd>6+DAKl|`f3~xH?XTHXe)z0c&~E}S
zMZA3Y#RM_Tc~yBtm6iZg*@yD}Ph^$RTfVE_Tr6B0hh<XRsq`qles6=lb;@3HxfQ`8
zQxcwu932MNhf{&$O3h96miZ%T8dth|1F8-NKkvvPBYkg8lbqqrNy+%OPl;AHO4oUs
z;kc6xZrE97W+kbe=~wtQI&WQS=_Qx_FAdSnqwj^L#3MHEPRc%^W~RWi1oe|dm_{?c
zJ{+BWhdv{nd~VS5?ClW_ouAw4QnWQ(t$WqD%x=wZJUO=OZTBqcNqt)K;ennwa*1%K
zIPoRRl<WCM1&?FTT-iGF#MJos=Tk2Hfxdq(l6=>qHf!i?ryH4SiJ017{Ia`8`eB{%
z@b~;XzZ@kWE8GkeY|PLSjE;_W?z8o&s_ZuOvRbxSyc8q$^u-IehBc2b&oA;gO0Ffm
z;iO#tXiehJ@!dyHBip#rRw_2{T2tGNPoe3zi!G?6yEBFYGnwXEUvV^wGN`w;`Iv45
zB#E=E{`w_XvO~h*=PumR^h?_1#Xmjac0cN$#>Vcdtb4@FBI_yT=jZ4AWp;ht<7ZpU
zt>xmemS^-F9A1iR?pOcx<OWbPA0?UXwW*j}>a~#_WumJ7k$#cHjEN!ZIDh=p-7%ND
zdzn*uT8l2pW(o4MFC?4~zr;lS`)0ey-L>{Jm({Xv1xv28a;O+2y=j$urzQCJclWJ^
z5ViwSE>E5wR-7-a*a$8ym9pr1TleDR9a~$z7okn<?Yt5a(S3b=bZd>x&4;+TDN;Ig
zM(o0y&M7PJbai!YzM0Bw(c0QNIzQH;BeX=l!N|byBsTW3L8i7rcwAiY-}dD@Cs<YT
zc%`M|(zP><MWZ9SNk&IUpS*aXeB;JJ$vD;90XTursVXL>rjh3Q1_ta+O-;(Zf8C@F
z@BL+TU!JZr68V&SORc)5CUR=`b~1i`e(sRygoMy4qpUS{l^H`?->~M^Z4G7voE4wW
zg}%rqUHM@z>UB-IU~R<kXzXFJ!@>ULn~s^f0oRqNfS0Z`^cMD!DTmb#Rr%<adq^mT
zvZpEpvtEo53%2Slu>0A{V&FEx?d{{Ebn)V74UO!}vjKaA49jD&SM>Y#HAV_*uS}QB
zplLBMFo@j#%=K8|Fdq1wZmz%oZk70eYq+KC`YhS>^mI{iakgcTXr5liyvN0hyE<Q8
zN^$!;$;HDXYBM19z0jup;oie5n`<+33kx@U3#^D!ppjjFqM5FhZPG+H`87iY8$?P<
z>M>WxeD2)2*=lNG^+N0G2M!!4vLD-7;xu(gRyLt2ocFL2UHALzH~%e8Nd5hC<(z`T
z{J-Csf-`qJlFuKI@pyh%G2}@^#JM=hyN&Jb#BmGh=J<a5rW5bA>FVw6eL+cSIALp@
z*Jt}~<GGQ#1G1h6l)^4tx|E8o_YDZ3XJKi&lUarxI(gyZMH5_+XVEqJev=pdQZt%;
zD7aM0B=K$fX_7fzmN6}g$Sn_c&QD!zxt825>bAdq-d^?f^`&KI#u;D!=x}eLP5d$I
z$)J;WzGm4D%d5sqDPf5{Jw5LXE8@G-wL4!b(R*(E)lN)XULFqM$no58BMt*cud<k>
zaplVD$RpuzMRtc}H&+e|L<UXu7Ctj?aUQ(aQ2BsNH`i?UBWB^BqfOx?NaoZXIyyQR
z5KX$ex*oGtJJzu$9?SjCi<4cC5aTAgKc@-@%{JFpJ+?NMOXr&fU%q~QxJ5L<^B;eP
zUj9hQO!>{J@A<7U;-S^m@>-b&Z7Xv>h_9Hb*plH@i#Ne;FaGNAVK(%58ZYgRux@N?
zJ0E3gaPP0C*~i!>R21D3rz6L2edH1peB|xD?d{b!)gSKu{+4xvOm=JiI8S&W)_eR$
zHM!{)uWD@3Sj8jGt74{Z_v2O1m!(*ykdk;YcgIXuE|q>XKeMebE>FCFpx#7u_}4F;
zk?g&P6=r|N*wC@DwcRHrdoJZF8WIw+IMEr<lY8st304WGKl=_IJlK3^8yVSCuE6)#
zOXjO~a70j6A`%iiCsrC{H%Tjt9mXlCsBZlFl+-MS2p>y*W^hPGCZ3dRyNK2IJp10*
z7cYV`GlezZUX=^udaIrJzVum9(J9HhlfGO^<U4i<W=hx&k+25)<A*3D$N=oQLdb!p
z*68CBuJc@E+jkj_)CO{gXs(APqnLe+l{mXR-H*p%J!vsCKCY;tK}UixIG3t1{PX84
zAw3C;FMOMSGrb~9yyR&AkQ+TL6b`XCsrdfoyh)kD;yc}EdhrHa6FDuVMa5o>dfHUm
zI7gHPJdeIF+;x#p@--8pbZ~2PnJ%=@vL_6AyqBK7=2pVJdJeDkD&!`yM=j5UdAYcJ
zvGDKxubWJ&o0{I7vrX%=K&h)gJyV(@^>b`YbL1{jnURgPb#>W}6B<j?{ViUb3kh_+
z&)dvM4zRHui)k}8i`V=Qg*x%tf3Y}8&#U?pA0laum)>t<V?!kN(p0a6`;yQjrXz<$
zMZ>n%Dz<Xh1}yzjtSU}mvlLSH1}l)=&&_p+?#{c@t=h9SQuiqE@ne&vKRr&%!jzGk
z(Fbks(x_pTBP9;6n}ix0I+a?))nqPR3Jt63_$GYcW`}IBUg{y;XLV7&r#|fHuGiOJ
z3fLR8ZE~_J{aTUjgKh)&V6(Ot9i^@dbiEg2Pd&Z+`)fy{9NE9WzZK=>y+u2Qe*J1h
zL6h)U$r3$5@h$u24y1|D^$!s;_x>SJtouLMZLYh1{``4%Vd3zhL*99Ld2|gPo2%ox
z#rE8YtzG*>?$_3yLn*SgvpXmt5VW}HSYX}1^9*tayq@S(t*sY5*QUftfV8FE7L8Wt
z$4qYB%C+hhGu^83-~D4e!K=Q%pN*H7cdWQ0MRB1;xZ+ch-A%XU=}X4Ohr+lJ=@;DG
zB)9v>^#p{6)06GkGrG_&i^~JpPCE?^4V@`-`}@c9YmU%yw?%W$&DC41;?@`2<E51@
zT*xZP;Jj+)WaGD{9WcGCC(GY!y1(l4=Gog<@+J5Fc%99CFyxM_n}w@eXNBs$KDIcK
z=hUvZmzj39`jrm6f7I`Tx;;ELRu{p4Wp;Kp@m71h)I#fV1WIgN9M$gK=fk*UR))xa
zwno>yx)g&ej0Q?wSN{E*|9E<4Hq>jKOjcGFW!?`Djr7RM5(#0KZVuyCq1?ICxXPP?
z<izdI<Z5ba9{27M-G`b<C^0MR3D2b$BIchG>8z+}XvPu9;y>zxYueg^R~N>A3=LJH
z8nvviF4)=GC0$NYaB+7}>9Qbd;d5zsG1lPihJ#y|mWWa%KH%b0u`y+H!s`1~-M>pc
zw-UI;#h)P$n6Pm?s<99CGJd9n${0`g6|c|M(73p|&dqpj?Z1C3-qqiEz%>A=FoN7f
zD~Q-=k3Jp%XfixC)fU7k_{kz4aD@!<otReG^Wp9t1U_=>n8VN38yo-raxa0NZG9`F
zuQkq}Z!hcB^;j1Pe^F@cxxDFe>UK*XRoLYx?Lb^T2RK^Rr#D=R@4j{~XZxt-k?Qi4
zf$u{^t(M}DQ$a<zRkQoR*Umh)D^aF<*rhI{Yh`>gKhYX@npsg%almz)jp!Ka>a+s`
z1D5?CA{hm>W)U}hYq{N<F=g8BR%LO^Zo<9P@}@Tnccf{(p`7{7oG!G`6L|QKs<M%B
zZj|F&1A1ZJTc4@tzs1-;&1{e5Y|*Lw^ReuC$J?8J>ra%zObT@t+IF6Lw%j*mePv)F
zwT3B(M*~1CKsmL#t1A*<jO=_j|Hl_6se=`Ks*HwygvuUukePQ1br+ubYf!-K+x+vD
zpuE0m^rg9Dt+J(|^xc^eMF$kPl^l1yt@@){_KWTswI=uE4PXbO6RdNyvwVBQ&<fCy
z`uh7LsY@v2TpatqhOKDn@*uf;7WbR5?1TiFq<e<(ZL#asw?p*g!*ho$yeXk`26E5+
zU!o`Y2YJ$Gg7wO$a)UqkDy#h8R8nC1>vEah;ZL8+d3wscfB$}Eb#-{s?RRP>xg0IK
zrugnNx>3b0bB9p(4$H_GylH^ash1Ijrhm+K=*(NK3<2u`X7XX-*L|nuz*0{;Pn#_K
zY%BWkfsU0`8Hq>~0R$R)fohzjU!C!dmDyp(>As%n5mHcq(6{!~DK@<Y!AJB9K9#yU
z#C!g`EhR0jvN&&|%L7=5Mi~D5dE@bRsk1;Q9OgfO|EK->kLt_IZ__WdPIX=QxxO*u
z)zH$C-b$+-#<jk&;R;Cf=F00m1kwWb$#tjosXF7~Q92(PA8%l+Iet17eQwByCQC<1
zIZ)Z1;`3Co;{;t#kLj~zuYJkM$vTDB{WaG9P>!9*Y=3{vz*0or7Juo#=Y8E};rcWB
z))TJ4xYN$x`+0bH{D}q%$R~R1;{pHO`|J1Hl-f1aeX^QgSReoags|%+PT4nV{Ot=*
z#<U6PWZjRAJ&>3--}iuQ$Mj?o<5aaD^_Zs_*uIx3kDR@|{e=q`5*@}ry12Owd{qT5
zkVmoy1_mZ2CMI5d%B~yrMl+oWk-shIYv=F53fUbr?4yCa@n<r$940<<TYO1-5EHYX
zf`X#9wwCt5fyWgUvSC~YTSUdJ`-XtQS5{W|*4RaiNVL*5dx+f~9W|Nk&I~OnIn69=
zVA7Lg>Nxd1^3x}El$@7eD56ZG?{vSnTb(y$6|;Ppo-P>1W%qYdv$U)Xg|xD%DG-=N
zU`<U`mFM{JaKuAWmqkk~lSuT%*wfB?Pz@};z6<118vXgx_ylX@TiZ)7PVmpMjQ#u>
z_UxJBnKL9|Tu-B;aY3`jGS<%leAAt2_!{!WxrwIU%b5n!*p5%X|BhMTOb;La^XFpQ
zi<8Bk8<J?8$vFAeSc&k62!@aG(ke3pWdxq`*+FB5$Rn{tG@(_^c4-?Pv-*C*w6Vl~
zOj%}il-@+wtpCH^UHioz+f-~s8I-y0-MhC6<f7^0M>DM5#Kc6zeaVvO7q}8&_A$$G
zf1C1E|LOh`AYH1;BD){^j+^aD@Y<9BS~95%Vr<N5?JIVOmboV>sQvyWHVX*9xOPoN
zSveW4;hj#lH-Q1c6t7RDF8%Hf)i!YT3zgk`4pb;`d{X^{&`js+P>p-j#pCfmes!cc
zFRoI1+m1Ick}S;6lT5F$%4%zCo9Mnutn|F$7^m7WQ0D&BG#c<6QR)I{8W<d$YFOdf
znjp);_jb3G^Yk^MMP9ph0I-8gNa*qGtZn)Fyvp|iD-}n_6E_<|j%0g<H@##yLbGSj
zPrzcLP9lM_@2xovxX(x<m)}}?;e?3~?O~JjMJxg6o1&fl`Ib%gsID6LnFZlfv9)=c
z*adtIk?ZfS6^9{WX3>MgqoQ7Y*^a=Wpk^Wn5Pt02WRk3%X>jiRdD5YonRcKM(qG@Q
z4<q2Ne%Gk*ytlr-e#&$G)b9OaT)>zg<4zBDjH6-13DVy$7;0&2D@$QR%m|d7b{9Q<
zya#9`jB76~Z6yjX)lPx8{FWslXan~D7H@+I=oZ-?M9$^qop6+70~yRS@7Sgk2Am?8
zyY=p!5a^hkoE#+2XGgF9>MPFuoGx(W;K2t&Lq@2u50WK60IH(;CPM!lP74UwvAVj7
z);{2|aOst5oHAmE<)p>cz?}FTA&1E?{01I#=ed*s6$8brzT4sfP`9WkDb<hC+uPak
zUtZEk)i?&=DrVhx0elR{yW{B5qyPJe$&DffH1XiEvR(#MvHCbxu=k;fiNJt>fPO2B
z?)PnLGZi+=66kFtsTY4DN~!kj*}kd$t)ZufrOY5SJUqOg#qicGglfI{iII^Jc@%G?
zxw`z<ud4fLXlU#m9G;r8D208Ra5O)`3ZNCn#ecrR3KbeK%y_W;9zg8ee2XwR0n0IC
z-_NSR9j5+0PBNdJ*t-CKFYXW8FXM6Q#*G_Xyu8Qn{d2UtQOqetV)~KvLrDoe9bNU@
zkA{v3$HM$@Fn2n7`m-ohkyQRA!+<1MYqYox`^3ZqPJOP4H-Ycu{jG(BEpY#LvEhRK
z(<m|J|0V^G8kQrC9af-MTz+%aJYUcF=1rAY=R050_Cm*fr(YPN^6cm_gtbB_`*ofb
z%T}}FhZP<VEe{g)o?2K~_##jEd?}aG<Ne|x2sqohk@HT|eM*RR^qmhG%hNM6Y9MM7
z_x=I&<P|?sQ=|1go$5Ir=e&9SpB%8*o&%>ULE0&I?=G826~S>6WF8Hd=i+mt<-Vj2
zb1q@wPypUY*P|$LiHV$(U*3>0#Br3^{n&2kxyp^od@){%1-0&j<3u%Nk`*vh^WK6Z
z4<0;l{QY$|4k|R128AyP8L*CJ5^Dl76unu$3r7M1bz`hK((%u?y}3OX5npevmxuw)
zng9f^ulxwjyWQ#k%)pH*U#|^)@HXxnA0NLjSOIZ-{exo?svsdPAQPk6rQJlV`-<oi
zE`{@4g8o3j@t3b&<$y@P)l63a)nyG<;3u)^t?=<710e)YhgS)DfAG|)=bjt?0@<Ou
zlZ*{e<=p8$>bW|`48+`uW1O3t<9BFk03FaRb32KQddwjkUtrZM^Qm6;C_>LjWcE*P
z`#k#ICwxypKbKq@LvtDPH<(tp+Sw8M54F;X9{nXlpBc4m(M@{1J%LwJlJ)-U1kV>v
zQ{Mv(TCO)YHybxTIY>}uf)3HwwNea|7Mtp};`Foc{?_`KAgd-P_dxXa(9fR=DGI@q
z)YLVDgIjX>+O*_J)qPIahj3&$xw-s&e3f-|6gD<9_QJ0JNs+XE9Rv)j-RAlKHiG~E
zFws{lMg{Shw!R`eAH;PSq$CtU6b*p*pxy$=n=K!1lfW?z;gJ0p!gd;Bi*n#Nmr^nq
zm9L*)2Uv<7M;0fgmjw1wLr13(tV+UVmUDA+({b{P6941(-M@=%hqs|`pUdp-e67Y&
zU0r?72AHgZp@b9}unNtGV9)&}&bJ8u{kE-+W!Lud)t~!`Z%mZ)*(YLj{}<Ix@7ZUU
z&FIcWnI@;C5MO|JxG~!J)TxFh$YPt?bCCy3NA$8ls5CS*{#ZW?53d1GiO5V#OS8Xo
zw=V?zYjZBt%SbE+oqPz4tWa~`b=)gCx!G!bBMO(7-&<?W(|p}|?B2gccLAoYM0gMd
zHiMpOTb|ytX<4VKOVKnuJUsS^nlv79cSO2AJ|IB{CV>RJE&g!i$B*dx#iJ%9B>Vpo
zvjCvjJN6tf+S=TJ20&1UNPW3!EyS_=(jU1fQ`E1B<~vKE(FU$#j5#?ufLM)<7f@Da
zCX1E@kb8hpz%k<=<8Fdco#~kHmv&#0iV*7$NlW9CS)V<dn3kbmU|IY-dC7a0kN-GT
zSY+gR3k!iV_vP}bW)pCN6L-Eo&b`%+6SZ;HU5AWG)Rn|E=Dkcz^=QZD-?9$o2+^aC
z0q3HMbMo^CqG4Wst;(iXU<s6>lcA*=dkPRuH2+Sw)>3cY?JKQjxjC1vUS*)7qB>zS
z5R0SE_1f|z>iuigxL<=6UV+{-OppAq5S)T}PYySjHpLKn=+z9pNc=QU?+BJv*U_;z
zrK54UCZMUYF%j~_)l7rys}wiHVhF7Ue?On0m1{F71ODZYVgf{a7(^G@CTjUjL{wDt
zTlu}y6`TJA@2xGRYGvpFlBB9UJLR^>cX#s3<*4?;Y({@s$6vIl9REciEhE;53b~v0
zL?R$6Ts>^Qt)t^8G$NF&;FmAC(8maw26(F8Qf#4t!|MqE8(L<>p+9bA@u*LnEusLx
zNPZ&15EZJK24&_0r4lKMp;`(QPzxYe(grJ>#c|m=IGhE!c>erhZtn5RmoK9udp~(X
zn|h!!>1@F4{CqMt*;wi1aFReL*+mtV8sqE;%IMz9=~_q7z--cEUn!?*G+%meQ1$}X
z6U7N71j%Cp{pebmTO>hDp^1`aIZf#qc&?g;g@?P&lr2ZZ{{$M>$+uw45n7mdogj*{
zMgr7>wiDac(a`~|9+`7&-!37&yoX3O$a?^VVhJAeR4CP^)xJ~&<hf6XCCmmc55Qi2
zwkNbhi$SLbo{Pg#K8=W|{giYT=`0_({1+0U1Ca7X9{SAE`JMa4t;vTfwl;(j^_<6!
zJtcr2kUt$AT|{JllKH*^?Ch#gxqdW+jyFC%%)N9Et;>H8o3e+8)P6C`q1Ew(21~JA
zx4#y|qEOhYkT$wGCRA7ses|ns#dc|C5Jiy`A|kOD2PEvt;Ym2OOE-MY&?io>CV)mU
zvpc18VjsG=V#tAmP-+gI^|zh+e)6b6=|k1mKR`XuOX?r9_%xY})CccJ{o2dOScl9f
zJ|Y<xf>78_!`=#sKmpH%a3EmFwGWPgIEt{?*j5PAmDthgUt7oI(#(P13vNurZJHHW
z8I}%M+1<W<8^BP5rGST*x3kRM8QTq@L5>{UZuC9xcIn!5NuE`2l$_7@W5%_V^`RW?
zya^uF&_-|f6&*Y4zl)F`xtDB2_UU9BQ=m*#4Gt!t6=%CITfb6E_?Tnb@?7$+MW9=?
zl9Cbu`4N00(1j3l(Z_CXN8;C8<P&`s)J+aKO+Z88vPC+u$RFWqGD%S79PtgcwuZ2+
zvu`RMI&{eV^Ghm7>VF2xVzA~Y?BD{Ln}FsL0mnMkWG~^w5U$+P(rowDWLC+6zQFS&
z>_;itBpub4Dxl{BI8=Ula|J?R>x+|*06Ce2bO@+Q&^B}~g1B6Kaf0eT6+O=^z!%wh
z=|=HEk8mhKo?9EvxEmD>jba!VPN)$@39@l2XbR4L$Nf;5f0U8Rb+82*TU$+_w~UUD
z3tH)YaOB6MaM_goWg-zSo!y4$NyO5i-ee;#A;e^7XU9t3-3deppXRx!=}wet%f6z^
zx;L)TvipmYnhP>`mT(`l0$>)1eA{=!0irukI8~(b`G7raE8P{Fe$SpU-`B8hG7@<L
zA`ZG^=je#OgS$00H61*1Bp8Ag={IQwCnqtcM-&_!93(HqZB#)ER3hFB9X)w660pgX
zuRHDXVMvA%)xQe<)$3(2rF0N_lMjsyr<Bxz`-r)yUsV9+Xb?S6!EvOeur$_}E4H}#
z`H6>7SO0WL_6jpIbE;F8R{D|7X<uJ*ke!S#LYtuSg!=@f25v!iKJ{c=T%3}M3Rz`$
zcj37;%Y;ktS8jcby==jH%gjum<|}k+z6Xon?_`oj6mS2+BPJFFjZv^ycCMD5&_@C0
zhlYnA_?+$7GEh}jMWKLds|JqAB5FbjBmgt$l5fJSdL&gPitxzj=s8PEp$z@P@+r2u
zx;jW=8L@_fz}OZS<0QL04{xr^b(o)UadF8q?>NN$C(*p|&-Apag2Hx`t^CvalRtlM
zXZUN>kKKe%hLrU6_x~>a3hFwHMrE2&<Rwr+Y;5ckE~VSy!Ba1{@Dnr&BX|rD2}C{D
zT})%5j_4D%5sKgH+TfNdl9l7$;$A${(cFs9%`vwN=a!eX-GV?{hAK(pT}G&&&EoK?
z8X5v1za#1*y?jO*@)(~7BpaSWOER7KoZ7r#k^jQ9B??vUr3X)eL(<qAjnti}jhf{Z
z9;>3rld<xIFJG>L_t&LomMfzV61l;{^D}DZI_>l`(Q%UN*RGj=eaF%e$}x_#22)sB
zaY3^o(Dv)C=q`(saJ_!ycN-jr4i0Z65DrC(Ryv`uxNQ9GUWI-}kldEiuMj&yeHXw{
z?G}GsCLjWk3Q#U~brv)!rTcf}6axRk-^ngle1gcx%<_K%!sRO?mwYA(FQ@CRCKoq%
z5-toR+YjOZg%tu@WlPH=)UtUf1sxS$TcXlzNE3pTz&Nw{of|E3W1C8p&<&8KcW6Lh
z)6n=?PCE$^*#>LS^y@p_Mfgk?OO*G66@&4Q{m!iTSsO@C1aGrz$)ML(9CWQLU|0~y
zAia$G&Q3zKfDoo>EE~?NM#v86Gd{%X(dj`K)9gcI&{v2;o_(_c0KKZe|N1p4VX3Q5
z)G0f*Z@2&T=^(lYfhJgkU-hw>n3>IwJu^I{MD!BH^sevyAN%{`5D?$*y$2c~r=TFE
zWn~Qla6sv)GtQocGEA_=wWX;@r7$!ES~|MqR@xmqc3dfOvO=w_0w4NTXu|>F_Gatz
zW8l9qWoik8)c@fmj7wN{>iHI$fUOUMgU8VScCkrDAc0ae-T<`m#=FQ*Xu7$$;7gT}
z%z+Od4#7bnd_5>$$r>9o&JVeiuBK}RLth{iDijw_PtV2Q-D<0K1iUvl=L1$oB|s3P
zs-8*d_*7t}&&}uK<MZZnihs@ws>zEJx2sW8(TU;Lj7&_tY5fAxIvj|1wplQfZ_5C2
za9Bh{y+;vM2*-RTy^9Vg5A=)>8c^1GPn=-DNq=yj`3N_r7=|gNYJv$>R-VCRi_)b_
znr6kl$a*6qPB;oC5CI_$%|hqX$Thn}a_#!{+a1Y%_J6kTX8rW)(teP_8axlggM(61
zrL`j0Y1#9jYAT*PN5B%;sQVBzCoR6Hd!Dl5-TLw4Kd%tRCSfZheKd7yEF1Xl@n4u|
zXm+=nuxdWt<5{v++B+raW;j;%)0&6gx*+p&=-|5ynuX&GzV3FNZ*5@<4#DY%{jCm!
zDlab|;r^f<Gzsw3z`%eYX?QlNMki`0Yq|PSbui&gP{KgA{UMIPXF_sgv3hyta&X=v
zlae_ps|}m1x+%nB+8CIaj79d@IyotzNYj7w=3Gw=to}t1C}Sm$|L?Kd|35bM|CqyF
z<|Z-KEl<x0G|OETzPw+X*0-VNI>itHFJK^}Dgd`0EnWJ4hkH_i=)s#CYlEWXnT8b~
zA+0}1{zPzE!q~7*Z!E^+mVHPkX>D#!hFbdPd;TL3WbAAWJkJR3#nVoI8iCbXgoF(x
zzY$6UA#Wv`+qD)Bdd9#d>32+Ob>FSH_q{jgw{O?!wzLQ|1px&ZqQ18`=IHegv@9%2
z5YpgIaYB^}-Y*X5!_1{bSaulEm_<z?RLoqnwiUGTRWN%@1e|-_+41HIF9<-IP`O36
zj!uQA^ul=iEuar*>U$3y*u@a}u(h#q2vx!iB^EKo=U40C;IIm=uUqPJi~x&VT<7@r
zfvgAt+ONQeK~xPRg9yTYA6$o63>p}hmH?Bla<cqK((lUh&?lLp(L=<R-B_eYmB*X{
zVBJ|9I01zqO^!YiE>DD%mj?qB=o_&Cr<O3{03QM7VT}`dClQZ8BJhG5vG=fJ(F*1W
zY7a|?_#sMdJAOe5+$8`yFcRlHkc1lZc;7L^_%<YtZl3vGAYYTA$_F4ZamxelYW%Xr
z$br<n*O(ouX}fcBfK<Py{`1+^%!<tyujAc<@Gpdjm0u~L-%6Z?2~q+kjr7OPDwh)S
z0^n}0b-!e4K-7yD&B^B<{ra9SNOH6O@eL>jBm`l>5<X?<ix0T{K7z4`M_m?UHePG{
z5f$7=j_ii0g`vRV;bD|5BhhFOjuGgdgvyP&oeLHi^zRf%J#|_Z5lhvCi!gia=+O|=
z3IW0{&gq-96$9x3b_WQ|kYZwGZHA<q)JoSo*)D%s`bKWgV=g63ryMi?#0e46&fZ==
z%5?0cwY9Z!l>@}Pc(+M@&-FP)P-R;T%@q1W90>J){(K+s<~~_Z55LD>1-OtGX5e*j
zMT%P;P!5T}7-WVJ+5KCET<136Qoi0LYwM#jGBRg69%pAnn7Gc*&$~c2hF8IcJ`boX
zkQtDp*;pUKrjj6Y59>6>G=XuTmz;}lQ%<WQ$GuwMOdr+P#3ab_`f>N}y?PjdK*$58
z6Eh#6(;}eyq6Zg7!_|n{3W%))icOGBK+f#voAetS8#~xyADEO2K?G?rl-n*-V=0E-
zt6N~n0$Hx;Hpz=Nvt8SL?4^^Yq)c3oSAc*LMHOj1|K*BW|CHiE%4#x<zY@Nt^UMH&
zkTKUZxS+}A%y=JuXj_bUCt%{gr70Dli5M{}jh(LPT^69pVO+;tW{v(W&!{2ph(-hk
zhxrq-PbJPagbR;0l&vFlN<sqa<9+2+<Kb#QjM9A0F;yY38@fO=LROt2woW*kK(8qs
zP+FT1|16O6Av~L4^H<i^9DdaACV`QwLY{`1Sr$=WF#0PMo*N}=<nZ2%N=r+1Ac7Hw
zH574d8u_C-`|dS&qw`O9Z6^a}c6MH-J9t%H{eGQsRaaM+^^9s}Hym1+`3r1p=@`2d
z7cbN|_$&&O0iogwjAOIahPM&aO>f;Im`0AnxGF?!<ZS)*-2sA@bfa2WyIw{?>yN^n
zVz~Rx(^__fl>_Z;1hm`)T*ldB7Ys2D2nYZ&A~9`a*2k^;PQeN#rjF2%vBJK&+6$>i
z9z_FIz-lHO3}6xlW?Aa$!=qRsJZE3v%iiw;Ffkz_5Q71yB~?>*<MjuquHk6m4`Y@{
z?n<%|^ofOqWqn+B>joHH@(w=A>L0>~5BosRv6~%|Be)L12PZ#N?r_?JcM*~f5p8p^
zQ_W#z_J+*H;uY8dj@``XWxwn<v~dEavve2ll?{TdizuWS`<5dln7&3?opK2G#3s<f
zo&h3uMVwDhgghykB(1NjvjZ@p>--3gQ`gzK5VAzSf4>Tb0-#gJ%G`TFAo7_R8|PkN
z(HkdNQDj@Jrq;j<{`8kT1E8|U<N{7Ovdcs{*DsZe7=K2nRaI3%-y7&ydvgKq0fb#Y
zv~$?#n7eohnBJV>QQ*WO?ZyDb7zan_cj}Ick0EH_jSoXZLsY+0&eI2whgzF$Fs+K<
zmIB0FD|Okc8-*b`Z3AWuheaR1We#SG66eB~@PLzn7m7pqDi2uJ2yFKeSU@G?O)Vus
zA5rC@cLXV<TrQt}1ujN7z$8LUEtj-?#P;oiKhd{TZsq*s+tR(<N8Fcq=moXjsfT!_
z7f4YeD^8t04RPKZiH^R0;Iz}bS~2Xc<@dZmbRuB12ruehVZxw>4h#rFOg>=_Ovlwq
zON$AQqG2!&{T|g1=4kUW26cvz{Us^wK=jN?VJ^Qj3Ux4;0Ob1qiN5yCzXL`P+9M+M
zETPmPvA9m2WQ5lWhi-OhNg>MA*w~nuMuS$uEhw1Y>X?r$!$<<(zhSAnlMEy<^scxW
z<BbhyVW6KlQf?8EtXdg#lE9!Kc_h!+SgO?Y=%{aa_?19{p%QoE<3m7FzQ0y1f9@PP
zcvf{!k8W{it&xZ=JQ;{aZwxDB00tDJOjQ$PF2QF61IE-w2Z{k`4<8>NCq`1RrNj&!
z%z<ODD<vGZL!8EEQN9oH@lk@rrb;IzB{d(zz$Z+?W(#q3N42|9%^<T@L9e@yCr)kn
zGd>=WkibELeM^Rv);u1LxhUAAP+O89YjC^uK;TbOjue2d38&$Vxw&~@yayin6y|>j
z#@M!XoP2l_^5H=)u9wwI&fPw@e5&pnWJ0(JPe`Dt1R+|5dWHE4u+Z|YP4~oKJMMyw
zG-p|~L>)1GRTv6Q7~1?X)?i32bmHRTfTp_n7LlJ-r0^A5>5Rb&H85!-^4Am7+>zI2
zpbmIT3p=3fI*s}VYX|RctirUoprVq3c1SpaFrxTWOnWw^AV0#wf$)nFw_t-zUJ;Qe
zL~g);go{+-JYxocPYy}~>uV2(%wtSy#@LisnTTG_&^rkb)QG|_!WxVw>IcyWq@x-z
z>J+#c348=-5^rl=9~1O*ejd@H#qMzTZXF~Z@cOZZ(BR-Bxe+?9<I*6;zdol@qZxci
z(GMVyHpX7I=Etv4RK=G}lPa7$M@vsn#_;zLFbz4w5q(DR4s=T3<1<mFkjz!_jD)L#
zwBwYN{Lr_-#KZ(${sNFaGCLKAquibbL`X<c80Led?_Xu4?sgVwhlC+~FKj79OhJbL
zMKd!V?8jqvY1VwbU=SAwohQsck)SujuTdfIKr_3X-i0ATYNaq+dwXMi^_O&Q8r`Ug
z?o19y&^Saw0Cay87aNN>OoT>ot=xkJ7zH{FZgCd#ZsX$($T*^hKZNUlyYm%yNP6c4
zB7l3^xj;uK&$|Bwq91<z1*i^~k*obrJ$BLGGDBRA;7Bo}S$c7%SPI_<Si(PkfnwOs
zzMJ$L;!X6d(HWL{<*=9U>YgPeWD2R=O=sDUpdkhqz|J5*yhIp-GDh|kT5XA;0O|TB
zmHvRx^kZZsSqgyU9Oi4b`&3!$BNqWinrX^GEZxf3)4PJ8(|(kLnmiUe3^v#VFonZ>
z{CF3FiXu^Hp5%OuCe#}uZBeZ;Zz^DS?NOb^&vpMg<0SY%mV@PSa3zQt9%4{SS^2}n
zxNa<p6y}XafR`cp$rS#5F*0+BmR+3DljY)sfj8!EK#q%Jf0$`IwdA}LiaF`@N9WbW
zX!)yGBQ)%$&7^5Fw15XS{acnoet=sV7{rT`mpV<c!h;en+`s}}rl#_f+M?~Fx8oHT
zr{1u*IKn>%4|ruWSPq@O%UU1vd}>m2<Oulsnwpeab><-dlI=c8tyxz<PLt15LjVtH
zGST>`7}f3RzHQq!sETBmf+?MF62l}$QBl#ufrMF|55L)sAVguNd9}RMeHNAxCM$__
zgPRr>7N#*R!3VR}{KOwHSq#jfRAE+*`{)K=s6t8-G3A0bdcyMC6Tsbwo_YuuaIFcW
z4#UWVPXJY(fHfH1!{n75{N{+MI~c#vN;ieba`ladAEE}Fz1hkq!+90VkuX!R3(Gn2
zIQ<-WIb&bs?AY1a8*?r~RmKz$cl@1>WD3IRM;9pitqUsqoqzm_%@~>~;ZTq#Vom99
zZ3|MzptnT|qYnb(qPVDgLwI|3|2_$f{WM5M5)u-`a|4if6g%8P^Mx2i*tgEk&e$ws
ziVpAr1ESawfI?_m$z2wNjjk0ix3t8=$$5r~R4FX&dP#vOX=nL8cXW9F%*;&D2ZUOH
z-~h$}`2Zy%ViR+}NGixYZ}jp7K!nN3$rHOQ2tgUrWJ<>cO!i>ZPCkqaGb~iQcAd-6
z`ho6(DH(w76Bb`6;n(6Xpa%cTUmnZNf_R6NibXw-<mY&+@rKBftSohj9TqWT`E8sS
zSuxb*X&oG_s1>F{CS$BN^2&B-tjcSN=4JJGE_D<cx^BsTn7x^y3UV8**jU;RWuquv
z)0KSq^75H!3iz=+&{09mhzTi!%v*3mm)EjFg~|-tG00Sep%>t5KIa&e`r~G;&rj=7
zXCJDouP4AvyvMv!2N?sq3eBii5zrhmwHi2w7(GNB(u`z<zQd3Y!2AUCjN;y+l9J9s
z8^f{bkR5Fn`SxSYi~!Zb)oL!5fn7%Ib~Fp~^mY@EW{c=GD>>|Fj^97GLKS-Yzt=$i
z*K4#dwGHoS#*$&6Oe2isf8Gxfz^gFc>Jm)?m7B^W#7LKT%@!Gl#hc8PgZE$I<zNz}
M^Q!0G%Nu$BA1F&+SpWb4

diff --git a/android/docs/help/basic/using-the-banner.md b/android/docs/help/basic/using-the-banner.md
index ff637645cf..75862f6df4 100644
--- a/android/docs/help/basic/using-the-banner.md
+++ b/android/docs/help/basic/using-the-banner.md
@@ -10,18 +10,6 @@ Keyman keyboards now always display a banner above the keyboard for one of the f
 * Display a Keyman-themed banner so popups and gestures for the top row of keys are visible
 * Reserved for future functionality
 
-## Controlling the Keyboard Banner Mode
-
-The banner mode can be controlled by going to Keyman Settings --> (select an installed language)
-
-At the bottom of the language settings menu are three controls for the banner:
-
-![](../android_images/disable-suggestions.png)
-
-* Disable suggestions (Display image banner instead)
-* Predictions only (Suggestion banner displays predictions)
-* Predictions with corrections (Suggestion banner displays predictions and corrections)
-
 ## Using the Suggestion Banner
 
 If a [dictionary is installed](installing-dictionaries) and enabled for the active Keyman keyboard, the banner will display suggestions that can be selected.

From 2014f715c14145218dfa364370c67ab3afa48d43 Mon Sep 17 00:00:00 2001
From: "Joshua A. Horton" <joshua_horton@sil.org>
Date: Wed, 15 Jan 2025 12:22:44 +0700
Subject: [PATCH 05/53] fix(android): restores Android help changes previously
 lumped-in with auto-correct disable

---
 .../help/android_images/disable-suggestions.png | Bin 0 -> 15434 bytes
 android/docs/help/basic/using-the-banner.md     |  12 ++++++++++++
 2 files changed, 12 insertions(+)
 create mode 100644 android/docs/help/android_images/disable-suggestions.png

diff --git a/android/docs/help/android_images/disable-suggestions.png b/android/docs/help/android_images/disable-suggestions.png
new file mode 100644
index 0000000000000000000000000000000000000000..a4a21b98cb5d277509ebbb21bf92141ead7d7048
GIT binary patch
literal 15434
zcmZ|02RN5){5MP}Qjr;DR5nE_A|on7A!Qd5l5Dc0%qWyqvPmh)UfFvmNs=uRWy{EV
zKiB<#o_8JZdmqPrcgOX+e%E=P-|-pWD^OikaraKvog^eAyOl1S(;y)sJ%j(xquh@F
zj*b4^i$BO6<dv>c;vWx6lZPZEoFq!;<gU8JPW3x`T=~1%FxRd0Lae`pLb=D(C!%cm
zs3-+#@qHK4%SG0i%!>^aN9@lZxNjgwOU}T!H&Qx+U!6}q@e|Yc&sEbGJD<K(`ZyE%
z!r`=r8O!IHPDPGSqoc7l-uIT*qz4_!IhssFn<%|Gugbp-=qHh*^$j>rp2$fV8AW+U
zj#l8v-uw8K-XRhSxwO1<Bs;y;Z%au1*-q&_+kLR5(77v?OKG{7oAE615ph%MviZ%t
zY`=IerN*3VBWsG0#66n%*zF61)4P-v<Ybud>R$W&`+{I2Z-e=n?NSHgW*wXkHn#bE
zj~^f}`fyN*P9SopQW$%W2m>h}rFXL9EaQu|YmLgcRH|K_cpB(qqC-BKR~kq1|I2(x
ze&IqI7nK?>@1*P6=X3?O_;<bc>&r>a4;)Z`vnf6{n?|0*W$i~#PO7}Lw9AjjiI;n}
z?#;~uAHw7x^mu5meR<1feuevCQ=9&=EY*9lCqvS@CKX>P@@-S6CFfodVxn$%|59{$
z)?m*y)mNV^GM;Rta|b$UrY|^^SUXa^_tVQbdR)qLA9h5Z+id^5UWV4<b}|md&8lR2
z#m3{z1J8ck?%E^O{f;~L``wz(p4Ft~?dSiy_V?fG%}r+YHu~QpBko78?Wdyq^hR&D
z%x~$3ZZe)UC&~Z2%H;Q|t`O5vGt=4DD2_nolBF}f>g_E7cW)fNJ<6Q8E6|GlDs9kd
zIr5~i933GpQ&Ulk&Se^@b&`?X<1G2B?#7=pZ)WRc>1B)=+;pjUNW$Q&T<>(r@O?++
zm+KX_Vuz=a^2d^o#=I%<<dzm@epHuyvHsh4*NQ;m!dfOzv)pK}t+~@Xj=q-q!(%s6
ze{!I&B3wR~<#$5vM;d|udBV%<At7wc!dZv@)z(;t8Q1tZybFD#<zGE(B(IKGxOyl@
zDEubh&|tulQJxOFlLG^j?$3>s9Ys&mh3$^cdeY-~BhSoH$5lL%sOfpUpQc2~lxxgn
zL35F+)oYI*NA2nv1m5g(LH0uwempX#?=H$1PB=T845jB@HenupMp?PW;*Fq8qRaZb
z>8Ym5rSxngE9T1k{j08)s#1m{InPH9>U7|As&Bq?nQ5}f&l#?kXtl5F8+4b|dD=tk
z^}dz+m>46QFYYmX>=rd$T+;os+5$^C&v-l^NbOaR3@WwxF&g~)T6@<s%Br<4Y9by(
z-;6xCk*_C_5dE>Mv~Q<$ec0yN)qhQw(=-{Gxh2CNq~+Z)<&n1(5T(Q+{`bWbj{oky
zW0dxd;QI15sLV!tyXv!@mH+#068g4WCby)+ax0d&Mon(!>$OZ#XfY7W-Y3AePb)q0
z_bTJ*^|FM|3k%k6Hn!&}lGCUT;UPYsZFuax?*85I_2fy{w>NU>Lc9LXS1Dzf-N?+M
ztN9(r+@5=6c>OwuV5hmd*?UcaNDb4EOdgE+*NBXEY3-4(@d$0{75_^;TBpUPEcHt3
z&6E4LkMursk?Pg4RU_5Wu~@mY_V(A^ZU3#Jen9<llxg3IkmbBV&08T0Xa7vlg<83~
z(HW@feaN^T)bp>x{n1Rp!?55OpP%LNYq@)gqxG%GZILb;Fwc(AJ(KjwqVAA>lPE)<
z15NHJ!we?A(~pG*sNX$gPkK(i*SFzG*}xS>=E8t4FHVnV-aUFHWyO=0*lh`YmlJby
zYbxuNDrasA8&+;w+MM@K+P95bo|{vNZuh-)TPIu28TR&eM&}g`QR|QpMpoqz0mk4m
z50zbar}}qm?4>_Mgh&{r{Q_rgXF%G{Yt*wD(!BzfhGGx&m?vHTMYvw;qZXp9X5;tl
zk5$NcWf&|;Q+&SgYl?+PNSy`6pxS@Yp;bCm#HF<BJ)^f)hG%1jKmu!ERn^@o|G^E)
zb2u$^r3+!_j;PuEBkywic8OnGzv2w5+gh^5n}|;Z_ZYP=-yli~yF)>i`kSk1R|Q^)
z?Kz+Bw&?bNRmoVRzy8#^#5&_yBRSe><q4APjq4d&ovp(6pXUFTefKIdUEApHcM7eu
z8CrvRJ9M5_{m-3J+sPzPof1jWy<<`QL3T81BQzm}UijqB^FM~vdDbnb6*GwplrsDC
zH_CE2N#l)rdKdYJ;sxvUkEfJrWVapQ3KZX;|MEl0j!EO1!I(HQ-=GO9nRRDj%IbTX
zk{7NDFz(s2$MGFgtA5+}L|2mW@69O+#`aSRi&Sp|YZA(2d%LH6hsx-P6XJJ2{yO!n
zt>p$!>9--O1MKpYZ|~o-_;OR@0{scr-4;ldEp171m5P9%L({}b>r5QXI<js_`fK!8
zV?_NCTa7m>C&YH1rFj3#{y4TnzU|+|+n)~bDmL^^ZuEx+(*92_l)L3jt$ildks{|k
zVXlz%==4l{$k2x-Tf^R;Q;Ea*+toe^r!j@F2Ge;K3=C$tJz!K4jlSp=Dar56+4o@N
z5#={6?E<|F9U&$Zb+cUERRJNPS`*Qy$t@!_$DjUDD)Am;TC2DskQnz&HdD+}&Pq-A
zgqMH9RQk$t!LaR|h!DFkBjdyWi3GP}m-SQg=jF4qvS?G9YB{v8=sFDxjW6~*pUIAj
zWexmr)LKgW_1^1bJ`o0nb6L553Fs4h?~m;c6bvGg?=hp{)jSvBPX?5FLQFfUsUt;B
zFfrLL{$ms~EWdEhjJ3y%zR=xYouVk6&3tOAz4Glz^KnB)tL(s#5jkxs$|=9WjmeD%
z#GU>`c2f(nc`v#770l%Brig7)9(UA|w3Fz$c5*aa*Zfd}<u;Z+wi7F-B=@G>ef1!Q
zx2@FU_Q=$u&~S(71@|#Oqlt0$+EkJMmZj}>d-ZeKncjrdv!9lhN90`7jmMUF^n<p_
zN=fpZKei9B9ZVL8<)n13HITlp&lkDpm^-Oiwob&XfXw5g!X%GoUb~P)byJtd4GEUd
z1aTqU1*qKR(vIG;s{Aac<MGWhbBB&cpqsmMJi`oW)Sjx(?Tj=ZN36;?{_CDb)Uxh3
zjAu6W)4TKzty|U%j+Du|*BH4=SAI5JlAkei_GH+~P5!SpjhG!w(|FTtd52ma7bi38
zxM`#={tl`rUzDhfxfdxue9(<RD-ZSkj9kwSj?nX_)?_ZS@fOIc>@b?(%v<)D>t-4p
z+&Dx$Vf5#N=4%5>rqrChgI03pF(-<{Nrc#Zm*g``*#|fF$dUvcnaf_@;PmE{da?Of
zBuMm6yR0e8YI}vCoc83gBG+hTrLZSKo8DXxd>6+DAKl|`f3~xH?XTHXe)z0c&~E}S
zMZA3Y#RM_Tc~yBtm6iZg*@yD}Ph^$RTfVE_Tr6B0hh<XRsq`qles6=lb;@3HxfQ`8
zQxcwu932MNhf{&$O3h96miZ%T8dth|1F8-NKkvvPBYkg8lbqqrNy+%OPl;AHO4oUs
z;kc6xZrE97W+kbe=~wtQI&WQS=_Qx_FAdSnqwj^L#3MHEPRc%^W~RWi1oe|dm_{?c
zJ{+BWhdv{nd~VS5?ClW_ouAw4QnWQ(t$WqD%x=wZJUO=OZTBqcNqt)K;ennwa*1%K
zIPoRRl<WCM1&?FTT-iGF#MJos=Tk2Hfxdq(l6=>qHf!i?ryH4SiJ017{Ia`8`eB{%
z@b~;XzZ@kWE8GkeY|PLSjE;_W?z8o&s_ZuOvRbxSyc8q$^u-IehBc2b&oA;gO0Ffm
z;iO#tXiehJ@!dyHBip#rRw_2{T2tGNPoe3zi!G?6yEBFYGnwXEUvV^wGN`w;`Iv45
zB#E=E{`w_XvO~h*=PumR^h?_1#Xmjac0cN$#>Vcdtb4@FBI_yT=jZ4AWp;ht<7ZpU
zt>xmemS^-F9A1iR?pOcx<OWbPA0?UXwW*j}>a~#_WumJ7k$#cHjEN!ZIDh=p-7%ND
zdzn*uT8l2pW(o4MFC?4~zr;lS`)0ey-L>{Jm({Xv1xv28a;O+2y=j$urzQCJclWJ^
z5ViwSE>E5wR-7-a*a$8ym9pr1TleDR9a~$z7okn<?Yt5a(S3b=bZd>x&4;+TDN;Ig
zM(o0y&M7PJbai!YzM0Bw(c0QNIzQH;BeX=l!N|byBsTW3L8i7rcwAiY-}dD@Cs<YT
zc%`M|(zP><MWZ9SNk&IUpS*aXeB;JJ$vD;90XTursVXL>rjh3Q1_ta+O-;(Zf8C@F
z@BL+TU!JZr68V&SORc)5CUR=`b~1i`e(sRygoMy4qpUS{l^H`?->~M^Z4G7voE4wW
zg}%rqUHM@z>UB-IU~R<kXzXFJ!@>ULn~s^f0oRqNfS0Z`^cMD!DTmb#Rr%<adq^mT
zvZpEpvtEo53%2Slu>0A{V&FEx?d{{Ebn)V74UO!}vjKaA49jD&SM>Y#HAV_*uS}QB
zplLBMFo@j#%=K8|Fdq1wZmz%oZk70eYq+KC`YhS>^mI{iakgcTXr5liyvN0hyE<Q8
zN^$!;$;HDXYBM19z0jup;oie5n`<+33kx@U3#^D!ppjjFqM5FhZPG+H`87iY8$?P<
z>M>WxeD2)2*=lNG^+N0G2M!!4vLD-7;xu(gRyLt2ocFL2UHALzH~%e8Nd5hC<(z`T
z{J-Csf-`qJlFuKI@pyh%G2}@^#JM=hyN&Jb#BmGh=J<a5rW5bA>FVw6eL+cSIALp@
z*Jt}~<GGQ#1G1h6l)^4tx|E8o_YDZ3XJKi&lUarxI(gyZMH5_+XVEqJev=pdQZt%;
zD7aM0B=K$fX_7fzmN6}g$Sn_c&QD!zxt825>bAdq-d^?f^`&KI#u;D!=x}eLP5d$I
z$)J;WzGm4D%d5sqDPf5{Jw5LXE8@G-wL4!b(R*(E)lN)XULFqM$no58BMt*cud<k>
zaplVD$RpuzMRtc}H&+e|L<UXu7Ctj?aUQ(aQ2BsNH`i?UBWB^BqfOx?NaoZXIyyQR
z5KX$ex*oGtJJzu$9?SjCi<4cC5aTAgKc@-@%{JFpJ+?NMOXr&fU%q~QxJ5L<^B;eP
zUj9hQO!>{J@A<7U;-S^m@>-b&Z7Xv>h_9Hb*plH@i#Ne;FaGNAVK(%58ZYgRux@N?
zJ0E3gaPP0C*~i!>R21D3rz6L2edH1peB|xD?d{b!)gSKu{+4xvOm=JiI8S&W)_eR$
zHM!{)uWD@3Sj8jGt74{Z_v2O1m!(*ykdk;YcgIXuE|q>XKeMebE>FCFpx#7u_}4F;
zk?g&P6=r|N*wC@DwcRHrdoJZF8WIw+IMEr<lY8st304WGKl=_IJlK3^8yVSCuE6)#
zOXjO~a70j6A`%iiCsrC{H%Tjt9mXlCsBZlFl+-MS2p>y*W^hPGCZ3dRyNK2IJp10*
z7cYV`GlezZUX=^udaIrJzVum9(J9HhlfGO^<U4i<W=hx&k+25)<A*3D$N=oQLdb!p
z*68CBuJc@E+jkj_)CO{gXs(APqnLe+l{mXR-H*p%J!vsCKCY;tK}UixIG3t1{PX84
zAw3C;FMOMSGrb~9yyR&AkQ+TL6b`XCsrdfoyh)kD;yc}EdhrHa6FDuVMa5o>dfHUm
zI7gHPJdeIF+;x#p@--8pbZ~2PnJ%=@vL_6AyqBK7=2pVJdJeDkD&!`yM=j5UdAYcJ
zvGDKxubWJ&o0{I7vrX%=K&h)gJyV(@^>b`YbL1{jnURgPb#>W}6B<j?{ViUb3kh_+
z&)dvM4zRHui)k}8i`V=Qg*x%tf3Y}8&#U?pA0laum)>t<V?!kN(p0a6`;yQjrXz<$
zMZ>n%Dz<Xh1}yzjtSU}mvlLSH1}l)=&&_p+?#{c@t=h9SQuiqE@ne&vKRr&%!jzGk
z(Fbks(x_pTBP9;6n}ix0I+a?))nqPR3Jt63_$GYcW`}IBUg{y;XLV7&r#|fHuGiOJ
z3fLR8ZE~_J{aTUjgKh)&V6(Ot9i^@dbiEg2Pd&Z+`)fy{9NE9WzZK=>y+u2Qe*J1h
zL6h)U$r3$5@h$u24y1|D^$!s;_x>SJtouLMZLYh1{``4%Vd3zhL*99Ld2|gPo2%ox
z#rE8YtzG*>?$_3yLn*SgvpXmt5VW}HSYX}1^9*tayq@S(t*sY5*QUftfV8FE7L8Wt
z$4qYB%C+hhGu^83-~D4e!K=Q%pN*H7cdWQ0MRB1;xZ+ch-A%XU=}X4Ohr+lJ=@;DG
zB)9v>^#p{6)06GkGrG_&i^~JpPCE?^4V@`-`}@c9YmU%yw?%W$&DC41;?@`2<E51@
zT*xZP;Jj+)WaGD{9WcGCC(GY!y1(l4=Gog<@+J5Fc%99CFyxM_n}w@eXNBs$KDIcK
z=hUvZmzj39`jrm6f7I`Tx;;ELRu{p4Wp;Kp@m71h)I#fV1WIgN9M$gK=fk*UR))xa
zwno>yx)g&ej0Q?wSN{E*|9E<4Hq>jKOjcGFW!?`Djr7RM5(#0KZVuyCq1?ICxXPP?
z<izdI<Z5ba9{27M-G`b<C^0MR3D2b$BIchG>8z+}XvPu9;y>zxYueg^R~N>A3=LJH
z8nvviF4)=GC0$NYaB+7}>9Qbd;d5zsG1lPihJ#y|mWWa%KH%b0u`y+H!s`1~-M>pc
zw-UI;#h)P$n6Pm?s<99CGJd9n${0`g6|c|M(73p|&dqpj?Z1C3-qqiEz%>A=FoN7f
zD~Q-=k3Jp%XfixC)fU7k_{kz4aD@!<otReG^Wp9t1U_=>n8VN38yo-raxa0NZG9`F
zuQkq}Z!hcB^;j1Pe^F@cxxDFe>UK*XRoLYx?Lb^T2RK^Rr#D=R@4j{~XZxt-k?Qi4
zf$u{^t(M}DQ$a<zRkQoR*Umh)D^aF<*rhI{Yh`>gKhYX@npsg%almz)jp!Ka>a+s`
z1D5?CA{hm>W)U}hYq{N<F=g8BR%LO^Zo<9P@}@Tnccf{(p`7{7oG!G`6L|QKs<M%B
zZj|F&1A1ZJTc4@tzs1-;&1{e5Y|*Lw^ReuC$J?8J>ra%zObT@t+IF6Lw%j*mePv)F
zwT3B(M*~1CKsmL#t1A*<jO=_j|Hl_6se=`Ks*HwygvuUukePQ1br+ubYf!-K+x+vD
zpuE0m^rg9Dt+J(|^xc^eMF$kPl^l1yt@@){_KWTswI=uE4PXbO6RdNyvwVBQ&<fCy
z`uh7LsY@v2TpatqhOKDn@*uf;7WbR5?1TiFq<e<(ZL#asw?p*g!*ho$yeXk`26E5+
zU!o`Y2YJ$Gg7wO$a)UqkDy#h8R8nC1>vEah;ZL8+d3wscfB$}Eb#-{s?RRP>xg0IK
zrugnNx>3b0bB9p(4$H_GylH^ash1Ijrhm+K=*(NK3<2u`X7XX-*L|nuz*0{;Pn#_K
zY%BWkfsU0`8Hq>~0R$R)fohzjU!C!dmDyp(>As%n5mHcq(6{!~DK@<Y!AJB9K9#yU
z#C!g`EhR0jvN&&|%L7=5Mi~D5dE@bRsk1;Q9OgfO|EK->kLt_IZ__WdPIX=QxxO*u
z)zH$C-b$+-#<jk&;R;Cf=F00m1kwWb$#tjosXF7~Q92(PA8%l+Iet17eQwByCQC<1
zIZ)Z1;`3Co;{;t#kLj~zuYJkM$vTDB{WaG9P>!9*Y=3{vz*0or7Juo#=Y8E};rcWB
z))TJ4xYN$x`+0bH{D}q%$R~R1;{pHO`|J1Hl-f1aeX^QgSReoags|%+PT4nV{Ot=*
z#<U6PWZjRAJ&>3--}iuQ$Mj?o<5aaD^_Zs_*uIx3kDR@|{e=q`5*@}ry12Owd{qT5
zkVmoy1_mZ2CMI5d%B~yrMl+oWk-shIYv=F53fUbr?4yCa@n<r$940<<TYO1-5EHYX
zf`X#9wwCt5fyWgUvSC~YTSUdJ`-XtQS5{W|*4RaiNVL*5dx+f~9W|Nk&I~OnIn69=
zVA7Lg>Nxd1^3x}El$@7eD56ZG?{vSnTb(y$6|;Ppo-P>1W%qYdv$U)Xg|xD%DG-=N
zU`<U`mFM{JaKuAWmqkk~lSuT%*wfB?Pz@};z6<118vXgx_ylX@TiZ)7PVmpMjQ#u>
z_UxJBnKL9|Tu-B;aY3`jGS<%leAAt2_!{!WxrwIU%b5n!*p5%X|BhMTOb;La^XFpQ
zi<8Bk8<J?8$vFAeSc&k62!@aG(ke3pWdxq`*+FB5$Rn{tG@(_^c4-?Pv-*C*w6Vl~
zOj%}il-@+wtpCH^UHioz+f-~s8I-y0-MhC6<f7^0M>DM5#Kc6zeaVvO7q}8&_A$$G
zf1C1E|LOh`AYH1;BD){^j+^aD@Y<9BS~95%Vr<N5?JIVOmboV>sQvyWHVX*9xOPoN
zSveW4;hj#lH-Q1c6t7RDF8%Hf)i!YT3zgk`4pb;`d{X^{&`js+P>p-j#pCfmes!cc
zFRoI1+m1Ick}S;6lT5F$%4%zCo9Mnutn|F$7^m7WQ0D&BG#c<6QR)I{8W<d$YFOdf
znjp);_jb3G^Yk^MMP9ph0I-8gNa*qGtZn)Fyvp|iD-}n_6E_<|j%0g<H@##yLbGSj
zPrzcLP9lM_@2xovxX(x<m)}}?;e?3~?O~JjMJxg6o1&fl`Ib%gsID6LnFZlfv9)=c
z*adtIk?ZfS6^9{WX3>MgqoQ7Y*^a=Wpk^Wn5Pt02WRk3%X>jiRdD5YonRcKM(qG@Q
z4<q2Ne%Gk*ytlr-e#&$G)b9OaT)>zg<4zBDjH6-13DVy$7;0&2D@$QR%m|d7b{9Q<
zya#9`jB76~Z6yjX)lPx8{FWslXan~D7H@+I=oZ-?M9$^qop6+70~yRS@7Sgk2Am?8
zyY=p!5a^hkoE#+2XGgF9>MPFuoGx(W;K2t&Lq@2u50WK60IH(;CPM!lP74UwvAVj7
z);{2|aOst5oHAmE<)p>cz?}FTA&1E?{01I#=ed*s6$8brzT4sfP`9WkDb<hC+uPak
zUtZEk)i?&=DrVhx0elR{yW{B5qyPJe$&DffH1XiEvR(#MvHCbxu=k;fiNJt>fPO2B
z?)PnLGZi+=66kFtsTY4DN~!kj*}kd$t)ZufrOY5SJUqOg#qicGglfI{iII^Jc@%G?
zxw`z<ud4fLXlU#m9G;r8D208Ra5O)`3ZNCn#ecrR3KbeK%y_W;9zg8ee2XwR0n0IC
z-_NSR9j5+0PBNdJ*t-CKFYXW8FXM6Q#*G_Xyu8Qn{d2UtQOqetV)~KvLrDoe9bNU@
zkA{v3$HM$@Fn2n7`m-ohkyQRA!+<1MYqYox`^3ZqPJOP4H-Ycu{jG(BEpY#LvEhRK
z(<m|J|0V^G8kQrC9af-MTz+%aJYUcF=1rAY=R050_Cm*fr(YPN^6cm_gtbB_`*ofb
z%T}}FhZP<VEe{g)o?2K~_##jEd?}aG<Ne|x2sqohk@HT|eM*RR^qmhG%hNM6Y9MM7
z_x=I&<P|?sQ=|1go$5Ir=e&9SpB%8*o&%>ULE0&I?=G826~S>6WF8Hd=i+mt<-Vj2
zb1q@wPypUY*P|$LiHV$(U*3>0#Br3^{n&2kxyp^od@){%1-0&j<3u%Nk`*vh^WK6Z
z4<0;l{QY$|4k|R128AyP8L*CJ5^Dl76unu$3r7M1bz`hK((%u?y}3OX5npevmxuw)
zng9f^ulxwjyWQ#k%)pH*U#|^)@HXxnA0NLjSOIZ-{exo?svsdPAQPk6rQJlV`-<oi
zE`{@4g8o3j@t3b&<$y@P)l63a)nyG<;3u)^t?=<710e)YhgS)DfAG|)=bjt?0@<Ou
zlZ*{e<=p8$>bW|`48+`uW1O3t<9BFk03FaRb32KQddwjkUtrZM^Qm6;C_>LjWcE*P
z`#k#ICwxypKbKq@LvtDPH<(tp+Sw8M54F;X9{nXlpBc4m(M@{1J%LwJlJ)-U1kV>v
zQ{Mv(TCO)YHybxTIY>}uf)3HwwNea|7Mtp};`Foc{?_`KAgd-P_dxXa(9fR=DGI@q
z)YLVDgIjX>+O*_J)qPIahj3&$xw-s&e3f-|6gD<9_QJ0JNs+XE9Rv)j-RAlKHiG~E
zFws{lMg{Shw!R`eAH;PSq$CtU6b*p*pxy$=n=K!1lfW?z;gJ0p!gd;Bi*n#Nmr^nq
zm9L*)2Uv<7M;0fgmjw1wLr13(tV+UVmUDA+({b{P6941(-M@=%hqs|`pUdp-e67Y&
zU0r?72AHgZp@b9}unNtGV9)&}&bJ8u{kE-+W!Lud)t~!`Z%mZ)*(YLj{}<Ix@7ZUU
z&FIcWnI@;C5MO|JxG~!J)TxFh$YPt?bCCy3NA$8ls5CS*{#ZW?53d1GiO5V#OS8Xo
zw=V?zYjZBt%SbE+oqPz4tWa~`b=)gCx!G!bBMO(7-&<?W(|p}|?B2gccLAoYM0gMd
zHiMpOTb|ytX<4VKOVKnuJUsS^nlv79cSO2AJ|IB{CV>RJE&g!i$B*dx#iJ%9B>Vpo
zvjCvjJN6tf+S=TJ20&1UNPW3!EyS_=(jU1fQ`E1B<~vKE(FU$#j5#?ufLM)<7f@Da
zCX1E@kb8hpz%k<=<8Fdco#~kHmv&#0iV*7$NlW9CS)V<dn3kbmU|IY-dC7a0kN-GT
zSY+gR3k!iV_vP}bW)pCN6L-Eo&b`%+6SZ;HU5AWG)Rn|E=Dkcz^=QZD-?9$o2+^aC
z0q3HMbMo^CqG4Wst;(iXU<s6>lcA*=dkPRuH2+Sw)>3cY?JKQjxjC1vUS*)7qB>zS
z5R0SE_1f|z>iuigxL<=6UV+{-OppAq5S)T}PYySjHpLKn=+z9pNc=QU?+BJv*U_;z
zrK54UCZMUYF%j~_)l7rys}wiHVhF7Ue?On0m1{F71ODZYVgf{a7(^G@CTjUjL{wDt
zTlu}y6`TJA@2xGRYGvpFlBB9UJLR^>cX#s3<*4?;Y({@s$6vIl9REciEhE;53b~v0
zL?R$6Ts>^Qt)t^8G$NF&;FmAC(8maw26(F8Qf#4t!|MqE8(L<>p+9bA@u*LnEusLx
zNPZ&15EZJK24&_0r4lKMp;`(QPzxYe(grJ>#c|m=IGhE!c>erhZtn5RmoK9udp~(X
zn|h!!>1@F4{CqMt*;wi1aFReL*+mtV8sqE;%IMz9=~_q7z--cEUn!?*G+%meQ1$}X
z6U7N71j%Cp{pebmTO>hDp^1`aIZf#qc&?g;g@?P&lr2ZZ{{$M>$+uw45n7mdogj*{
zMgr7>wiDac(a`~|9+`7&-!37&yoX3O$a?^VVhJAeR4CP^)xJ~&<hf6XCCmmc55Qi2
zwkNbhi$SLbo{Pg#K8=W|{giYT=`0_({1+0U1Ca7X9{SAE`JMa4t;vTfwl;(j^_<6!
zJtcr2kUt$AT|{JllKH*^?Ch#gxqdW+jyFC%%)N9Et;>H8o3e+8)P6C`q1Ew(21~JA
zx4#y|qEOhYkT$wGCRA7ses|ns#dc|C5Jiy`A|kOD2PEvt;Ym2OOE-MY&?io>CV)mU
zvpc18VjsG=V#tAmP-+gI^|zh+e)6b6=|k1mKR`XuOX?r9_%xY})CccJ{o2dOScl9f
zJ|Y<xf>78_!`=#sKmpH%a3EmFwGWPgIEt{?*j5PAmDthgUt7oI(#(P13vNurZJHHW
z8I}%M+1<W<8^BP5rGST*x3kRM8QTq@L5>{UZuC9xcIn!5NuE`2l$_7@W5%_V^`RW?
zya^uF&_-|f6&*Y4zl)F`xtDB2_UU9BQ=m*#4Gt!t6=%CITfb6E_?Tnb@?7$+MW9=?
zl9Cbu`4N00(1j3l(Z_CXN8;C8<P&`s)J+aKO+Z88vPC+u$RFWqGD%S79PtgcwuZ2+
zvu`RMI&{eV^Ghm7>VF2xVzA~Y?BD{Ln}FsL0mnMkWG~^w5U$+P(rowDWLC+6zQFS&
z>_;itBpub4Dxl{BI8=Ula|J?R>x+|*06Ce2bO@+Q&^B}~g1B6Kaf0eT6+O=^z!%wh
z=|=HEk8mhKo?9EvxEmD>jba!VPN)$@39@l2XbR4L$Nf;5f0U8Rb+82*TU$+_w~UUD
z3tH)YaOB6MaM_goWg-zSo!y4$NyO5i-ee;#A;e^7XU9t3-3deppXRx!=}wet%f6z^
zx;L)TvipmYnhP>`mT(`l0$>)1eA{=!0irukI8~(b`G7raE8P{Fe$SpU-`B8hG7@<L
zA`ZG^=je#OgS$00H61*1Bp8Ag={IQwCnqtcM-&_!93(HqZB#)ER3hFB9X)w660pgX
zuRHDXVMvA%)xQe<)$3(2rF0N_lMjsyr<Bxz`-r)yUsV9+Xb?S6!EvOeur$_}E4H}#
z`H6>7SO0WL_6jpIbE;F8R{D|7X<uJ*ke!S#LYtuSg!=@f25v!iKJ{c=T%3}M3Rz`$
zcj37;%Y;ktS8jcby==jH%gjum<|}k+z6Xon?_`oj6mS2+BPJFFjZv^ycCMD5&_@C0
zhlYnA_?+$7GEh}jMWKLds|JqAB5FbjBmgt$l5fJSdL&gPitxzj=s8PEp$z@P@+r2u
zx;jW=8L@_fz}OZS<0QL04{xr^b(o)UadF8q?>NN$C(*p|&-Apag2Hx`t^CvalRtlM
zXZUN>kKKe%hLrU6_x~>a3hFwHMrE2&<Rwr+Y;5ckE~VSy!Ba1{@Dnr&BX|rD2}C{D
zT})%5j_4D%5sKgH+TfNdl9l7$;$A${(cFs9%`vwN=a!eX-GV?{hAK(pT}G&&&EoK?
z8X5v1za#1*y?jO*@)(~7BpaSWOER7KoZ7r#k^jQ9B??vUr3X)eL(<qAjnti}jhf{Z
z9;>3rld<xIFJG>L_t&LomMfzV61l;{^D}DZI_>l`(Q%UN*RGj=eaF%e$}x_#22)sB
zaY3^o(Dv)C=q`(saJ_!ycN-jr4i0Z65DrC(Ryv`uxNQ9GUWI-}kldEiuMj&yeHXw{
z?G}GsCLjWk3Q#U~brv)!rTcf}6axRk-^ngle1gcx%<_K%!sRO?mwYA(FQ@CRCKoq%
z5-toR+YjOZg%tu@WlPH=)UtUf1sxS$TcXlzNE3pTz&Nw{of|E3W1C8p&<&8KcW6Lh
z)6n=?PCE$^*#>LS^y@p_Mfgk?OO*G66@&4Q{m!iTSsO@C1aGrz$)ML(9CWQLU|0~y
zAia$G&Q3zKfDoo>EE~?NM#v86Gd{%X(dj`K)9gcI&{v2;o_(_c0KKZe|N1p4VX3Q5
z)G0f*Z@2&T=^(lYfhJgkU-hw>n3>IwJu^I{MD!BH^sevyAN%{`5D?$*y$2c~r=TFE
zWn~Qla6sv)GtQocGEA_=wWX;@r7$!ES~|MqR@xmqc3dfOvO=w_0w4NTXu|>F_Gatz
zW8l9qWoik8)c@fmj7wN{>iHI$fUOUMgU8VScCkrDAc0ae-T<`m#=FQ*Xu7$$;7gT}
z%z+Od4#7bnd_5>$$r>9o&JVeiuBK}RLth{iDijw_PtV2Q-D<0K1iUvl=L1$oB|s3P
zs-8*d_*7t}&&}uK<MZZnihs@ws>zEJx2sW8(TU;Lj7&_tY5fAxIvj|1wplQfZ_5C2
za9Bh{y+;vM2*-RTy^9Vg5A=)>8c^1GPn=-DNq=yj`3N_r7=|gNYJv$>R-VCRi_)b_
znr6kl$a*6qPB;oC5CI_$%|hqX$Thn}a_#!{+a1Y%_J6kTX8rW)(teP_8axlggM(61
zrL`j0Y1#9jYAT*PN5B%;sQVBzCoR6Hd!Dl5-TLw4Kd%tRCSfZheKd7yEF1Xl@n4u|
zXm+=nuxdWt<5{v++B+raW;j;%)0&6gx*+p&=-|5ynuX&GzV3FNZ*5@<4#DY%{jCm!
zDlab|;r^f<Gzsw3z`%eYX?QlNMki`0Yq|PSbui&gP{KgA{UMIPXF_sgv3hyta&X=v
zlae_ps|}m1x+%nB+8CIaj79d@IyotzNYj7w=3Gw=to}t1C}Sm$|L?Kd|35bM|CqyF
z<|Z-KEl<x0G|OETzPw+X*0-VNI>itHFJK^}Dgd`0EnWJ4hkH_i=)s#CYlEWXnT8b~
zA+0}1{zPzE!q~7*Z!E^+mVHPkX>D#!hFbdPd;TL3WbAAWJkJR3#nVoI8iCbXgoF(x
zzY$6UA#Wv`+qD)Bdd9#d>32+Ob>FSH_q{jgw{O?!wzLQ|1px&ZqQ18`=IHegv@9%2
z5YpgIaYB^}-Y*X5!_1{bSaulEm_<z?RLoqnwiUGTRWN%@1e|-_+41HIF9<-IP`O36
zj!uQA^ul=iEuar*>U$3y*u@a}u(h#q2vx!iB^EKo=U40C;IIm=uUqPJi~x&VT<7@r
zfvgAt+ONQeK~xPRg9yTYA6$o63>p}hmH?Bla<cqK((lUh&?lLp(L=<R-B_eYmB*X{
zVBJ|9I01zqO^!YiE>DD%mj?qB=o_&Cr<O3{03QM7VT}`dClQZ8BJhG5vG=fJ(F*1W
zY7a|?_#sMdJAOe5+$8`yFcRlHkc1lZc;7L^_%<YtZl3vGAYYTA$_F4ZamxelYW%Xr
z$br<n*O(ouX}fcBfK<Py{`1+^%!<tyujAc<@Gpdjm0u~L-%6Z?2~q+kjr7OPDwh)S
z0^n}0b-!e4K-7yD&B^B<{ra9SNOH6O@eL>jBm`l>5<X?<ix0T{K7z4`M_m?UHePG{
z5f$7=j_ii0g`vRV;bD|5BhhFOjuGgdgvyP&oeLHi^zRf%J#|_Z5lhvCi!gia=+O|=
z3IW0{&gq-96$9x3b_WQ|kYZwGZHA<q)JoSo*)D%s`bKWgV=g63ryMi?#0e46&fZ==
z%5?0cwY9Z!l>@}Pc(+M@&-FP)P-R;T%@q1W90>J){(K+s<~~_Z55LD>1-OtGX5e*j
zMT%P;P!5T}7-WVJ+5KCET<136Qoi0LYwM#jGBRg69%pAnn7Gc*&$~c2hF8IcJ`boX
zkQtDp*;pUKrjj6Y59>6>G=XuTmz;}lQ%<WQ$GuwMOdr+P#3ab_`f>N}y?PjdK*$58
z6Eh#6(;}eyq6Zg7!_|n{3W%))icOGBK+f#voAetS8#~xyADEO2K?G?rl-n*-V=0E-
zt6N~n0$Hx;Hpz=Nvt8SL?4^^Yq)c3oSAc*LMHOj1|K*BW|CHiE%4#x<zY@Nt^UMH&
zkTKUZxS+}A%y=JuXj_bUCt%{gr70Dli5M{}jh(LPT^69pVO+;tW{v(W&!{2ph(-hk
zhxrq-PbJPagbR;0l&vFlN<sqa<9+2+<Kb#QjM9A0F;yY38@fO=LROt2woW*kK(8qs
zP+FT1|16O6Av~L4^H<i^9DdaACV`QwLY{`1Sr$=WF#0PMo*N}=<nZ2%N=r+1Ac7Hw
zH574d8u_C-`|dS&qw`O9Z6^a}c6MH-J9t%H{eGQsRaaM+^^9s}Hym1+`3r1p=@`2d
z7cbN|_$&&O0iogwjAOIahPM&aO>f;Im`0AnxGF?!<ZS)*-2sA@bfa2WyIw{?>yN^n
zVz~Rx(^__fl>_Z;1hm`)T*ldB7Ys2D2nYZ&A~9`a*2k^;PQeN#rjF2%vBJK&+6$>i
z9z_FIz-lHO3}6xlW?Aa$!=qRsJZE3v%iiw;Ffkz_5Q71yB~?>*<MjuquHk6m4`Y@{
z?n<%|^ofOqWqn+B>joHH@(w=A>L0>~5BosRv6~%|Be)L12PZ#N?r_?JcM*~f5p8p^
zQ_W#z_J+*H;uY8dj@``XWxwn<v~dEavve2ll?{TdizuWS`<5dln7&3?opK2G#3s<f
zo&h3uMVwDhgghykB(1NjvjZ@p>--3gQ`gzK5VAzSf4>Tb0-#gJ%G`TFAo7_R8|PkN
z(HkdNQDj@Jrq;j<{`8kT1E8|U<N{7Ovdcs{*DsZe7=K2nRaI3%-y7&ydvgKq0fb#Y
zv~$?#n7eohnBJV>QQ*WO?ZyDb7zan_cj}Ick0EH_jSoXZLsY+0&eI2whgzF$Fs+K<
zmIB0FD|Okc8-*b`Z3AWuheaR1We#SG66eB~@PLzn7m7pqDi2uJ2yFKeSU@G?O)Vus
zA5rC@cLXV<TrQt}1ujN7z$8LUEtj-?#P;oiKhd{TZsq*s+tR(<N8Fcq=moXjsfT!_
z7f4YeD^8t04RPKZiH^R0;Iz}bS~2Xc<@dZmbRuB12ruehVZxw>4h#rFOg>=_Ovlwq
zON$AQqG2!&{T|g1=4kUW26cvz{Us^wK=jN?VJ^Qj3Ux4;0Ob1qiN5yCzXL`P+9M+M
zETPmPvA9m2WQ5lWhi-OhNg>MA*w~nuMuS$uEhw1Y>X?r$!$<<(zhSAnlMEy<^scxW
z<BbhyVW6KlQf?8EtXdg#lE9!Kc_h!+SgO?Y=%{aa_?19{p%QoE<3m7FzQ0y1f9@PP
zcvf{!k8W{it&xZ=JQ;{aZwxDB00tDJOjQ$PF2QF61IE-w2Z{k`4<8>NCq`1RrNj&!
z%z<ODD<vGZL!8EEQN9oH@lk@rrb;IzB{d(zz$Z+?W(#q3N42|9%^<T@L9e@yCr)kn
zGd>=WkibELeM^Rv);u1LxhUAAP+O89YjC^uK;TbOjue2d38&$Vxw&~@yayin6y|>j
z#@M!XoP2l_^5H=)u9wwI&fPw@e5&pnWJ0(JPe`Dt1R+|5dWHE4u+Z|YP4~oKJMMyw
zG-p|~L>)1GRTv6Q7~1?X)?i32bmHRTfTp_n7LlJ-r0^A5>5Rb&H85!-^4Am7+>zI2
zpbmIT3p=3fI*s}VYX|RctirUoprVq3c1SpaFrxTWOnWw^AV0#wf$)nFw_t-zUJ;Qe
zL~g);go{+-JYxocPYy}~>uV2(%wtSy#@LisnTTG_&^rkb)QG|_!WxVw>IcyWq@x-z
z>J+#c348=-5^rl=9~1O*ejd@H#qMzTZXF~Z@cOZZ(BR-Bxe+?9<I*6;zdol@qZxci
z(GMVyHpX7I=Etv4RK=G}lPa7$M@vsn#_;zLFbz4w5q(DR4s=T3<1<mFkjz!_jD)L#
zwBwYN{Lr_-#KZ(${sNFaGCLKAquibbL`X<c80Led?_Xu4?sgVwhlC+~FKj79OhJbL
zMKd!V?8jqvY1VwbU=SAwohQsck)SujuTdfIKr_3X-i0ATYNaq+dwXMi^_O&Q8r`Ug
z?o19y&^Saw0Cay87aNN>OoT>ot=xkJ7zH{FZgCd#ZsX$($T*^hKZNUlyYm%yNP6c4
zB7l3^xj;uK&$|Bwq91<z1*i^~k*obrJ$BLGGDBRA;7Bo}S$c7%SPI_<Si(PkfnwOs
zzMJ$L;!X6d(HWL{<*=9U>YgPeWD2R=O=sDUpdkhqz|J5*yhIp-GDh|kT5XA;0O|TB
zmHvRx^kZZsSqgyU9Oi4b`&3!$BNqWinrX^GEZxf3)4PJ8(|(kLnmiUe3^v#VFonZ>
z{CF3FiXu^Hp5%OuCe#}uZBeZ;Zz^DS?NOb^&vpMg<0SY%mV@PSa3zQt9%4{SS^2}n
zxNa<p6y}XafR`cp$rS#5F*0+BmR+3DljY)sfj8!EK#q%Jf0$`IwdA}LiaF`@N9WbW
zX!)yGBQ)%$&7^5Fw15XS{acnoet=sV7{rT`mpV<c!h;en+`s}}rl#_f+M?~Fx8oHT
zr{1u*IKn>%4|ruWSPq@O%UU1vd}>m2<Oulsnwpeab><-dlI=c8tyxz<PLt15LjVtH
zGST>`7}f3RzHQq!sETBmf+?MF62l}$QBl#ufrMF|55L)sAVguNd9}RMeHNAxCM$__
zgPRr>7N#*R!3VR}{KOwHSq#jfRAE+*`{)K=s6t8-G3A0bdcyMC6Tsbwo_YuuaIFcW
z4#UWVPXJY(fHfH1!{n75{N{+MI~c#vN;ieba`ladAEE}Fz1hkq!+90VkuX!R3(Gn2
zIQ<-WIb&bs?AY1a8*?r~RmKz$cl@1>WD3IRM;9pitqUsqoqzm_%@~>~;ZTq#Vom99
zZ3|MzptnT|qYnb(qPVDgLwI|3|2_$f{WM5M5)u-`a|4if6g%8P^Mx2i*tgEk&e$ws
ziVpAr1ESawfI?_m$z2wNjjk0ix3t8=$$5r~R4FX&dP#vOX=nL8cXW9F%*;&D2ZUOH
z-~h$}`2Zy%ViR+}NGixYZ}jp7K!nN3$rHOQ2tgUrWJ<>cO!i>ZPCkqaGb~iQcAd-6
z`ho6(DH(w76Bb`6;n(6Xpa%cTUmnZNf_R6NibXw-<mY&+@rKBftSohj9TqWT`E8sS
zSuxb*X&oG_s1>F{CS$BN^2&B-tjcSN=4JJGE_D<cx^BsTn7x^y3UV8**jU;RWuquv
z)0KSq^75H!3iz=+&{09mhzTi!%v*3mm)EjFg~|-tG00Sep%>t5KIa&e`r~G;&rj=7
zXCJDouP4AvyvMv!2N?sq3eBii5zrhmwHi2w7(GNB(u`z<zQd3Y!2AUCjN;y+l9J9s
z8^f{bkR5Fn`SxSYi~!Zb)oL!5fn7%Ib~Fp~^mY@EW{c=GD>>|Fj^97GLKS-Yzt=$i
z*K4#dwGHoS#*$&6Oe2isf8Gxfz^gFc>Jm)?m7B^W#7LKT%@!Gl#hc8PgZE$I<zNz}
M^Q!0G%Nu$BA1F&+SpWb4

literal 0
HcmV?d00001

diff --git a/android/docs/help/basic/using-the-banner.md b/android/docs/help/basic/using-the-banner.md
index 75862f6df4..ff637645cf 100644
--- a/android/docs/help/basic/using-the-banner.md
+++ b/android/docs/help/basic/using-the-banner.md
@@ -10,6 +10,18 @@ Keyman keyboards now always display a banner above the keyboard for one of the f
 * Display a Keyman-themed banner so popups and gestures for the top row of keys are visible
 * Reserved for future functionality
 
+## Controlling the Keyboard Banner Mode
+
+The banner mode can be controlled by going to Keyman Settings --> (select an installed language)
+
+At the bottom of the language settings menu are three controls for the banner:
+
+![](../android_images/disable-suggestions.png)
+
+* Disable suggestions (Display image banner instead)
+* Predictions only (Suggestion banner displays predictions)
+* Predictions with corrections (Suggestion banner displays predictions and corrections)
+
 ## Using the Suggestion Banner
 
 If a [dictionary is installed](installing-dictionaries) and enabled for the active Keyman keyboard, the banner will display suggestions that can be selected.

From bcca646dedcb76e830a07924707c0129a1346928 Mon Sep 17 00:00:00 2001
From: Joshua Horton <jahorton@sil.org>
Date: Mon, 20 Jan 2025 15:04:53 +0700
Subject: [PATCH 06/53] Revert "Merge pull request #12940 from
 keymanapp/fix/web/unit-test-affected-by-autocorrect"

This reverts commit 217592ff9f844cabfa158fa1003446668888255e, reversing
changes made to fc15e616a729ba9bfa2ef096fc149098474a30f6.
---
 .../interfaces/prediction/predictionContext.tests.js | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/web/src/test/auto/headless/engine/interfaces/prediction/predictionContext.tests.js b/web/src/test/auto/headless/engine/interfaces/prediction/predictionContext.tests.js
index 9aa9cd8015..4217aeb69b 100644
--- a/web/src/test/auto/headless/engine/interfaces/prediction/predictionContext.tests.js
+++ b/web/src/test/auto/headless/engine/interfaces/prediction/predictionContext.tests.js
@@ -161,21 +161,15 @@ describe("PredictionContext", () => {
     assert.equal(updateFake.callCount, 3);
     suggestions = updateFake.thirdCall.args[0];
 
-    // Note:  this unit test was originally written with auto-correct on!
-    // #11941 was written 2024-07-25 (added unit test for auto-correction method)
-    // #12169 was written 2024-08-14, which is what added THIS unit test.
-
     // This does re-use the apply-revert oriented mocking.
     // Should skip the (second) "apple", "apply", "apps" round, as it became outdated
     // by its following request before its response could be received.
-    assert.deepEqual(suggestions.map((obj) => obj.displayAs), ['applied']); // '“apple”' included with auto-correct enabled.
-    // Is not displayed; we only display it if auto-correct is on, as 'applied' would be automatic then.
-    assert.equal(predictiveContext.keepSuggestion.displayAs, '“apple”');
-    // assert.equal(suggestions.find((obj) => obj.tag == 'keep').displayAs, '“apple”'); // with auto-correct enabled.
+    assert.deepEqual(suggestions.map((obj) => obj.displayAs), ['“apple”', 'applied']);
+    assert.equal(suggestions.find((obj) => obj.tag == 'keep').displayAs, '“apple”');
     assert.equal(suggestions.find((obj) => obj.transform.deleteLeft != 0).displayAs, 'applied');
     // Our reused mocking doesn't directly provide the 'keep' suggestion; we
     // need to remove it before testing for set equality.
-    assert.deepEqual(suggestions /*.splice(1)*/, expected);
+    assert.deepEqual(suggestions.splice(1), expected);
   });
 
   it('sendUpdateState retrieves the most recent suggestion set', async function() {

From 7c1d15ff963f825b7ca9c400fddb5524a5a6515a Mon Sep 17 00:00:00 2001
From: Joshua Horton <jahorton@sil.org>
Date: Mon, 20 Jan 2025 16:00:48 +0700
Subject: [PATCH 07/53] chore(web): deletes unit-test line with now-invalid
 assumption

---
 .../engine/interfaces/prediction/predictionContext.tests.js      | 1 -
 1 file changed, 1 deletion(-)

diff --git a/web/src/test/auto/headless/engine/interfaces/prediction/predictionContext.tests.js b/web/src/test/auto/headless/engine/interfaces/prediction/predictionContext.tests.js
index 4217aeb69b..ddcdc4853e 100644
--- a/web/src/test/auto/headless/engine/interfaces/prediction/predictionContext.tests.js
+++ b/web/src/test/auto/headless/engine/interfaces/prediction/predictionContext.tests.js
@@ -166,7 +166,6 @@ describe("PredictionContext", () => {
     // by its following request before its response could be received.
     assert.deepEqual(suggestions.map((obj) => obj.displayAs), ['“apple”', 'applied']);
     assert.equal(suggestions.find((obj) => obj.tag == 'keep').displayAs, '“apple”');
-    assert.equal(suggestions.find((obj) => obj.transform.deleteLeft != 0).displayAs, 'applied');
     // Our reused mocking doesn't directly provide the 'keep' suggestion; we
     // need to remove it before testing for set equality.
     assert.deepEqual(suggestions.splice(1), expected);

From 606c7e77675b0bafd93e9c35bce268fe45d90caa Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Thu, 10 Jul 2025 09:06:52 -0500
Subject: [PATCH 08/53] change(web): do not set deleteRight when merging
 Transforms without it

---
 .../predictive-text/templates/src/common.ts    | 11 ++++++++---
 .../templates/tests/common.tests.js            | 12 ++++--------
 .../cases/edit-distance/context-tracker.js     | 18 +++++++++---------
 3 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/web/src/engine/predictive-text/templates/src/common.ts b/web/src/engine/predictive-text/templates/src/common.ts
index f23ad6095b..b59bbccb48 100644
--- a/web/src/engine/predictive-text/templates/src/common.ts
+++ b/web/src/engine/predictive-text/templates/src/common.ts
@@ -55,13 +55,18 @@ export function buildMergedTransform(first: Transform, second: Transform): Trans
     }
   }
 
-  return {
+  const returnedObj: Transform = {
     insert: mergedFirstInsert + second.insert,
-    deleteLeft: first.deleteLeft + mergedSecondDelete,
+    deleteLeft: first.deleteLeft + mergedSecondDelete
+  }
+
+  if(first.deleteRight != undefined || second.deleteRight != undefined) {
     // As `first` would affect the context before `second` could take effect,
     // this is the correct way to merge `deleteRight`.
-    deleteRight: (first.deleteRight || 0) + (second.deleteRight || 0)
+    returnedObj.deleteRight = (first.deleteRight || 0) + (second.deleteRight || 0)
   }
+
+  return returnedObj;
 }
 
 /**
diff --git a/web/src/engine/predictive-text/templates/tests/common.tests.js b/web/src/engine/predictive-text/templates/tests/common.tests.js
index dfd5c875cb..5abd27cd75 100644
--- a/web/src/engine/predictive-text/templates/tests/common.tests.js
+++ b/web/src/engine/predictive-text/templates/tests/common.tests.js
@@ -22,8 +22,7 @@ describe('Common utility functions', function() {
 
       let final = {
         insert: 'applebanana',
-        deleteLeft: 0,
-        deleteRight: 0
+        deleteLeft: 0
       };
 
       let mergedTransform = models.buildMergedTransform(apple, banana);
@@ -43,8 +42,7 @@ describe('Common utility functions', function() {
 
       let final = {
         insert: 'applebanana',
-        deleteLeft: 2,
-        deleteRight: 0
+        deleteLeft: 2
       };
 
       let mergedTransform = models.buildMergedTransform(apple, banana);
@@ -64,8 +62,7 @@ describe('Common utility functions', function() {
 
       let final = {
         insert: 'bananapple',  // the 'apple' transform removes the final 'a' from 'banana'.
-        deleteLeft: 0,
-        deleteRight: 0
+        deleteLeft: 0
       };
 
       let mergedTransform = models.buildMergedTransform(banana, apple);
@@ -85,8 +82,7 @@ describe('Common utility functions', function() {
 
       let final = {
         insert: 'bananapple',  // the 'apple' transform removes the final 'a' from 'banana'.
-        deleteLeft: 2,
-        deleteRight: 0
+        deleteLeft: 2
       };
 
       let mergedTransform = models.buildMergedTransform(banana, apple);
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
index 65298a2dd3..0c21a26b21 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
@@ -209,14 +209,14 @@ describe('ContextTracker', function() {
 
       let baseContextMatch = ContextTracker.modelContextState(existingContext.left);
       let newContextMatch = ContextTracker.attemptMatchContext(
-        newContext.left, 
-        baseContextMatch, 
+        newContext.left,
+        baseContextMatch,
         tokenizeTransformDistribution(tokenizer, {left: "an"}, [{sample: transform, p: 1}])
       );
       assert.isNotNull(newContextMatch?.state);
       assert.deepEqual(newContextMatch.state.tokens.map(token => token.raw), rawTokens);
       // We want to preserve all text preceding the new token when applying a suggestion.
-      assert.deepEqual(newContextMatch.preservationTransform, { insert: 'd ', deleteLeft: 0, deleteRight: 0});
+      assert.deepEqual(newContextMatch.preservationTransform, { insert: 'd ', deleteLeft: 0});
 
       // The 'wordbreak' transform
       let state = newContextMatch.state;
@@ -242,14 +242,14 @@ describe('ContextTracker', function() {
 
       let baseContextMatch = ContextTracker.modelContextState(existingContext.left);
       let newContextMatch = ContextTracker.attemptMatchContext(
-        newContext.left, 
-        baseContextMatch, 
+        newContext.left,
+        baseContextMatch,
         tokenizeTransformDistribution(tokenizer, {left: "apple a day keeps the doc"}, [{sample: transform, p: 1}])
       );
       assert.isNotNull(newContextMatch?.state);
       assert.deepEqual(newContextMatch.state.tokens.map(token => token.raw), rawTokens);
       // We want to preserve all text preceding the new token when applying a suggestion.
-      assert.deepEqual(newContextMatch.preservationTransform, { insert: 'tor ', deleteLeft: 0, deleteRight: 0 });
+      assert.deepEqual(newContextMatch.preservationTransform, { insert: 'tor ', deleteLeft: 0 });
 
       // The 'wordbreak' transform
       let state = newContextMatch.state;
@@ -271,7 +271,7 @@ describe('ContextTracker', function() {
       let newContext = models.tokenize(defaultBreaker, {
         left: "text'\""
       });
-      // The reason it's a problem - current internal logic isn't prepared to shift 
+      // The reason it's a problem - current internal logic isn't prepared to shift
       // from 1 to 3 tokens in a single step.
       assert.equal(newContext.left.length, 3);
 
@@ -280,8 +280,8 @@ describe('ContextTracker', function() {
         deleteLeft: 0
       }
       let problemContextMatch = ContextTracker.attemptMatchContext(
-        newContext.left, 
-        baseContextMatch, 
+        newContext.left,
+        baseContextMatch,
         tokenizeTransformDistribution(tokenizer, {left: "text'"}, [{sample: transform, p: 1}])
       );
       assert.isNull(problemContextMatch);

From 9bbed384e63395ce999e94fafeefe6d5fc0d4d90 Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Tue, 15 Jul 2025 13:23:41 -0500
Subject: [PATCH 09/53] feat(web): adds isSubstitutionAlignable to assist
 validating context matches after edits

This adds one new method within the predictive-text worker space: isSubstitutionAlignable. The method is designed to report whether or not two words are "related enough" to consider as an appropriate word-level "substitution" when matching the incoming context against previously-seen contexts - a process useful for facilitating delayed reversions, among other things.

It is not yet integrated with the main body of worker code, however.
---
 .../src/main/correction/context-tracker.ts    | 86 +++++++++++++++++++
 .../cases/edit-distance/context-tracker.js    | 73 ++++++++++++++++
 2 files changed, 159 insertions(+)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
index 516ac779fb..29cd455668 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
@@ -346,6 +346,92 @@ interface ContextMatchResult {
 }
 
 export class ContextTracker extends CircularArray<TrackedContextState> {
+  /**
+   * Aligns two tokens on a character-by-character basis as needed for higher, token-level alignment
+   * operations.
+   * @param incomingToken The incoming token value
+   * @param matchingToken The pre-existing token value to use for comparison and alignment
+   * @param forNearCaret  If `false`, disallows any substitutions and activates a leading-edge alignment
+   * validation mode.
+   * @returns
+   */
+  static isSubstitutionAlignable(
+    incomingToken: string,
+    matchingToken: string,
+    forNearCaret?: boolean
+  ): boolean {
+    // 1 - Determine the edit path for the word.
+    let subEditPath = ClassicalDistanceCalculation.computeDistance(
+      [...matchingToken].map(value => ({key: value})),
+      [...incomingToken].map(value => ({key: value})),
+      // Diagonal width to consider must be at least 2, as adding a single
+      // whitespace after a token tends to add two tokens: one for whitespace,
+      // one for the empty token to follow it.
+      3
+    ).editPath();
+
+    const firstInsert = subEditPath.indexOf('insert');
+    const firstDelete = subEditPath.indexOf('delete');
+
+    // 2 - deletions and insertions should be mutually exclusive.
+    // A fixed, unedited word can't slide across both 'left' and 'right' boundaries at the same time.
+    if(firstInsert != -1 && firstDelete != -1) {
+      return false;
+    };
+
+    // 3 - checks exclusive to leading-edge conditions
+    if(!forNearCaret) {
+      const firstSubstitute = subEditPath.indexOf('substitute');
+      const firstMatch      = subEditPath.indexOf('match');
+      if(firstSubstitute > -1) {
+        return false;
+      } else if(firstMatch > -1) {
+        // Should not have inserts on both sides of matched text!
+        if(firstInsert > -1 && firstInsert < firstMatch && subEditPath.lastIndexOf('insert') > firstMatch) {
+          return false;
+        } else if(firstDelete > -1 && firstDelete < firstMatch && subEditPath.lastIndexOf('delete') > firstMatch) {
+          return false;
+        }
+      }
+
+      // Further checks below are oriented for text/tokens at the caret.
+      return true;
+    }
+
+    // 4 - check the stats for total edits of each type and validate that edits don't overly exceed
+    // original characters.
+    const editCount = {
+      matchMove: 0,
+      rawEdit: 0
+    };
+
+    subEditPath.forEach((entry) => {
+      switch(entry) {
+        case 'transpose-end':
+        case 'transpose-start':
+        case 'match':
+          editCount.matchMove++;
+          break;
+        case 'insert':
+        case 'transpose-insert':
+        case 'delete':
+        case 'transpose-delete':
+        case 'substitute':
+          editCount.rawEdit++;
+      }
+    });
+
+    // We shouldn't have more raw substitutions, inserts, and deletes than matches + transposes,
+    // though allowing +1 as a fudge factor.
+    // The 'a' => 'à' pattern can be a reasonably common Keyman keyboard rule and
+    // is one substitution, zero matches in NFC.
+    if(editCount.matchMove + 1 < editCount.rawEdit) {
+      return false;
+    }
+
+    return true;
+  }
+
   static attemptMatchContext(
     tokenizedContext: Token[],
     matchState: TrackedContextState,
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
index 0c21a26b21..65b16c5fc2 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
@@ -22,6 +22,79 @@ describe('ContextTracker', function() {
     }];
   }
 
+  describe('isSubstitutionAlignable', () => {
+    it(`returns true:  'ca' => 'can'`, () => {
+      assert.isTrue(ContextTracker.isSubstitutionAlignable('can', 'ca'));
+    });
+
+    // Leading word in context window starts sliding out of said window.
+    it(`returns true:  'can' => 'an'`, () => {
+      assert.isTrue(ContextTracker.isSubstitutionAlignable('an', 'can'));
+    });
+
+    // Same edits on both sides:  not valid.
+    it(`returns false: 'apple' => 'grapples'`, () => {
+      assert.isFalse(ContextTracker.isSubstitutionAlignable('grapples', 'apple'));
+    });
+
+    // Edits on one side:  valid.
+    it(`returns true: 'apple' => 'grapple'`, () => {
+      assert.isTrue(ContextTracker.isSubstitutionAlignable('grapple', 'apple'));
+    });
+
+    // Edits on one side:  valid.
+    it(`returns true: 'apple' => 'grapple'`, () => {
+      assert.isTrue(ContextTracker.isSubstitutionAlignable('apples', 'apple'));
+    });
+
+    // Same edits on both sides:  not valid.
+    it(`returns false: 'grapples' => 'apple'`, () => {
+      assert.isFalse(ContextTracker.isSubstitutionAlignable('apple', 'grapples'));
+    });
+
+    // Substitution:  not valid when not permitted via parameter.
+    it(`returns false:  'apple' => 'banana'`, () => {
+      // edit path:  'insert' ('b' of banana), 'match' (on leading a), rest are 'substitute'.
+      assert.isFalse(ContextTracker.isSubstitutionAlignable('banana', 'apple'));
+    });
+
+    // Substitution:  not valid if too much is substituted, even if allowed via parameter.
+    it(`returns false:  'apple' => 'banana' (subs allowed)`, () => {
+      // edit path:  'insert' ('b' of banana), 'match' (on leading a), rest are 'substitute'.
+      // 1 match vs 4 substitute = no bueno.  It'd require too niche of a keyboard rule.
+      assert.isFalse(ContextTracker.isSubstitutionAlignable('banana', 'apple', true));
+    });
+
+    it(`returns true: 'a' => 'à' (subs allowed)`, () => {
+      assert.isTrue(ContextTracker.isSubstitutionAlignable('à', 'a', true));
+    });
+
+    // Leading substitution:  valid if enough of the remaining word matches.
+    // Could totally happen from a legit Keyman keyboard rule.
+    it(`returns true: 'can' => 'van' (subs allowed)`, () => {
+      assert.isTrue(ContextTracker.isSubstitutionAlignable('van', 'can', true));
+    });
+
+    // Trailing substitution:  invalid if not allowed.
+    it(`returns false: 'can' => 'cap' (subs not allowed)`, () => {
+      assert.isFalse(ContextTracker.isSubstitutionAlignable('cap', 'can'));
+    });
+
+    // Trailing substitution:  valid.
+    it(`returns false: 'can' => 'cap' (subs allowed)`, () => {
+      assert.isTrue(ContextTracker.isSubstitutionAlignable('cap', 'can', true));
+    });
+
+    it(`returns true:  'clasts' => 'clasps' (subs allowed)`, () => {
+      assert.isTrue(ContextTracker.isSubstitutionAlignable('clasps', 'clasts', true));
+    });
+
+    // random deletion at the start + later substitution = still permitted
+    it(`returns false:  'clasts' => 'clasps' (subs allowed)`, () => {
+      assert.isTrue(ContextTracker.isSubstitutionAlignable('lasps', 'clasts', true));
+    });
+  });
+
   describe('attemptMatchContext', function() {
     it("properly matches and aligns when lead token is removed", function() {
       let existingContext = models.tokenize(defaultBreaker, {

From 8c052968aeee2b1ba516ecb6e77cee59459870b5 Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Thu, 10 Jul 2025 16:29:38 -0500
Subject: [PATCH 10/53] refactor(web): new method - attemptTokenizedAlignment

Following from #14363, this method performs context alignment calculations that may be
used to match forms of the context before and after an edit by aligning their tokens and
validating any edits that may have occurred.

Note that no 'tracked context' states are manipulated or altered by this method - it
solely calculates the alignment deltas needed to align the two contexts.  Other methods
may then take these values and determine the edits that occurred during the associated
context transition as needed.

Note that the `attemptTokenizedAlignment` method is not integrated into the main codebase
for the predictive-text worker at this time.  That said, this method _does_ integrate
the `isSubstitutionAlignable` method introduced by #14363.
---
 .../src/main/correction/context-tracker.ts    | 210 +++++++++++-
 .../cases/edit-distance/context-tracker.js    | 312 ++++++++++++++++++
 2 files changed, 521 insertions(+), 1 deletion(-)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
index 29cd455668..287b8e6d73 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
@@ -345,6 +345,46 @@ interface ContextMatchResult {
   tailTokensAdded: number;
 }
 
+/**
+ * Represents token-count values resulting from an alignment attempt between two
+ * different modeled context states.
+ */
+type TrackedContextStateAlignment = {
+  /**
+   * Denotes whether or not alignment is possible between two contexts.
+   */
+  canAlign: false
+} | {
+  /**
+   * Denotes whether or not alignment is possible between two contexts.
+   */
+  canAlign: true,
+  /**
+   * Notes the number of tokens added to the head of the 'incoming'/'new' context
+   * of the contexts being aligned.  If negative, the incoming context deleted
+   * a token found in the 'original' / base context.
+   *
+   * For the alignment, [base context index] + leadTokenShift = [incoming context index].
+   */
+  leadTokenShift: number,
+  /**
+   * The count of tokens perfectly aligned, with no need for edits, for two successfully-
+   * alignable contexts.
+   */
+  matchLength: number,
+  /**
+   * The count of tokens at the tail perfectly aligned (existing in both contexts) but
+   * edited for two successfully-alignable contexts.  These tokens directly follow those
+   * that need no edits.
+   */
+  tailEditLength: number,
+  /**
+   * The count of new tokens added at the end of the incoming context for two aligned contexts.
+   * If negative, the incoming context deleted a previously-existing token from the original.
+   */
+  tailTokenShift: number
+};
+
 export class ContextTracker extends CircularArray<TrackedContextState> {
   /**
    * Aligns two tokens on a character-by-character basis as needed for higher, token-level alignment
@@ -432,6 +472,171 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
     return true;
   }
 
+  static attemptTokenizedAlignment(
+    incomingTokenization: string[],
+    tokenizationToMatch: string[]
+  ): TrackedContextStateAlignment {
+
+    // Inverted order, since 'match' existed before our new context.
+    let mapping = ClassicalDistanceCalculation.computeDistance(
+      tokenizationToMatch.map(value => ({key: value})),
+      incomingTokenization.map(value => ({key: value})),
+      // Diagonal width to consider must be at least 2, as adding a single
+      // whitespace after a token tends to add two tokens: one for whitespace,
+      // one for the empty token to follow it.
+      3
+    );
+
+    let editPath = mapping.editPath();
+    const firstMatch = editPath.indexOf('match');
+    const lastMatch = getEditPathLastMatch(editPath);
+    if(firstMatch == -1) {
+      // If there are no matches, there's no alignment.
+      return {
+        canAlign: false
+      };
+    }
+
+    // Transpositions are not allowed at the token level during context alignment.
+    if(editPath.find((entry) => entry.indexOf('transpose') > -1)) {
+      return {
+        canAlign: false
+      };
+    }
+
+    let matchLength = lastMatch - firstMatch + 1;
+    let tailInsertLength = 0;
+    let tailDeleteLength = 0;
+    for(let i = lastMatch; i < editPath.length; i++) {
+      if(editPath[i] == 'insert') {
+        tailInsertLength++;
+      } else if(editPath[i] == 'delete') {
+        tailDeleteLength++;
+      }
+    }
+    if(tailInsertLength > 0 && tailDeleteLength > 0) {
+      // Something's gone weird if this happens; that should appear as a substitution instead.
+      // Otherwise, we have a VERY niche edit scenario.
+      return {
+        canAlign: false
+      };
+    }
+    const tailSubstituteLength = (editPath.length - 1 - lastMatch) - tailInsertLength - tailDeleteLength;
+
+    // Assertion:  for a long context, the bulk of the edit path should be a
+    // continuous block of 'match' entries.  If there's anything else in
+    // the middle, we have a context mismatch.
+    if(firstMatch > -1) {
+      for(let i = firstMatch+1; i < lastMatch; i++) {
+        if(editPath[i] != 'match') {
+          return {
+            canAlign: false
+          };
+        }
+      }
+    }
+
+    // If we have a perfect match with a pre-existing context, no mutations have
+    // happened; we have a 100% perfect match.
+    if(firstMatch == 0 && lastMatch == editPath.length - 1) {
+      return {
+        canAlign: true,
+        leadTokenShift: 0,
+        matchLength,
+        tailEditLength: tailSubstituteLength,
+        tailTokenShift: tailInsertLength - tailDeleteLength
+      };
+    }
+
+    // The edit path calc tries to put substitutes first, before inserts.
+    // We don't want that on the leading edge.
+    const lastEarlyInsert = editPath.lastIndexOf('insert', firstMatch);
+    const firstSubstitute = editPath.indexOf('substitute');
+    if(firstSubstitute > -1 && firstSubstitute < firstMatch && firstSubstitute < lastEarlyInsert) {
+      editPath[firstSubstitute] = 'insert';
+      editPath[lastEarlyInsert] = 'substitute';
+    }
+
+    // If mutations HAVE happened, we need to double-check the context-state alignment.
+    let priorEdit: typeof editPath[0];
+    let leadTokensRemoved = 0;
+    let leadSubstitutions = 0;
+
+    // The `i` index below aligns based upon the index within the `tokenizationToMatch` sequence
+    // and how it would have to be edited to align to the `incomingTokenization` sequence.
+    for(let i = 0; i < firstMatch; i++) {
+      switch(editPath[i]) {
+        case 'delete':
+          // All deletions should appear at the sliding window edge; if a deletion appears
+          // after the edge, but before the first match, something's wrong.
+          if(priorEdit && priorEdit != 'delete') {
+            return {
+              canAlign: false
+            };
+          }
+          leadTokensRemoved++;
+          break;
+        case 'substitute':
+          // We only allow for one leading token to be substituted.
+          //
+          // Any extras in the front would be pure inserts, not substitutions, due to
+          // the sliding context window and its implications.
+          if(leadSubstitutions++ > 0) {
+            return {
+              canAlign: false
+            };
+          }
+
+          // Find the word before and after substitution.
+          const incomingSub = incomingTokenization[i - (leadTokensRemoved > 0 ? leadTokensRemoved : 0)];
+          const matchingSub = tokenizationToMatch[i + (leadTokensRemoved < 0 ? leadTokensRemoved : 0)];
+
+          // Double-check the word - does the 'substituted' word itself align?
+          if(!this.isSubstitutionAlignable(incomingSub, matchingSub)) {
+            return {
+              canAlign: false
+            };
+          }
+
+          // There's no major need to drop parts of a token being 'slid' out of the context window.
+          // We'll leave it intact and treat it as a 'match'
+          matchLength++;
+          break;
+        case 'insert':
+          // Only allow an insert at the leading edge, as with 'delete's.
+          if(priorEdit && priorEdit != 'insert') {
+            return {
+              canAlign: false
+            };
+          }
+          // In case of backspaces, it's also possible to 'insert' a 'new'
+          // token - an old one that's slid back into view.
+          leadTokensRemoved--;
+          break;
+        default:
+          // No 'match' can exist before the first found index for a 'match'.
+          // No 'transpose-' edits should exist within this section, either.
+          return {
+            canAlign: false
+          };
+      }
+      priorEdit = editPath[i];
+    }
+
+    // If we need some form of tail-token substitution verification, add that here.
+
+    return {
+      canAlign: true,
+      // leadTokensRemoved represents the number of tokens that must be removed from the base context
+      // when aligning the contexts.  Externally, it's more helpful to think in terms of the count added
+      // to the incoming context.
+      leadTokenShift: -leadTokensRemoved + 0, // add 0 in case of a 'negative zero', which affects unit tests.
+      matchLength,
+      tailEditLength: tailSubstituteLength,
+      tailTokenShift: tailInsertLength - tailDeleteLength
+    };
+  }
+
   static attemptMatchContext(
     tokenizedContext: Token[],
     matchState: TrackedContextState,
@@ -496,7 +701,7 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
           // No 'insert' should exist on the leading edge of context when the
           // context window slides.
           //
-          // No 'transform' edits should exist within this section, either.
+          // No 'transpose' edits should exist within this section, either.
           return null;
       }
     }
@@ -783,6 +988,9 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
 
     let tokenize = determineModelTokenizer(model);
 
+    if(transformDistribution?.length == 0) {
+      transformDistribution = null;
+    }
     const inputTransform = transformDistribution?.[0];
     let transformTokenLength = 0;
     let tokenizedDistribution: Distribution<Transform[]> = null;
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
index 65b16c5fc2..a3845b0e2d 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
@@ -95,6 +95,318 @@ describe('ContextTracker', function() {
     });
   });
 
+  describe('attemptTokenizedAlignment', () => {
+    it("properly matches and aligns when contexts match", () => {
+      const baseContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [...baseContext];
+
+      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
+      assert.deepEqual(computedAlignment, {
+        canAlign: true,
+        leadTokenShift: 0,
+        matchLength: 5,
+        tailEditLength: 0,
+        tailTokenShift: 0
+      });
+    });
+
+    it("detects unalignable contexts - no matching tokens", () => {
+      const baseContext = [
+        'swift', 'tan', 'wolf', 'leaped', 'across'
+      ];
+      const newContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+
+      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
+      assert.deepEqual(computedAlignment, {canAlign: false});
+    });
+
+    it("detects unalignable contexts - too many mismatching tokens", () => {
+      const baseContext = [
+        'swift', 'tan', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+
+      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
+      assert.deepEqual(computedAlignment, {canAlign: false});
+    });
+
+    it("fails alignment for leading-edge word substitutions", () => {
+      const baseContext = [
+        'swift', 'brown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+
+      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
+      assert.deepEqual(computedAlignment, {canAlign: false});
+    });
+
+    it("fails alignment for small leading-edge word substitutions", () => {
+      const baseContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'sick', 'brown', 'fox', 'jumped', 'over'
+      ];
+
+      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
+      assert.deepEqual(computedAlignment, {canAlign: false});
+    });
+
+    it("properly matches and aligns when lead token is modified", () => {
+      const baseContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'uick', 'brown', 'fox', 'jumped', 'over'
+      ];
+
+      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
+      assert.deepEqual(computedAlignment, {
+        canAlign: true,
+        leadTokenShift: 0,
+        matchLength: 5,
+        tailEditLength: 0,
+        tailTokenShift: 0
+      });
+    });
+
+    it("properly matches and aligns when lead token is removed", () => {
+      const baseContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'brown', 'fox', 'jumped', 'over'
+      ];
+
+      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
+      assert.deepEqual(computedAlignment, {
+        canAlign: true,
+        leadTokenShift: -1,
+        matchLength: 4,
+        tailEditLength: 0,
+        tailTokenShift: 0
+      });
+    });
+
+    it("properly matches and aligns when lead token is added", () => {
+      const baseContext = [
+        'brown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+
+      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
+      assert.deepEqual(computedAlignment, {
+        canAlign: true,
+        leadTokenShift: 1,
+        matchLength: 4,
+        tailEditLength: 0,
+        tailTokenShift: 0
+      });
+    });
+
+    it("properly matches and aligns when lead tokens are removed and modified", () => {
+      const baseContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'ox', 'jumped', 'over'
+      ];
+
+      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
+      assert.deepEqual(computedAlignment, {
+        canAlign: true,
+        leadTokenShift: -2,
+        matchLength: 3,
+        tailEditLength: 0,
+        tailTokenShift: 0
+      });
+    });
+
+    it("properly matches and aligns when lead tokens are added and modified", () => {
+      const baseContext = [
+        'rown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+
+      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
+      assert.deepEqual(computedAlignment, {
+        canAlign: true,
+        leadTokenShift: 1,
+        matchLength: 4,
+        tailEditLength: 0,
+        tailTokenShift: 0
+      });
+    });
+
+    it("properly matches and aligns when lead token is removed and tail token is added", () => {
+      const baseContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'brown', 'fox', 'jumped', 'over', 'the'
+      ];
+
+      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
+      assert.deepEqual(computedAlignment, {
+        canAlign: true,
+        leadTokenShift: -1,
+        matchLength: 4,
+        tailEditLength: 0,
+        tailTokenShift: 1
+      });
+    });
+
+    it("properly matches and aligns when lead token and tail token are modified", () => {
+      const baseContext = [
+        'quick', 'brown', 'fox', 'jumped', 'ove'
+      ];
+      const newContext = [
+        'uick', 'brown', 'fox', 'jumped', 'over'
+      ];
+
+      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
+      assert.deepEqual(computedAlignment, {
+        canAlign: true,
+        leadTokenShift: 0,
+        matchLength: 4, // we treat 'quick' and 'uick' as the same
+        tailEditLength: 1,
+        tailTokenShift: 0
+      });
+    });
+
+    it("properly matches and aligns when lead token and tail token are modified + new token appended", () => {
+      const baseContext = [
+        'quick', 'brown', 'fox', 'jumped', 'ove'
+      ];
+      const newContext = [
+        'uick', 'brown', 'fox', 'jumped', 'over', 't'
+      ];
+
+      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
+      assert.deepEqual(computedAlignment, {
+        canAlign: true,
+        leadTokenShift: 0,
+        matchLength: 4, // we treat 'quick' and 'uick' as the same
+        tailEditLength: 1,
+        tailTokenShift: 1
+      });
+    });
+
+    it("properly handles context window sliding backward", () => {
+      const baseContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'e', 'quick', 'brown', 'fox', 'jumped', 'ove'
+      ];
+
+      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
+      assert.deepEqual(computedAlignment, {
+        canAlign: true,
+        leadTokenShift: 1,
+        matchLength: 4, // we treat 'quick' and 'uick' as the same
+        tailEditLength: 1,
+        tailTokenShift: 0
+      });
+    });
+
+    it("properly handles context window sliding far backward", () => {
+      const baseContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'the', 'quick', 'brown', 'fox', 'jumped'
+      ];
+
+      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
+      assert.deepEqual(computedAlignment, {
+        canAlign: true,
+        leadTokenShift: 1,
+        matchLength: 4, // we treat 'quick' and 'uick' as the same
+        tailEditLength: 0,
+        tailTokenShift: -1
+      });
+    });
+
+    it("properly handles context window sliding farther backward", () => {
+      const baseContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'the', 'quick', 'brown', 'fox', 'jumpe'
+      ];
+
+      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
+      assert.deepEqual(computedAlignment, {
+        canAlign: true,
+        leadTokenShift: 1,
+        matchLength: 3, // we treat 'quick' and 'uick' as the same
+        tailEditLength: 1,
+        tailTokenShift: -1
+      });
+    });
+
+    it("fails alignment for mid-head deletion", () => {
+      const baseContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'quick', 'fox', 'jumped', 'over'
+      ];
+
+      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
+      assert.deepEqual(computedAlignment, {canAlign: false});
+    });
+
+    it("fails alignment for mid-head insertion", () => {
+      const baseContext = [
+        'quick', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+
+      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
+      assert.deepEqual(computedAlignment, {canAlign: false});
+    });
+
+    it("fails alignment for mid-tail deletion", () => {
+      const baseContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'quick', 'brown', 'fox', 'over'
+      ];
+
+      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
+      assert.deepEqual(computedAlignment, {canAlign: false});
+    });
+
+    it("fails alignment for mid-tail insertion", () => {
+      const baseContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'quick', 'brown', 'fox', 'jumped', 'far', 'over'
+      ];
+
+      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
+      assert.deepEqual(computedAlignment, {canAlign: false});
+    });
+  });
+
   describe('attemptMatchContext', function() {
     it("properly matches and aligns when lead token is removed", function() {
       let existingContext = models.tokenize(defaultBreaker, {

From a1c10de821171c67e39d480768dbcfd06281b97e Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Tue, 15 Jul 2025 12:35:24 -0500
Subject: [PATCH 11/53] refactor(web): reworks attemptMatchContext to use new
 attemptTokenizedAlignment method

Following from #14364, this PR integrates the new method with the main predictive-text context-tracking code, significantly reworking the `attemptMatchContext` method in the process.  While further refactoring of the latter method is planned, this step allows us to verify that the new methods integrate properly with the main codebase in their current form.

This also comes with the benefit of simplifying `attemptMatchContext` _significantly_ - large parts of its code were refactored into `attemptTokenizedAlignment`, and the new logic patterns are generally more straightforward to parse and understand.
---
 .../src/main/correction/context-tracker.ts    | 519 ++++++++++--------
 .../cases/edit-distance/context-tracker.js    |   2 +-
 2 files changed, 278 insertions(+), 243 deletions(-)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
index 287b8e6d73..581ed948b7 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
@@ -31,6 +31,17 @@ function textToCharTransforms(text: string, transformId?: number) {
   return perCharTransforms;
 }
 
+/**
+ * Determines the proper 'last match' index for a tokenized sequence based on its edit path.
+ *
+ * In particular, this method is designed to handle the following case:
+ * ['to', 'apple', ' ', ''] => ['to', 'apply', ' ', 'n']
+ *
+ * The ' ' is unedited, but as it follows the edited 'apple' => 'apply', 'to' is the true
+ * "last edited" token.
+ * @param editPath
+ * @returns
+ */
 export function getEditPathLastMatch(editPath: EditOperation[]) {
   const editLength = editPath.length;
   // Special handling: appending whitespace to whitespace with the default wordbreaker.
@@ -488,6 +499,44 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
     );
 
     let editPath = mapping.editPath();
+    // Special case:  new context bootstrapping - first token often substitutes.
+    // The text length is small enough that no words should be able to rotate out the start of the context.
+    // Special handling needed in case of no 'match'; the rest of the method assumes at least one 'match'.
+    if(editPath.length <= 3 && (editPath[0] == 'substitute' || editPath[0] == 'match')) {
+      let matchCount = 0;
+      let subCount = 0;
+      for(let i = 0; i < editPath.length; i++) {
+        if(editPath[i] == 'substitute') {
+          subCount++;
+          if(!this.isSubstitutionAlignable(incomingTokenization[i], tokenizationToMatch[i], true)) {
+            return {
+              canAlign: false
+            };
+          }
+        } else if(editPath[i] == 'match') {
+          // If a substitution is already recorded, treat the 'match' as a substitution.
+          if(subCount > 0) {
+            subCount++;
+          } else {
+            matchCount++;
+          }
+        }
+      }
+
+      const insertCount = editPath.filter((entry) => entry == 'insert').length;
+      const deleteCount = editPath.filter((entry) => entry == 'delete').length;
+
+      return {
+        canAlign: true,
+        matchLength: matchCount,
+        leadTokenShift: 0,
+        tailEditLength: subCount,
+        tailTokenShift: insertCount - deleteCount
+      }
+    }
+
+    // From here on assumes that at least one 'match' exists on the path.
+    // It all works great... once the context is long enough for at least one stable token.
     const firstMatch = editPath.indexOf('match');
     const lastMatch = getEditPathLastMatch(editPath);
     if(firstMatch == -1) {
@@ -640,283 +689,260 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
   static attemptMatchContext(
     tokenizedContext: Token[],
     matchState: TrackedContextState,
+    // the distribution should be tokenized already.
     transformSequenceDistribution?: Distribution<Transform[]>
   ): ContextMatchResult {
     // Map the previous tokenized state to an edit-distance friendly version.
     let matchContext: string[] = matchState.toRawTokenization();
 
-    // Inverted order, since 'match' existed before our new context.
-    let mapping = ClassicalDistanceCalculation.computeDistance(
-      matchContext.map(value => ({key: value})),
-      tokenizedContext.map(value => ({key: value.text})),
-      // Must be at least 2, as adding a single whitespace after a token tends
-      // to add two tokens: one for whitespace, one for the empty token to
-      // follow it.
-      3
-    );
+    const alignmentResults = this.attemptTokenizedAlignment(tokenizedContext.map((token) => token.text), matchContext);
 
-    let editPath = mapping.editPath();
-    const firstMatch = editPath.indexOf('match');
-    const lastMatch = getEditPathLastMatch(editPath);
-
-    // Assertion:  for a long context, the bulk of the edit path should be a
-    // continuous block of 'match' entries.  If there's anything else in
-    // the middle, we have a context mismatch.
-    if(firstMatch) {
-      for(let i = firstMatch+1; i < lastMatch; i++) {
-        if(editPath[i] != 'match') {
-          return null;
-        }
-      }
+    if(!alignmentResults.canAlign) {
+      return null;
     }
 
-    // If we have a perfect match with a pre-existing context, no mutations have
-    // happened; just re-use the old context state.
-    if(firstMatch == 0 && lastMatch == editPath.length - 1) {
-      return { state: matchState, baseState: matchState, headTokensRemoved: 0, tailTokensAdded: 0 };
-    }
-
-    // If mutations HAVE happened, we have work to do.
-    let state = matchState;
-
-    let priorEdit: typeof editPath[0];
-    let poppedTokenCount = 0;
-    for(let i = 0; i < firstMatch; i++) {
-      switch(editPath[i]) {
-        case 'delete':
-          if(priorEdit && priorEdit != 'delete') {
-            return null;
-          }
-          if(state == matchState) {
-            state = new TrackedContextState(state);
-          }
-          state.popHead();
-          poppedTokenCount++;
-          break;
-        case 'substitute':
-          // There's no major need to drop parts of a token being 'slid' out of the context window.
-          // We'll leave it intact.
-          break;
-        default:
-          // No 'insert' should exist on the leading edge of context when the
-          // context window slides.
-          //
-          // No 'transpose' edits should exist within this section, either.
-          return null;
-      }
-    }
+    const {
+      leadTokenShift,
+      matchLength,
+      tailEditLength,
+      tailTokenShift
+    } = alignmentResults;
 
     const hasDistribution = transformSequenceDistribution && Array.isArray(transformSequenceDistribution);
 
-    // Reset priorEdit for the end-of-context updating loop.
-    priorEdit = undefined;
-
-    // Used to construct and represent the part of the incoming transform that
-    // does not land as part of the final token in the resulting context.  This
-    // component should be preserved by any suggestions that get applied.
-    let preservationTransform: Transform;
-    let pushedTokenCount = 0;
-
-    // Now to update the end of the context window.
-    for(let i = lastMatch+1; i < editPath.length; i++) {
-      const isLastToken = i == editPath.length - 1;
-
+    // If we have a perfect match with a pre-existing context, no mutations have
+    // happened; just re-use the old context state.
+    if(tailEditLength == 0 && leadTokenShift == 0 && tailTokenShift == 0) {
+      return { state: matchState, baseState: matchState, headTokensRemoved: 0, tailTokensAdded: 0 };
+    } else {
       // If we didn't get any input, we really should perfectly match
       // a previous context state.  If such a state is out of our cache,
       // it should simply be rebuilt.
       if(!hasDistribution) {
         return null;
       }
-      const transformDistIndex = i - (lastMatch + 1);
-      const tokenDistribution = transformSequenceDistribution.map((entry) => {
-        const sample = entry.sample[transformDistIndex];
-        if(!sample) {
-          return null;
-        }
-        return {
-          sample,
-          p: entry.p
-        };
-      });
+    }
 
-      const incomingToken = tokenizedContext[i - poppedTokenCount];
+    // If mutations HAVE happened, we have work to do.
+    let state = matchState;
 
-      // If the tokenized part of the input is a completely empty transform,
-      // replace it with null.  This can happen with our default wordbreaker
-      // immediately after a whitespace.  We don't want to include this
-      // transform as part of the input when doing correction-search.
-      let primaryInput = hasDistribution ? tokenDistribution[0]?.sample : null;
-
-      // If the incoming token has text but we have no transform (or 'insert') to match
-      // it with, abort the matching attempt.  We can't match this case well yet.
-      if(editPath[i] != 'delete') {
-        if(!incomingToken) {
-          return null;
-        } else if(!(primaryInput || editPath[i] == 'insert' ) && incomingToken?.text != '') {
-          return null;
-        }
+    if(leadTokenShift < 0) {
+      state = new TrackedContextState(state);
+      for(let i = 0; i > leadTokenShift; i--) {
+        state.popHead();
       }
-
-      if(primaryInput && primaryInput.insert == "" && primaryInput.deleteLeft == 0 && !primaryInput.deleteRight) {
-        primaryInput = null;
-      }
-
-      // If this token's transform component is not part of the final token,
-      // it's something we'll want to preserve even when applying suggestions
-      // for the final token.
+    } else if(leadTokenShift > 0) {
+      // TODO:  insert token(s) at the start to match the text that's back within the
+      // sliding context window.
       //
-      // Note:  will need a either a different approach or more specialized
-      // handling if/when supporting phrase-level (multi-token) suggestions.
-      if(!isLastToken) {
-        preservationTransform = preservationTransform && primaryInput ? buildMergedTransform(preservationTransform, primaryInput) : (primaryInput ?? preservationTransform);
+      // (was not part of original `attemptContextMatch`)
+      return null;
+    }
+
+    // If no TAIL mutations have happened, we're safe to return now.
+    if(tailEditLength == 0 && tailTokenShift == 0) {
+      return {
+        state: state,
+        baseState: matchState,
+        headTokensRemoved: -leadTokenShift,
+        tailTokensAdded: tailTokenShift
       }
+    }
+
+    // ***
+
+    // first non-matched tail index within the incoming context
+    const incomingTailUpdateIndex = matchLength + (leadTokenShift > 0 ? leadTokenShift : 0);
+    // first non-matched tail index in `matchState`, the base context state.
+    const matchingTailUpdateIndex = matchLength - (leadTokenShift < 0 ? leadTokenShift : 0);
+
+    // The assumed input from the input distribution is always at index 0.
+    const tokenizedPrimaryInput = hasDistribution ? transformSequenceDistribution[0].sample : null;
+    // first index:  original sample's tokenization
+    // second index:  token index within original sample
+    const tokenDistribution = transformSequenceDistribution.map((entry) => {
+      return entry.sample.map((sample) => {
+        return {
+          sample: sample,
+          p: entry.p
+        }
+      });
+    });
+
+    // // Gets distribution of token index 1s as excerpted from the sequences' distribution.
+    // let a = tokenDistribution.map((sequence) => sequence[1]);
+
+    // Using these as base indices...
+
+    let tailIndex = 0;
+    // let lastTailIndex = tailEditLength + (tailTokenShift > 0 ? tailTokenShift : 0);
+
+    // Used to construct and represent the part of the incoming transform that
+    // does not land as part of the final token in the resulting context.  This
+    // component should be preserved by any suggestions that get applied.
+    let preservationTransform: Transform;
+
+    for(let i = 0; i < tailEditLength; i++) {
+      // do tail edits
+      const incomingIndex = i + incomingTailUpdateIndex;
+      const matchingIndex = i + matchingTailUpdateIndex;
+
+      const incomingToken = tokenizedContext[incomingIndex];
+      const matchedToken = matchState.tokens[matchingIndex];
+
+      let primaryInput = hasDistribution ? tokenizedPrimaryInput[i] : null;
       const isBackspace = primaryInput && TransformUtils.isBackspace(primaryInput);
 
-      switch(editPath[i]) {
-        case 'substitute':
-          if(isLastToken) {
-            state = new TrackedContextState(state);
-          }
+      const isLastToken = incomingIndex == tokenizedContext.length - 1;
 
-          const sourceToken = matchState.tokens[i];
-          state.tokens[i - poppedTokenCount] = sourceToken;
-          const token = state.tokens[i - poppedTokenCount];
+      if(isLastToken) {
+        state = new TrackedContextState(state);
+        // If this token's transform component is not part of the final token,
+        // it's something we'll want to preserve even when applying suggestions
+        // for the final token.
+        //
+        // Note:  will need a either a different approach or more specialized
+        // handling if/when supporting phrase-level (multi-token) suggestions.
+      } else {
+        preservationTransform = preservationTransform && primaryInput ? buildMergedTransform(preservationTransform, primaryInput) : (primaryInput ?? preservationTransform);
+      }
+      state.tokens[incomingIndex] = matchedToken;
+      const token = matchedToken;
 
-          // TODO:  I'm beginning to believe that searchSpace should (eventually) be tracked
-          // on the tokens, rather than on the overall 'state'.
-          // - Reason:  phrase-level corrections / predictions would likely need a search-state
-          //   across per potentially-affected token.
-          // - Shifting the paradigm should be a separate work unit than the
-          //   context-tracker rework currently being done, though.
-          if(isBackspace) {
-            token.updateWithBackspace(incomingToken.text, primaryInput.id);
-            if(isLastToken) {
-              state.tokens.pop(); // pops `token`
-              // puts it back in, rebuilding a fresh search-space that uses the rebuilt
-              // keystroke distribution from updateWithBackspace.
-              state.pushTail(token);
-            }
-          } else {
-            token.update(
-              tokenDistribution,
-              incomingToken.text
-            );
+      // TODO:  I'm beginning to believe that searchSpace should (eventually) be tracked
+      // on the tokens, rather than on the overall 'state'.
+      // - Reason:  phrase-level corrections / predictions would likely need a search-state
+      //   across per potentially-affected token.
+      // - Shifting the paradigm should be a separate work unit than the
+      //   context-tracker rework currently being done, though.
+      if(isBackspace) {
+        token.updateWithBackspace(incomingToken.text, primaryInput.id);
+        if(isLastToken) {
+          state.tokens.pop(); // pops `token`
+          // puts it back in, rebuilding a fresh search-space that uses the rebuilt
+          // keystroke distribution from updateWithBackspace.
+          state.pushTail(token);
+        }
+      } else {
+        token.update(
+          tokenDistribution.map((seq) => seq[tailIndex]),
+          incomingToken.text
+        );
 
-            if(isLastToken) {
-              // Search spaces may not exist during some unit tests; the state
-              // may not have an associated model during some.
-              state.searchSpace[0]?.addInput(tokenDistribution);
-            }
-          }
-
-          // For this case, we were _likely_ called by
-          // ModelCompositor.acceptSuggestion(), which would have marked the
-          // accepted suggestion.
-          //
-          // Upon inspection, this doesn't seem entirely ideal.  It works for
-          // the common case, but not for specially crafted keystroke
-          // transforms.  That said, it's also very low impact.  Best as I can
-          // see, this is only really used for debugging info?
-          if(state != matchState && !isLastToken) {
-            token.replacementText = incomingToken.text;
-          }
-
-          break;
-        case 'insert':
-          if(priorEdit && priorEdit != 'substitute' && priorEdit != 'match' && priorEdit != 'insert') {
-            return null;
-          }
-
-          if(!preservationTransform) {
-            // Allows for consistent handling of "insert" cases; even if there's no edit
-            // from a prior token, having a defined transform here indicates that
-            // a new token has been produced.  This serves as a useful conditional flag
-            // for prediction logic.
-            preservationTransform = { insert: '', deleteLeft: 0 };
-          }
-
-          if(state == matchState) {
-            state = new TrackedContextState(state);
-          }
-
-          let pushedToken = new TrackedContextToken();
-          pushedToken.raw = incomingToken.text;
-
-          // TODO:  assumes that there was no shift in wordbreaking from the
-          // prior context to the current one.  This may actually be a major
-          // issue for dictionary-based wordbreaking!
-          //
-          // If there was such a shift, then we may have extra transforms
-          // originally on a 'previous' token that got moved into this one!
-          //
-          // Suppose we're using a dictionary-based wordbreaker and have
-          // `butterfl` for our context, which could become butterfly.  If the
-          // next keystroke results in `butterfli`, this would likely be
-          // tokenized `butter` `fli`.  (e.g: `fli` leads to `flight`.) How do
-          // we know to properly relocate the `f` and `l` transforms?
-          if(primaryInput) {
-            pushedToken.transformDistributions = tokenDistribution ? [tokenDistribution] : [];
-          } else if(incomingToken.text) {
-            // We have no transform data to match against an inserted token with text; abort!
-            // Refer to #12494 for an example case; we currently can't map previously-committed
-            // input transforms to a newly split-off token.
-            return null;
-          }
-          pushedToken.isWhitespace = incomingToken.isWhitespace;
-
-          // Auto-replaces the search space to correspond with the new token.
-          state.pushTail(pushedToken);
-          pushedTokenCount++;
-          break;
-        case 'match':
-          // The default (Unicode) wordbreaker returns an empty token after whitespace blocks.
-          // Adding new whitespace extends the whitespace block but preserves the empty token
-          // following it.
-          if(priorEdit == 'substitute' && tokenizedContext[tokenizedContext.length-1].text == '') {
-            // Keep the blank token as-is; no edit needed!
-            continue;
-          }
-          // else 'fallthrough' / return null
-        case 'delete':
-          // While we do keep a cache of recent contexts, logic constraints for handling
-          // multitaps makes it tricky to reliably use in all situations.
-          // It's best to handle `delete` cases directly for this reason.
-          for(let j = i + 1; j < editPath.length; j++) {
-            // If something _other_ than delete follows a 'delete' on the edit path,
-            // we probably have a context mismatch.
-            //
-            // It's possible to construct cases where this isn't true, but it's likely not
-            // worth trying to handle such rare cases.
-            if(editPath[j] != 'delete') {
-              return null;
-            }
-          }
-
-          // If ALL that remains are deletes, we're good to go.
-          //
-          // This may not be the token at the index, but since all that remains are deletes,
-          // we'll have deleted the correct total number from the end once all iterations
-          // are done.
-          if(state == matchState) {
-            state = new TrackedContextState(state);
-          }
-
-          state.tokens.pop();
-          break;
-        default:
-          // No 'transform' edits should exist within this section.
-          return null;
+        if(isLastToken) {
+          // Search spaces may not exist during some unit tests; the state
+          // may not have an associated model during some.
+          state.searchSpace[0]?.addInput(tokenDistribution.map((seq) => seq[tailIndex]));
+        }
       }
 
-      priorEdit = editPath[i];
+      // For this case, we were _likely_ called by
+      // ModelCompositor.acceptSuggestion(), which would have marked the
+      // accepted suggestion.
+      //
+      // Upon inspection, this doesn't seem entirely ideal.  It works for
+      // the common case, but not for specially crafted keystroke
+      // transforms.  That said, it's also very low impact.  Best as I can
+      // see, this is only really used for debugging info?
+      if(state != matchState && !isLastToken) {
+        // TODO:  eliminate
+        token.replacementText = incomingToken.text;
+      }
+
+      tailIndex++;
+    }
+
+    if(tailTokenShift < 0) {
+      if(state == matchState) {
+        state = new TrackedContextState(state);
+      }
+
+      // delete tail tokens
+      for(let i = 0; i > tailTokenShift; i--) {
+        // If ALL that remains are deletes, we're good to go.
+        //
+        // This may not be the token at the index, but since all that remains are deletes,
+        // we'll have deleted the correct total number from the end once all iterations
+        // are done.
+        state.tokens.pop();
+      }
+    } else {
+      if(state == matchState) {
+        state = new TrackedContextState(state);
+      }
+
+      for(let i = tailEditLength; i < tailEditLength + tailTokenShift; i++) {
+        // create tail tokens
+        const incomingIndex = i + incomingTailUpdateIndex;
+        const incomingToken = tokenizedContext[incomingIndex];
+        // // Assertion:  there should be no matching token; this should be a newly-appended token.
+        // const matchingIndex = i + tailEditLength + matchingTailUpdateIndex;
+
+        const primaryInput = hasDistribution ? tokenizedPrimaryInput[i] : null;
+
+        if(!preservationTransform) {
+          // Allows for consistent handling of "insert" cases; even if there's no edit
+          // from a prior token, having a defined transform here indicates that
+          // a new token has been produced.  This serves as a useful conditional flag
+          // for prediction logic.
+          preservationTransform = { insert: '', deleteLeft: 0 };
+        }
+
+        const isLastToken = incomingIndex == tokenizedContext.length - 1;
+        if(!isLastToken) {
+          preservationTransform = preservationTransform && primaryInput ? buildMergedTransform(preservationTransform, primaryInput) : (primaryInput ?? preservationTransform);
+        }
+
+        if(state == matchState) {
+          state = new TrackedContextState(state);
+        }
+
+        let pushedToken = new TrackedContextToken();
+        pushedToken.raw = incomingToken.text;
+
+        // TODO:  assumes that there was no shift in wordbreaking from the
+        // prior context to the current one.  This may actually be a major
+        // issue for dictionary-based wordbreaking!
+        //
+        // If there was such a shift, then we may have extra transforms
+        // originally on a 'previous' token that got moved into this one!
+        //
+        // Suppose we're using a dictionary-based wordbreaker and have
+        // `butterfl` for our context, which could become butterfly.  If the
+        // next keystroke results in `butterfli`, this would likely be
+        // tokenized `butter` `fli`.  (e.g: `fli` leads to `flight`.) How do
+        // we know to properly relocate the `f` and `l` transforms?
+        let tokenDistribComponent = tokenDistribution.map((seq) => {
+          const entry = seq[tailIndex];
+          if(!entry || TransformUtils.isEmpty(entry.sample)) {
+            return null;
+          } else {
+            return entry;
+          }
+        }).filter((entry) => !!entry);
+        if(primaryInput) {
+          pushedToken.transformDistributions = tokenDistribComponent.length > 0 ? [tokenDistribComponent] : [];
+        } else if(incomingToken.text) {
+          // We have no transform data to match against an inserted token with text; abort!
+          // Refer to #12494 for an example case; we currently can't map previously-committed
+          // input transforms to a newly split-off token.
+          return null;
+        }
+        pushedToken.isWhitespace = incomingToken.isWhitespace;
+
+        // Auto-replaces the search space to correspond with the new token.
+        state.pushTail(pushedToken);
+
+        tailIndex++;
+      }
     }
 
     return {
       state,
       baseState: matchState,
       preservationTransform,
-      headTokensRemoved: poppedTokenCount,
-      tailTokensAdded: pushedTokenCount
+      headTokensRemoved: alignmentResults.leadTokenShift < 0 ? -alignmentResults.leadTokenShift : 0,
+      tailTokensAdded: alignmentResults.tailTokenShift
     };
   }
 
@@ -997,6 +1023,15 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
     if(inputTransform) {
       // These two methods apply transforms internally; do not mutate context here.
       // This particularly matters for the 'distribution' variant.
+
+      // What if a pre-whitespace token has a final substitution as PART of an edit?
+      // Say, ['apple', ' ', ''] => ['apply', ' ', 'n']
+      // For now... we can't really handle that case well - modeling the 'e' => 'y' part.
+      // Will likely require improvements to tokenizeTransform(), which doesn't yet handle
+      // deleteLeft tokenization for transforms spanning tokens & whitespace.
+      //
+      // See: #14361.
+      // There's a good shot attemptTokenizedAlignment would be useful for it.
       transformTokenLength = tokenizeTransform(tokenize, context, inputTransform.sample).length;
       tokenizedDistribution = tokenizeTransformDistribution(tokenize, context, transformDistribution);
 
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
index a3845b0e2d..d0197e610b 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
@@ -518,7 +518,7 @@ describe('ContextTracker', function() {
 
       // The 'wordbreak' transform
       assert.equal(newContextMatch.headTokensRemoved, 0);
-      assert.equal(newContextMatch.tailTokensAdded, 0);
+      assert.equal(newContextMatch.tailTokensAdded, -2);
     });
 
     it("properly matches and aligns when an implied 'wordbreak' occurs (as when following \"'\")", function() {

From cd59e1fffb863d5622261530baba550fa5a30ac8 Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Mon, 21 Jul 2025 22:12:17 +0700
Subject: [PATCH 12/53] fix(web): correct typo in unit test name

Co-authored-by: Eberhard Beilharz <ermshiperete@users.noreply.github.com>
---
 .../src/tests/mocha/cases/edit-distance/context-tracker.js      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
index 65b16c5fc2..da3d0bdef4 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
@@ -90,7 +90,7 @@ describe('ContextTracker', function() {
     });
 
     // random deletion at the start + later substitution = still permitted
-    it(`returns false:  'clasts' => 'clasps' (subs allowed)`, () => {
+    it(`returns false:  'clasts' => 'lasps' (subs allowed)`, () => {
       assert.isTrue(ContextTracker.isSubstitutionAlignable('lasps', 'clasts', true));
     });
   });

From f264fca881924e2505dfef510d740054dd9cefdd Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Mon, 21 Jul 2025 15:52:03 -0500
Subject: [PATCH 13/53] change(web): split suggestion transforms into main-body
 + appended (whitespace) transforms

---
 .../i_got_distracted_by_hazel.json            | 72 +++++++++++++++----
 common/web/types/src/lexical-model-types.ts   | 11 ++-
 .../main/src/headless/languageProcessor.ts    |  6 ++
 .../src/main/model-compositor.ts              |  5 +-
 .../worker-thread/src/main/predict-helpers.ts | 66 ++++++++++-------
 .../mocha/cases/suggestion-finalization.js    | 25 +++++--
 .../mocha/cases/worker-custom-punctuation.js  |  2 +-
 .../mocha/cases/worker-model-compositor.js    | 13 ++--
 .../main/headless/languageProcessor.tests.js  | 27 ++++---
 9 files changed, 167 insertions(+), 60 deletions(-)

diff --git a/common/test/resources/json/models/future_suggestions/i_got_distracted_by_hazel.json b/common/test/resources/json/models/future_suggestions/i_got_distracted_by_hazel.json
index d01c7f5c5a..1c4dea5050 100644
--- a/common/test/resources/json/models/future_suggestions/i_got_distracted_by_hazel.json
+++ b/common/test/resources/json/models/future_suggestions/i_got_distracted_by_hazel.json
@@ -2,21 +2,33 @@
   [
     {
       "transform": {
-        "insert": "I ",
+        "insert": "I",
+        "deleteLeft": 0
+      },
+      "appendedTransform": {
+        "insert": " ",
         "deleteLeft": 0
       },
       "displayAs": "I"
     },
     {
       "transform": {
-        "insert": "I'm ",
+        "insert": "I'm",
+        "deleteLeft": 0
+      },
+      "appendedTransform": {
+        "insert": " ",
         "deleteLeft": 0
       },
       "displayAs": "I'm"
     },
     {
       "transform": {
-        "insert": "Oh ",
+        "insert": "Oh",
+        "deleteLeft": 0
+      },
+      "appendedTransform": {
+        "insert": " ",
         "deleteLeft": 0
       },
       "displayAs": "Oh"
@@ -25,21 +37,33 @@
   [
     {
       "transform": {
-        "insert": "love ",
+        "insert": "love",
+        "deleteLeft": 0
+      },
+      "appendedTransform": {
+        "insert": " ",
         "deleteLeft": 0
       },
       "displayAs": "love"
     },
     {
       "transform": {
-        "insert": "am ",
+        "insert": "am",
+        "deleteLeft": 0
+      },
+      "appendedTransform": {
+        "insert": " ",
         "deleteLeft": 0
       },
       "displayAs": "am"
     },
     {
       "transform": {
-        "insert": "got ",
+        "insert": "got",
+        "deleteLeft": 0
+      },
+      "appendedTransform": {
+        "insert": " ",
         "deleteLeft": 0
       },
       "displayAs": "got"
@@ -48,21 +72,33 @@
   [
     {
       "transform": {
-        "insert": "distracted by ",
+        "insert": "distracted by",
+        "deleteLeft": 0
+      },
+      "appendedTransform": {
+        "insert": " ",
         "deleteLeft": 0
       },
       "displayAs": "distracted by"
     },
     {
       "transform": {
-        "insert": "distracted ",
+        "insert": "distracted",
+        "deleteLeft": 0
+      },
+      "appendedTransform": {
+        "insert": " ",
         "deleteLeft": 0
       },
       "displayAs": "distracted"
     },
     {
       "transform": {
-        "insert": "a ",
+        "insert": "a",
+        "deleteLeft": 0
+      },
+      "appendedTransform": {
+        "insert": " ",
         "deleteLeft": 0
       },
       "displayAs": "a"
@@ -71,21 +107,33 @@
   [
     {
       "transform": {
-        "insert": "Hazel ",
+        "insert": "Hazel",
+        "deleteLeft": 0
+      },
+      "appendedTransform": {
+        "insert": " ",
         "deleteLeft": 0
       },
       "displayAs": "Hazel"
     },
     {
       "transform": {
-        "insert": "the ",
+        "insert": "the",
+        "deleteLeft": 0
+      },
+      "appendedTransform": {
+        "insert": " ",
         "deleteLeft": 0
       },
       "displayAs": "the"
     },
     {
       "transform": {
-        "insert": "a ",
+        "insert": "a",
+        "deleteLeft": 0
+      },
+      "appendedTransform": {
+        "insert": " ",
         "deleteLeft": 0
       },
       "displayAs": "a"
diff --git a/common/web/types/src/lexical-model-types.ts b/common/web/types/src/lexical-model-types.ts
index e2670d2e21..1b02b413d1 100644
--- a/common/web/types/src/lexical-model-types.ts
+++ b/common/web/types/src/lexical-model-types.ts
@@ -303,11 +303,18 @@ export interface Suggestion {
   id?: number;
 
   /**
-   * The suggested update to the buffer. Note that this transform should
-   * be applied AFTER the instigating transform, if any.
+   * Specifies the edits needed to correct and extend the currently-edited word
+   * (within the text buffer) to match the suggested word from the lexicon.
+   * Note that this transform should be applied BEFORE the instigating transform, if any.
    */
   readonly transform: Transform;
 
+  /**
+   * Applies extra language-appropriate whitespace and/or punctuation after the main
+   * Suggestion body as specified by the source LexicalModel.
+   */
+  appendedTransform?: Transform;
+
   /**
    * A string to display the suggestion to the typist.
    * This should aid the typist understand what the transform
diff --git a/web/src/engine/main/src/headless/languageProcessor.ts b/web/src/engine/main/src/headless/languageProcessor.ts
index 064d7f381a..2921396b5e 100644
--- a/web/src/engine/main/src/headless/languageProcessor.ts
+++ b/web/src/engine/main/src/headless/languageProcessor.ts
@@ -216,6 +216,9 @@ export class LanguageProcessor extends EventEmitter<LanguageProcessorEventMap> {
       // Step 1:  determine the final output text
       const final = Mock.from(original.preInput, false);
       final.apply(suggestion.transform);
+      if(suggestion.appendedTransform) {
+        final.apply(suggestion.appendedTransform);
+      }
 
       // Step 2:  build a final, master Transform that will produce the desired results from the CURRENT state.
       // In embedded mode, both Android and iOS are best served by calculating this transform and applying its
@@ -289,6 +292,9 @@ export class LanguageProcessor extends EventEmitter<LanguageProcessorEventMap> {
     // Step 1:  determine the final output text
     const final = Mock.from(original.preInput, false);
     final.apply(reversion.transform); // Should match original.transform, actually. (See applySuggestion)
+    if(reversion.appendedTransform) {
+      final.apply(reversion.appendedTransform);
+    }
 
     // Step 2:  build a final, master Transform that will produce the desired results from the CURRENT state.
     // In embedded mode, both Android and iOS are best served by calculating this transform and applying its
diff --git a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts
index 05da67ec98..164eaaba7f 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts
@@ -209,7 +209,7 @@ export class ModelCompositor {
 
   acceptSuggestion(suggestion: Suggestion, context: Context, postTransform?: Transform): Reversion {
     // Step 1:  generate and save the reversion's Transform.
-    let sourceTransform = suggestion.transform;
+    let sourceTransform = models.buildMergedTransform(suggestion.transform, suggestion.appendedTransform ?? { insert: '', deleteLeft: 0});
     let deletedLeftChars = KMWString.substr(context.left, -sourceTransform.deleteLeft, sourceTransform.deleteLeft);
     let insertedLength = KMWString.length(sourceTransform.insert);
 
@@ -270,6 +270,9 @@ export class ModelCompositor {
 
       contextState.tail.activeReplacementId = suggestion.id;
       let acceptedContext = models.applyTransform(suggestion.transform, context);
+      if(suggestion.appendedTransform) {
+        acceptedContext = models.applyTransform(suggestion.appendedTransform, context);
+      }
       this.contextTracker.analyzeState(this.lexicalModel, acceptedContext);
     }
 
diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts
index 961c1e18bc..33c81ea6c3 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts
@@ -624,12 +624,12 @@ export function processSimilarity(
  * unexpected ways.  For example, when typing numbers in English, we don't expect
  * '5' to auto-correct to '5th' just because there are no pure-number entries in
  * the lexicon rooted on '5'.
- * @param correction 
- * @returns 
+ * @param correction
+ * @returns
  */
 export function correctionValidForAutoSelect(correction: string) {
   let chars = [...correction];
-  
+
   // If the _correction_ - the actual, existing text - does not include any letters,
   // then predictions built upon it should not be considered valid for auto-correction.
   for(let c of chars) {
@@ -790,32 +790,42 @@ export function finalizeSuggestions(
     }
   });
 
-  // Apply 'after word' punctuation and other post-processing, setting suggestion IDs.
-  // We delay until now so that utility functions relying on the unmodified Transform may execute properly.
-  suggestions.forEach((suggestion) => {
-    // Valid 'keep' suggestions may have zero length; we still need to evaluate the following code
-    // for such cases.
+  if(punctuation.insertAfterWord !== "") {
+    // Apply 'after word' punctuation and other post-processing, setting suggestion IDs.
+    // We delay until now so that utility functions relying on the unmodified Transform may execute properly.
+    suggestions.forEach((suggestion) => {
+      // Valid 'keep' suggestions may have zero length; we still need to evaluate the following code
+      // for such cases.
 
-    // If we're mid-word, delete its original post-caret text.
-    const tokenization = tokenize(context);
-    if(tokenization && tokenization.caretSplitsToken) {
-      // While we wait on the ability to provide a more 'ideal' solution, let's at least
-      // go with a more stable, if slightly less ideal, solution for now.
-      //
-      // A predictive text default (on iOS, at least) - immediately wordbreak
-      // on suggestions accepted mid-word.
-      suggestion.transform.insert += punctuation.insertAfterWord;
+      // If we're mid-word, delete its original post-caret text.
+      const tokenization = tokenize(context);
+      if(tokenization && tokenization.caretSplitsToken) {
+        // While we wait on the ability to provide a more 'ideal' solution, let's at least
+        // go with a more stable, if slightly less ideal, solution for now.
+        //
+        // A predictive text default (on iOS, at least) - immediately wordbreak
+        // on suggestions accepted mid-word.
+        suggestion.appendedTransform = {
+          insert: punctuation.insertAfterWord,
+          deleteLeft: 0
+        };
 
-      // Do we need to manipulate the suggestion's transform based on the current state of the context?
-    } else if(!context.right) {
-      suggestion.transform.insert += punctuation.insertAfterWord;
-    } else if(punctuation.insertAfterWord != '') {
-      if(context.right.indexOf(punctuation.insertAfterWord) != 0) {
-        suggestion.transform.insert += punctuation.insertAfterWord;
+        // Do we need to manipulate the suggestion's transform based on the current state of the context?
+      } else if(!context.right) {
+        suggestion.appendedTransform = {
+          insert: punctuation.insertAfterWord,
+          deleteLeft: 0
+        };
+      } else if(punctuation.insertAfterWord != '') {
+        if(context.right.indexOf(punctuation.insertAfterWord) != 0) {
+          suggestion.appendedTransform = {
+            insert: punctuation.insertAfterWord,
+          deleteLeft: 0
+          };
+        }
       }
-    }
-
-  });
+    });
+  };
 
   return suggestions;
 }
@@ -860,6 +870,10 @@ export function toAnnotatedSuggestion(
     p: suggestion.p
   };
 
+  if(suggestion.appendedTransform) {
+    result.appendedTransform = suggestion.appendedTransform;
+  }
+
   if(suggestion.transformId !== undefined) {
     result.transformId = suggestion.transformId;
   }
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/suggestion-finalization.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/suggestion-finalization.js
index a1f7e65f17..1cf079fda3 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/suggestion-finalization.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/suggestion-finalization.js
@@ -165,7 +165,10 @@ describe('finalizeSuggestions', () => {
       const { unfinalized, expected } = build_its_is_set();
       const finalized = finalizeSuggestions(testModelWithSpacing, unfinalized, context, transform, false);
 
-      expected.forEach((entry) => entry.transform.insert += testModelWithSpacing.punctuation.insertAfterWord);
+      expected.forEach((entry) => entry.appendedTransform = {
+        insert: testModelWithSpacing.punctuation.insertAfterWord,
+        deleteLeft: 0
+      });
       assert.sameDeepOrderedMembers(finalized, expected);
     });
 
@@ -214,7 +217,10 @@ describe('finalizeSuggestions', () => {
 
       // The character after the caret isn't the whitespace we'd usually insert,
       // so we don't swallow it this time.
-      expected.forEach((entry) => entry.transform.insert += testModelWithSpacing.punctuation.insertAfterWord);
+      expected.forEach((entry) => entry.appendedTransform = {
+        insert: testModelWithSpacing.punctuation.insertAfterWord,
+        deleteLeft: 0
+      });
       assert.sameDeepOrderedMembers(finalized, expected);
     });
 
@@ -239,7 +245,10 @@ describe('finalizeSuggestions', () => {
       const { unfinalized, expected } = build_its_is_set();
       const finalized = finalizeSuggestions(testModelWithSpacing, unfinalized, context, transform, false);
 
-      expected.forEach((entry) => entry.transform.insert += testModelWithSpacing.punctuation.insertAfterWord);
+      expected.forEach((entry) => entry.appendedTransform = {
+        insert: testModelWithSpacing.punctuation.insertAfterWord,
+        deleteLeft: 0
+      });
       assert.sameDeepOrderedMembers(finalized, expected);
     });
   });
@@ -338,7 +347,10 @@ describe('finalizeSuggestions', () => {
       const { unfinalized, expected } = build_its_is_set('verbose');
       const finalized = finalizeSuggestions(testModelWithSpacing, unfinalized, context, transform, /* verbose */ true);
 
-      expected.forEach((entry) => entry.transform.insert += testModelWithSpacing.punctuation.insertAfterWord);
+      expected.forEach((entry) => entry.appendedTransform = {
+        insert: testModelWithSpacing.punctuation.insertAfterWord,
+        deleteLeft: 0
+      });
       assert.sameDeepOrderedMembers(finalized, expected);
     });
 
@@ -361,7 +373,10 @@ describe('finalizeSuggestions', () => {
       const { unfinalized, expected } = build_its_is_set();
       const finalized = finalizeSuggestions(testModelWithSpacing, unfinalized, context, transform, false);
 
-      expected.forEach((entry) => entry.transform.insert += testModelWithSpacing.punctuation.insertAfterWord);
+      expected.forEach((entry) => entry.appendedTransform = {
+        insert: testModelWithSpacing.punctuation.insertAfterWord,
+        deleteLeft: 0
+      });
       assert.sameDeepOrderedMembers(finalized, expected);
     });
   });
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/worker-custom-punctuation.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/worker-custom-punctuation.js
index 3982bdfe7a..ee027b907a 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/worker-custom-punctuation.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/worker-custom-punctuation.js
@@ -89,7 +89,7 @@ describe('Custom Punctuation', function () {
 
       // Check that it has been changed:
       for (var i = 0; i < dummySuggestions.length; i++) {
-        assert.isTrue(suggestions[i].transform.insert.endsWith(' '));
+        assert.isTrue(suggestions[i].appendedTransform.insert == ' ');
       }
     });
   })
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/worker-model-compositor.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/worker-model-compositor.js
index cc24991247..30e7bf5188 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/worker-model-compositor.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/worker-model-compositor.js
@@ -90,13 +90,14 @@ describe('ModelCompositor', function() {
         });
 
         assert.isDefined(keep);
-        assert.equal(keep.transform.insert, 'the ');
+        assert.equal(keep.transform.insert, 'the');
+        assert.isDefined(keep.appendedTransform?.insert, ' ');
 
         // Expect an appended space.
-        let expectedEntries = ['they ', 'there ', 'their ', 'these ', 'themselves '];
+        let expectedEntries = ['they', 'there', 'their', 'these', 'themselves'];
         expectedEntries.forEach(function(entry) {
           assert.isDefined(suggestions.find(function(suggestion) {
-            return suggestion.transform.insert == entry;
+            return suggestion.transform.insert == entry && suggestion.appendedTransform?.insert == ' ';
           }));
         });
       });
@@ -979,10 +980,14 @@ describe('ModelCompositor', function() {
       assert.equal(suggestions.length, 1);
 
       let expectedTransform = {
-        insert: 'hi ',  // Keeps current context the same, though it adds a wordbreak.
+        insert: 'hi',  // Keeps current context the same, though it adds a wordbreak.
         deleteLeft: 2
       }
       assert.deepEqual(suggestions[0].transform, expectedTransform);
+      assert.deepEqual(suggestions[0].appendedTransform, {
+        insert: ' ',
+        deleteLeft: 0
+      });
     });
 
     it('model with traversals: returns appropriate suggestions upon reversion', async function() {
diff --git a/web/src/test/auto/headless/engine/main/headless/languageProcessor.tests.js b/web/src/test/auto/headless/engine/main/headless/languageProcessor.tests.js
index 6d866ef8ed..a02f1b7474 100644
--- a/web/src/test/auto/headless/engine/main/headless/languageProcessor.tests.js
+++ b/web/src/test/auto/headless/engine/main/headless/languageProcessor.tests.js
@@ -117,9 +117,11 @@ describe('LanguageProcessor', function() {
           languageProcessor.predict(transcription).then(function(suggestions) {
             assert.isOk(suggestions);
             assert.equal(suggestions[0].displayAs, '«li»');
-            assert.equal(suggestions[0].transform.insert, 'li ');
+            assert.equal(suggestions[0].transform.insert, 'li');
+            assert.equal(suggestions[0].appendedTransform.insert, ' ');
             assert.equal(suggestions[1].displayAs, 'like');
-            assert.equal(suggestions[1].transform.insert, 'like ');
+            assert.equal(suggestions[1].transform.insert, 'like');
+            assert.equal(suggestions[1].appendedTransform.insert, ' ');
             done();
           }).catch(done);
         }).catch(function() {
@@ -154,7 +156,8 @@ describe('LanguageProcessor', function() {
               languageProcessor.predict(transcription).then(function(suggestions) {
                 assert.isOk(suggestions);
                 assert.equal(suggestions[1].displayAs, 'like');
-                assert.equal(suggestions[1].transform.insert, 'like ');
+                assert.equal(suggestions[1].transform.insert, 'like');
+                assert.equal(suggestions[1].appendedTransform.insert, ' ');
                 done();
               }).catch(done);
             }).catch(function() {
@@ -172,7 +175,8 @@ describe('LanguageProcessor', function() {
                 // The source suggestion is simply 'like'.
                 assert.isOk(suggestions);
                 assert.equal(suggestions[1].displayAs, 'like');
-                assert.equal(suggestions[1].transform.insert, 'like ');
+                assert.equal(suggestions[1].transform.insert, 'like');
+                assert.equal(suggestions[1].appendedTransform.insert, ' ');
                 done();
               }).catch(done);
             }).catch(function() {
@@ -189,7 +193,8 @@ describe('LanguageProcessor', function() {
               languageProcessor.predict(transcription).then(function(suggestions) {
                 assert.isOk(suggestions);
                 assert.equal(suggestions[1].displayAs, 'I');
-                assert.equal(suggestions[1].transform.insert, 'I ');
+                assert.equal(suggestions[1].transform.insert, 'I');
+                assert.equal(suggestions[1].appendedTransform.insert, ' ');
                 done();
               }).catch(done);
             }).catch(function() {
@@ -209,7 +214,8 @@ describe('LanguageProcessor', function() {
                 // The source suggestion is simply 'like'.
                 assert.isOk(suggestions);
                 assert.equal(suggestions[1].displayAs, 'LIKE');
-                assert.equal(suggestions[1].transform.insert, 'LIKE ');
+                assert.equal(suggestions[1].transform.insert, 'LIKE');
+                assert.equal(suggestions[1].appendedTransform.insert, ' ');
                 done();
               }).catch(done);
             }).catch(function() {
@@ -226,7 +232,8 @@ describe('LanguageProcessor', function() {
               languageProcessor.predict(transcription).then(function(suggestions) {
                 assert.isOk(suggestions);
                 assert.equal(suggestions[0].displayAs, 'I');
-                assert.equal(suggestions[0].transform.insert, 'I ');
+                assert.equal(suggestions[0].transform.insert, 'I');
+                assert.equal(suggestions[0].appendedTransform.insert, ' ');
                 done();
               }).catch(done);
             }).catch(function() {
@@ -247,7 +254,8 @@ describe('LanguageProcessor', function() {
                   // The source suggestion is simply 'like'.
                   assert.isOk(suggestions);
                   assert.equal(suggestions[1].displayAs, 'Like');
-                  assert.equal(suggestions[1].transform.insert, 'Like ');
+                  assert.equal(suggestions[1].transform.insert, 'Like');
+                assert.equal(suggestions[1].appendedTransform.insert, ' ');
                   done();
                 }).catch(done);
               }).catch(function() {
@@ -267,7 +275,8 @@ describe('LanguageProcessor', function() {
                   // The source suggestion is simply 'like'.
                   assert.isOk(suggestions);
                   assert.equal(suggestions[1].displayAs, 'Like');
-                  assert.equal(suggestions[1].transform.insert, 'Like ');
+                  assert.equal(suggestions[1].transform.insert, 'Like');
+                  assert.equal(suggestions[1].appendedTransform.insert, ' ');
                   done();
                 }).catch(done);
               }).catch(function() {

From 0f3130fbd463561dc5637ad0a253471bbf8fb784 Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Tue, 22 Jul 2025 23:48:37 +0700
Subject: [PATCH 14/53] fix(web): properly apply Transforms in sequence

Thanks to a PR review that caught an accidental restart that hid one of the Transforms

Co-authored-by: Eberhard Beilharz <ermshiperete@users.noreply.github.com>
---
 .../predictive-text/worker-thread/src/main/model-compositor.ts  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts
index 164eaaba7f..754593cc92 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts
@@ -271,7 +271,7 @@ export class ModelCompositor {
       contextState.tail.activeReplacementId = suggestion.id;
       let acceptedContext = models.applyTransform(suggestion.transform, context);
       if(suggestion.appendedTransform) {
-        acceptedContext = models.applyTransform(suggestion.appendedTransform, context);
+        acceptedContext = models.applyTransform(suggestion.appendedTransform, acceptedContext);
       }
       this.contextTracker.analyzeState(this.lexicalModel, acceptedContext);
     }

From 7e88fa2c410009a8cc3a740601713d29283279fd Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Wed, 23 Jul 2025 13:43:26 -0500
Subject: [PATCH 15/53] docs(web): enhance doc-comment for getEditPathLastMatch

Per review request by @ermshiperete
---
 .../worker-thread/src/main/correction/context-tracker.ts  | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
index 581ed948b7..68528c87bc 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
@@ -37,8 +37,12 @@ function textToCharTransforms(text: string, transformId?: number) {
  * In particular, this method is designed to handle the following case:
  * ['to', 'apple', ' ', ''] => ['to', 'apply', ' ', 'n']
  *
- * The ' ' is unedited, but as it follows the edited 'apple' => 'apply', 'to' is the true
- * "last edited" token.
+ * Edit path for this example case:
+ * ['match', 'substitute', 'match', 'substitute']
+ *
+ * In cases such as these, the whitespace match should be considered 'edited'. Whil the ' '
+ * is unedited, it follows the edited 'apple' => 'apply', so it must have been deleted and
+ * then re-inserted.  As a result, 'to' is the true "last matched" token.
  * @param editPath
  * @returns
  */

From 3c36ee2e736f9ae548c6784bb5dfcf3666c5d3db Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Thu, 24 Jul 2025 20:38:03 +0700
Subject: [PATCH 16/53] change(web): fix typo (per PR review)

Co-authored-by: Eberhard Beilharz <ermshiperete@users.noreply.github.com>
---
 .../worker-thread/src/main/correction/context-tracker.ts        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
index 68528c87bc..e87b6061e4 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
@@ -40,7 +40,7 @@ function textToCharTransforms(text: string, transformId?: number) {
  * Edit path for this example case:
  * ['match', 'substitute', 'match', 'substitute']
  *
- * In cases such as these, the whitespace match should be considered 'edited'. Whil the ' '
+ * In cases such as these, the whitespace match should be considered 'edited'. While the ' '
  * is unedited, it follows the edited 'apple' => 'apply', so it must have been deleted and
  * then re-inserted.  As a result, 'to' is the true "last matched" token.
  * @param editPath

From 0ceffce32e5c31b3af7a6df0f4eb129c928ad698 Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Wed, 30 Jul 2025 09:15:27 -0500
Subject: [PATCH 17/53] refactor(web): refactor ContextToken and usage pattern

Also places the class within its own separate source file.

This PR's changes aim to meet our _current_ needs while moving much closer to the design specified within the [Correction-Search + Context-Tracking design doc](https://docs.google.com/document/d/1RYwgW8zI8A7VLMq40BpWYkfJJQ-i6K0MIs4QcAMxglE/edit?tab=t.0).
---
 .../src/main/correction/context-token.ts      |  96 ++++++++
 .../src/main/correction/context-tracker.ts    | 216 +++---------------
 .../src/main/correction/distance-modeler.ts   |   6 +-
 .../src/main/model-compositor.ts              |  19 +-
 .../worker-thread/src/main/predict-helpers.ts |   6 +-
 .../cases/edit-distance/context-token.js      | 101 ++++++++
 .../cases/edit-distance/context-tracker.js    |  78 ++++---
 .../mocha/cases/worker-model-compositor.js    |   4 +-
 8 files changed, 286 insertions(+), 240 deletions(-)
 create mode 100644 web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts
 create mode 100644 web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-token.js

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts
new file mode 100644
index 0000000000..d886434932
--- /dev/null
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts
@@ -0,0 +1,96 @@
+import { buildMergedTransform } from "@keymanapp/models-templates";
+import { textToCharTransforms } from "./context-tracker.js";
+import { SearchSpace } from "./distance-modeler.js";
+
+import { LexicalModelTypes } from '@keymanapp/common-types';
+import Distribution = LexicalModelTypes.Distribution;
+import LexicalModel = LexicalModelTypes.LexicalModel;
+import Suggestion = LexicalModelTypes.Suggestion;
+import Transform = LexicalModelTypes.Transform;
+
+export class ContextToken {
+  /**
+   * Indicates whether or not the token is considered whitespace.
+   */
+  isWhitespace: boolean;
+
+  /**
+   * Contains all relevant correction-search data for use in generating
+   * corrections for this ContextToken instance.
+   */
+  readonly searchSpace: SearchSpace;
+
+  /* The next two fields will **not land here** in the final version for
+     epic/autocorrect / 19.0-beta!  That said, their future location has
+     not yet been reworked, so we'll keep them here for now. */
+
+  /**
+   * The set of suggestions generated for the current token
+   */
+  suggestions: Suggestion[];
+
+  /**
+   * The ID of the suggestion applied to the current token, if any.
+   *
+   * Is set to -1 when no such suggestion exists.
+   */
+  appliedSuggestionId: number = -1;
+
+  /**
+   * Constructs a new, empty instance for use with the specified LexicalModel.
+   * @param model
+   */
+  constructor(model: LexicalModel);
+  /**
+   * Constructs a new instance with pre-existing text for use with the specified LexicalModel.
+   * @param model
+   * @param rawText
+   */
+  constructor(model: LexicalModel, rawText: string);
+  /**
+   * This constructor deep-copies the specified instance.
+   * @param baseToken
+   */
+  constructor(baseToken: ContextToken);
+  constructor(param: ContextToken | LexicalModel, rawText?: string) {
+    if(param instanceof ContextToken) {
+      const priorToken = param;
+      this.isWhitespace = priorToken.isWhitespace;
+
+      // We need to construct a separate search space from other token copies.
+      //
+      // In case we are unable to perfectly track context (say, due to multitaps)
+      // we need to ensure that only fully-utilized keystrokes are considered.
+      this.searchSpace = new SearchSpace(priorToken.searchSpace);
+      this.suggestions = priorToken.suggestions.slice();
+
+      this.appliedSuggestionId = priorToken.appliedSuggestionId;
+    } else {
+      const model = param;
+
+      // May be altered outside of the constructor.
+      this.isWhitespace = false;
+      this.searchSpace = new SearchSpace(model);
+
+      rawText ||= '';
+
+      // Supports the old pathway for: updateWithBackspace(tokenText: string, transformId: number)
+      const rawTransformDistributions: Distribution<Transform>[] = textToCharTransforms(rawText).map(function(transform) {
+        return [{sample: transform, p: 1.0}];
+      });
+      rawTransformDistributions.forEach((entry) => this.searchSpace.addInput(entry));
+
+      this.suggestions = [];
+    }
+  }
+
+  /**
+   * Displays text corresponding to the net effects of the most likely inputs received
+   * that can correspond to the current instance.
+   */
+  get exampleInput(): string {
+    const transforms = this.searchSpace.inputSequence.map((dist) => dist[0].sample)
+    const composite = transforms.reduce((accum, current) => buildMergedTransform(accum, current), { insert: '', deleteLeft: 0});
+    return composite.insert;
+  }
+}
\ No newline at end of file
diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
index e87b6061e4..625d41e672 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
@@ -2,7 +2,6 @@ import { applyTransform, buildMergedTransform, Token } from '@keymanapp/models-t
 import { KMWString } from '@keymanapp/web-utils';
 
 import { ClassicalDistanceCalculation, EditOperation } from './classical-calculation.js';
-import { SearchSpace } from './distance-modeler.js';
 import TransformUtils from '../transformUtils.js';
 import { determineModelTokenizer } from '../model-helpers.js';
 import { tokenizeTransform, tokenizeTransformDistribution } from './transform-tokenization.js';
@@ -12,8 +11,9 @@ import Distribution = LexicalModelTypes.Distribution;
 import LexicalModel = LexicalModelTypes.LexicalModel;
 import Suggestion = LexicalModelTypes.Suggestion;
 import Transform = LexicalModelTypes.Transform;
+import { ContextToken } from './context-token.js';
 
-function textToCharTransforms(text: string, transformId?: number) {
+export function textToCharTransforms(text: string, transformId?: number) {
   let perCharTransforms: Transform[] = [];
 
   for(let i=0; i < KMWString.length(text); i++) {
@@ -21,10 +21,13 @@ function textToCharTransforms(text: string, transformId?: number) {
 
     let transform: Transform = {
       insert: char,
-      deleteLeft: 0,
-      id: transformId
+      deleteLeft: 0
     };
 
+    if(transformId) {
+      transform.id = transformId
+    }
+
     perCharTransforms.push(transform);
   }
 
@@ -63,165 +66,57 @@ export class TrackedContextSuggestion {
   tokenWidth: number;
 }
 
-export class TrackedContextToken {
-  raw: string;
-  replacementText: string;
-  isWhitespace?: boolean;
-
-  transformDistributions: Distribution<Transform>[] = [];
-  replacements: TrackedContextSuggestion[] = [];
-  activeReplacementId: number = -1;
-
-  constructor();
-  constructor(instance: TrackedContextToken);
-  constructor(instance?: TrackedContextToken) {
-    if(instance) {
-      Object.assign(this, instance);
-      // We don't alter the values in replacements, but we do wish to prevent aliasing
-      // of the array containing them.
-      this.replacements = instance.replacements.slice();
-    }
-  }
-
-  get currentText(): string {
-    if(this.replacementText === undefined || this.replacementText === null) {
-      return this.raw;
-    } else {
-      return this.replacementText;
-    }
-  }
-
-  get replacement(): TrackedContextSuggestion {
-    let replacementId = this.activeReplacementId;
-    return this.replacements.find(function(replacement) {
-      return replacement.suggestion.id == replacementId;
-    });
-  }
-
-  clearReplacements() {
-    this.activeReplacementId = -1;
-    this.replacements = []
-  }
-
-  /**
-   * Used for 14.0's backspace workaround, which flattens all previous Distribution<Transform>
-   * entries because of limitations with direct use of backspace transforms.
-   * @param tokenText
-   * @param transformId
-   */
-  updateWithBackspace(tokenText: string, transformId: number) {
-    // It's a backspace transform; time for special handling!
-    //
-    // For now, with 14.0, we simply compress all remaining Transforms for the token into
-    // multiple single-char transforms.  Probabalistically modeling BKSP is quite complex,
-    // so we simplify by assuming everything remaining after a BKSP is 'true' and 'intended' text.
-    //
-    // Note that we cannot just use a single, monolithic transform at this point b/c
-    // of our current edit-distance optimization strategy; diagonalization is currently...
-    // not very compatible with that.
-    let backspacedTokenContext: Distribution<Transform>[] = textToCharTransforms(tokenText, transformId).map(function(transform) {
-      return [{sample: transform, p: 1.0}];
-    });
-
-    this.raw = tokenText;
-    this.transformDistributions = backspacedTokenContext;
-    this.clearReplacements();
-  }
-
-  update(transformDistribution: Distribution<Transform>, tokenText?: string) {
-    // Preserve existing text if new text isn't specified.
-    tokenText = tokenText || (tokenText === '' ? '' : this.raw);
-
-    if(transformDistribution?.length > 0) {
-      this.transformDistributions.push(transformDistribution);
-    }
-
-    // Replace old token's raw-text with new token's raw-text.
-    this.raw = tokenText;
-    this.clearReplacements();
-  }
-}
-
 export class TrackedContextState {
   // Stores the post-transform Context.  Useful as a debugging reference, but also used to
   // pre-validate context state matches in case of discarded changes from multitaps.
   taggedContext: Context;
   model: LexicalModel;
 
-  tokens: TrackedContextToken[];
+  tokens: ContextToken[];
   /**
    * How many tokens were removed from the start of the best-matching ancestor.
    * Useful for restoring older states, e.g., when the user moves the caret backwards, we can recover the context at that position.
    */
   indexOffset: number;
 
-  // Tracks all search spaces starting at the current token.
-  // In the lm-layer's current form, this should only ever have one entry.
-  // Leaves 'design space' for if/when we add support for phrase-level corrections/predictions.
-  searchSpace: SearchSpace[] = [];
-
   constructor(source: TrackedContextState);
   constructor(model: LexicalModel);
   constructor(obj: TrackedContextState | LexicalModel) {
     if(obj instanceof TrackedContextState) {
       let source = obj;
       // Be sure to deep-copy the tokens!  Pointer-aliasing is bad here.
-      this.tokens = source.tokens.map(function(token) {
-        let copy = new TrackedContextToken();
-        Object.assign(copy, token);
-        copy.replacements = copy.replacements.slice();
-        copy.transformDistributions = copy.transformDistributions.slice();
-        return copy;
-      });
+      this.tokens = source.tokens.map((token) => new ContextToken(token));
 
       this.indexOffset = 0;
-      const lexicalModel = this.model = obj.model;
+      this.model = obj.model;
       this.taggedContext = obj.taggedContext;
-
-      if(lexicalModel?.traverseFromRoot) {
-        // We need to construct a separate search space from other ContextStates.
-        //
-        // In case we are unable to perfectly track context (say, due to multitaps)
-        // we need to ensure that only fully-utilized keystrokes are considered.
-        this.searchSpace = obj.searchSpace.map((space) => new SearchSpace(space));
-      }
     } else {
       let lexicalModel = obj;
       this.tokens = [];
       this.indexOffset = Number.MIN_SAFE_INTEGER;
       this.model = lexicalModel;
-
-      if(lexicalModel && lexicalModel.traverseFromRoot) {
-        this.searchSpace = [new SearchSpace(lexicalModel)];
-      }
     }
   }
 
-  get head(): TrackedContextToken {
+  get head(): ContextToken {
     return this.tokens[0];
   }
 
-  get tail(): TrackedContextToken {
+  get tail(): ContextToken {
     return this.tokens[this.tokens.length - 1];
   }
 
+  set tail(token: ContextToken) {
+    this.tokens[this.tokens.length - 1] = token;
+  }
+
   popHead() {
     this.tokens.splice(0, 1);
     this.indexOffset -= 1;
   }
 
-  pushTail(token: TrackedContextToken) {
-    if(this.model && this.model.traverseFromRoot) {
-      this.searchSpace = [new SearchSpace(this.model)]; // yeah, need to update SearchSpace for compatibility
-    } else {
-      this.searchSpace = [];
-    }
+  pushTail(token: ContextToken) {
     this.tokens.push(token);
-
-    let state = this;
-    if(state.searchSpace.length > 0) {
-      token.transformDistributions.forEach(distrib => state.searchSpace[0].addInput(distrib));
-    }
   }
 
   toRawTokenization() {
@@ -229,8 +124,8 @@ export class TrackedContextState {
 
     for(let token of this.tokens) {
       // Hide any tokens representing wordbreaks.  (Thinking ahead to phrase-level possibilities)
-      if(token.currentText !== null) {
-        sequence.push(token.currentText);
+      if(token.exampleInput !== null) {
+        sequence.push(token.exampleInput);
       }
     }
 
@@ -810,49 +705,17 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
       } else {
         preservationTransform = preservationTransform && primaryInput ? buildMergedTransform(preservationTransform, primaryInput) : (primaryInput ?? preservationTransform);
       }
-      state.tokens[incomingIndex] = matchedToken;
-      const token = matchedToken;
+      let token: ContextToken;
 
-      // TODO:  I'm beginning to believe that searchSpace should (eventually) be tracked
-      // on the tokens, rather than on the overall 'state'.
-      // - Reason:  phrase-level corrections / predictions would likely need a search-state
-      //   across per potentially-affected token.
-      // - Shifting the paradigm should be a separate work unit than the
-      //   context-tracker rework currently being done, though.
       if(isBackspace) {
-        token.updateWithBackspace(incomingToken.text, primaryInput.id);
-        if(isLastToken) {
-          state.tokens.pop(); // pops `token`
-          // puts it back in, rebuilding a fresh search-space that uses the rebuilt
-          // keystroke distribution from updateWithBackspace.
-          state.pushTail(token);
-        }
+        token = new ContextToken(matchState.model, incomingToken.text);
+        token.searchSpace.inputSequence.forEach((entry) => entry[0].sample.id = primaryInput.id);
       } else {
-        token.update(
-          tokenDistribution.map((seq) => seq[tailIndex]),
-          incomingToken.text
-        );
-
-        if(isLastToken) {
-          // Search spaces may not exist during some unit tests; the state
-          // may not have an associated model during some.
-          state.searchSpace[0]?.addInput(tokenDistribution.map((seq) => seq[tailIndex]));
-        }
-      }
-
-      // For this case, we were _likely_ called by
-      // ModelCompositor.acceptSuggestion(), which would have marked the
-      // accepted suggestion.
-      //
-      // Upon inspection, this doesn't seem entirely ideal.  It works for
-      // the common case, but not for specially crafted keystroke
-      // transforms.  That said, it's also very low impact.  Best as I can
-      // see, this is only really used for debugging info?
-      if(state != matchState && !isLastToken) {
-        // TODO:  eliminate
-        token.replacementText = incomingToken.text;
+        token = new ContextToken(matchedToken);
+        token.searchSpace.addInput(tokenDistribution.map((seq) => seq[tailIndex]));
       }
 
+      state.tokens[incomingIndex] = token;
       tailIndex++;
     }
 
@@ -901,8 +764,7 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
           state = new TrackedContextState(state);
         }
 
-        let pushedToken = new TrackedContextToken();
-        pushedToken.raw = incomingToken.text;
+        let pushedToken = new ContextToken(state.model);
 
         // TODO:  assumes that there was no shift in wordbreaking from the
         // prior context to the current one.  This may actually be a major
@@ -925,7 +787,10 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
           }
         }).filter((entry) => !!entry);
         if(primaryInput) {
-          pushedToken.transformDistributions = tokenDistribComponent.length > 0 ? [tokenDistribComponent] : [];
+          let transformDistribution = tokenDistribComponent.length > 0 ? tokenDistribComponent : null;
+          if(transformDistribution) {
+            pushedToken.searchSpace.addInput(transformDistribution);
+          }
         } else if(incomingToken.text) {
           // We have no transform data to match against an inserted token with text; abort!
           // Refer to #12494 for an example case; we currently can't map previously-committed
@@ -955,20 +820,12 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
     lexicalModel: LexicalModel
   ): TrackedContextState {
     let baseTokens = tokenizedContext.map(function(entry) {
-      let token = new TrackedContextToken();
-      token.raw = entry.text;
+      let token = new ContextToken(lexicalModel, entry.text);
+
       if(entry.isWhitespace) {
         token.isWhitespace = true;
       }
 
-      if(token.raw) {
-        token.transformDistributions = textToCharTransforms(token.raw).map(function(transform) {
-          return [{sample: transform, p: 1.0}];
-        });
-      } else {
-        // Helps model context-final wordbreaks.
-        token.transformDistributions = [];
-      }
       return token;
     });
 
@@ -976,18 +833,11 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
     let state = new TrackedContextState(lexicalModel);
 
     while(baseTokens.length > 0) {
-      // We don't have a pre-existing distribution for this token, so we'll build one as
-      // if we'd just produced the token from a backspace.
-      if(baseTokens.length == 1) {
-        baseTokens[0].updateWithBackspace(baseTokens[0].raw, null);
-      }
       state.pushTail(baseTokens.splice(0, 1)[0]);
     }
 
     if(state.tokens.length == 0) {
-      let token = new TrackedContextToken();
-      token.raw = '';
-
+      let token = new ContextToken(lexicalModel);
       state.pushTail(token);
     }
 
diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts
index 316d3bc3e4..f61b804b4a 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts
@@ -366,7 +366,7 @@ export class SearchSpace {
 
   private tierOrdering: SearchSpaceTier[] = [];
   private selectionQueue: PriorityQueue<SearchSpaceTier>;
-  private inputSequence: Distribution<Transform>[] = [];
+  inputSequence: Distribution<Transform>[] = [];
   private minInputCost: number[] = [];
   private rootNode: SearchNode;
 
@@ -411,9 +411,9 @@ export class SearchSpace {
 
     const model = arg1;
     if(!model) {
-      throw "The LexicalModel parameter must not be null / undefined.";
+      throw new Error("The LexicalModel parameter must not be null / undefined.");
     } else if(!model.traverseFromRoot) {
-      throw "The provided model does not implement the `traverseFromRoot` function, which is needed to support robust correction searching.";
+      throw new Error("The provided model does not implement the `traverseFromRoot` function, which is needed to support robust correction searching.");
     }
 
     this.selectionQueue = new PriorityQueue<SearchSpaceTier>(this.QUEUE_SPACE_COMPARATOR);
diff --git a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts
index 754593cc92..031931d566 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts
@@ -181,12 +181,7 @@ export class ModelCompositor {
     // Store the suggestions on the final token of the current context state (if it exists).
     // Or, once phrase-level suggestions are possible, on whichever token serves as each prediction's root.
     if(postContextState) {
-      postContextState.tail.replacements = suggestions.map(function(suggestion) {
-        return {
-          suggestion: suggestion,
-          tokenWidth: 1
-        }
-      });
+      postContextState.tail.suggestions = suggestions;
     }
 
     return suggestions;
@@ -268,7 +263,7 @@ export class ModelCompositor {
         contextState = this.contextTracker.analyzeState(this.lexicalModel, context).state;
       }
 
-      contextState.tail.activeReplacementId = suggestion.id;
+      contextState.tail.appliedSuggestionId = suggestion.id;
       let acceptedContext = models.applyTransform(suggestion.transform, context);
       if(suggestion.appendedTransform) {
         acceptedContext = models.applyTransform(suggestion.appendedTransform, acceptedContext);
@@ -307,7 +302,7 @@ export class ModelCompositor {
     for(let c = this.contextTracker.count - 1; c >= 0; c--) {
       let contextState = this.contextTracker.item(c);
 
-      if(contextState.tail.activeReplacementId == -reversion.id) {
+      if(contextState.tail.appliedSuggestionId == -reversion.id) {
         contextMatchFound = true;
         break;
       }
@@ -318,18 +313,16 @@ export class ModelCompositor {
     }
 
     // Remove all contexts more recent than the one we're reverting to.
-    while(this.contextTracker.newest.tail.activeReplacementId != -reversion.id) {
+    while(this.contextTracker.newest.tail.appliedSuggestionId != -reversion.id) {
       this.contextTracker.popNewest();
     }
 
-    this.contextTracker.newest.tail.activeReplacementId = -1;
+    this.contextTracker.newest.tail.appliedSuggestionId = -1;
 
     // Will need to be modified a bit if/when phrase-level suggestions are implemented.
     // Those will be tracked on the first token of the phrase, which won't be the tail
     // if they cover multiple tokens.
-    let suggestions = this.contextTracker.newest.tail.replacements.map(function(trackedSuggestion) {
-      return trackedSuggestion.suggestion;
-    });
+    let suggestions = this.contextTracker.newest.tail.suggestions;
 
     suggestions.forEach(function(suggestion) {
       // A reversion's transform ID is the additive inverse of its original suggestion;
diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts
index 33c81ea6c3..bc1c731939 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts
@@ -202,7 +202,7 @@ export async function correctAndEnumerate(
   // let's just note that right now, there will only ever be one.
   //
   // The 'eventual' logic will be significantly more complex, though still manageable.
-  let searchSpace = postContextState.searchSpace[0];
+  const searchSpace = postContextState.tail.searchSpace;
 
   // No matter the prediction, once we know the root of the prediction, we'll always 'replace' the
   // same amount of text.  We can handle this before the big 'prediction root' loop.
@@ -250,11 +250,11 @@ export async function correctAndEnumerate(
 
   // Did the wordbreaker (or similar) append a blank token before the caret?  If so,
   // preserve that by preventing corrections from triggering left-deletion.
-  if(tailToken.raw == '') {
+  if(tailToken.exampleInput == '') {
     deleteLeft = 0;
   }
 
-  const isTokenStart = tailToken.transformDistributions.length <= 1;
+  const isTokenStart = tailToken.searchSpace.inputSequence.length <= 1;
 
   // TODO:  whitespace, backspace filtering.  Do it here.
   //        Whitespace is probably fine, actually.  Less sure about backspace.
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-token.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-token.js
new file mode 100644
index 0000000000..4486ca3ded
--- /dev/null
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-token.js
@@ -0,0 +1,101 @@
+import { assert } from 'chai';
+
+import { ContextToken } from '#./correction/context-token.js';
+import { ExecutionTimer } from '#./correction/execution-timer.js';
+import * as models from '#./models/index.js';
+
+import { default as defaultBreaker } from '@keymanapp/models-wordbreakers';
+
+import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs';
+
+var TrieModel = models.TrieModel;
+
+var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'),
+  {wordBreaker: defaultBreaker});
+
+describe('ContextToken', function() {
+  describe("<constructor>", () => {
+    it("(model: LexicalModel)", async () => {
+      let token = new ContextToken(plainModel);
+
+      assert.isEmpty(token.searchSpace.inputSequence);
+      assert.isEmpty(token.exampleInput);
+      assert.isFalse(token.isWhitespace);
+      assert.isEmpty(token.suggestions);
+      assert.equal(token.appliedSuggestionId, -1);
+
+      // While searchSpace has no inputs, it _can_ match lexicon entries (via insertions).
+      let searchIterator = token.searchSpace.getBestMatches(new ExecutionTimer(Number.POSITIVE_INFINITY, Number.POSITIVE_INFINITY));
+      let firstEntry = await searchIterator.next();
+      assert.isFalse(firstEntry.done);
+    });
+
+    it("(model: LexicalModel, text: string)", () => {
+      let token = new ContextToken(plainModel, "and");
+
+      assert.isNotEmpty(token.searchSpace.inputSequence);
+
+      assert.equal(token.searchSpace.inputSequence.map((entry) => entry[0].sample.insert).join(''), 'and');
+      token.searchSpace.inputSequence.forEach((entry) => assert.equal(entry[0].sample.deleteLeft, 0));
+      assert.deepEqual(token.searchSpace.inputSequence, [..."and"].map((char) => {
+        return [{
+          sample: {
+            insert: char,
+            deleteLeft: 0
+          },
+          p: 1.0
+        }];
+      }));
+      assert.equal(token.exampleInput, 'and');
+
+      assert.isFalse(token.isWhitespace);
+
+      // Is only set with a different value later, outside the constructor.
+      assert.isEmpty(token.suggestions);
+      assert.equal(token.appliedSuggestionId, -1);
+    });
+
+    it("(token: ContextToken", () => {
+      // Same as in a test above, since we verified that it works correctly.
+      let baseToken = new ContextToken(plainModel, "and");
+      baseToken.suggestions = [
+        {
+          transform: {
+            insert: 'd ',
+            deleteLeft: 0
+          },
+          id: 37,
+          transformId: 1,
+          displayAs: '"and"',
+          tag: 'keep',
+          autoAccept: true
+        },
+        {
+          transform: {
+            insert: 'Andes ',
+            deleteLeft: 2
+          },
+          id: 38,
+          transformId: 1,
+          displayAs: 'Andes',
+        }
+      ]
+      baseToken.appliedSuggestionId = 37;
+
+      let clonedToken = new ContextToken(baseToken);
+
+      assert.notEqual(clonedToken.suggestions, baseToken.suggestions);
+      assert.deepEqual(clonedToken.suggestions, baseToken.suggestions);
+
+      assert.notEqual(clonedToken.searchSpace, baseToken.searchSpace);
+      // Deep equality on .searchSpace can't be directly checked due to the internal complexities involved.
+      // We CAN check for the most important members, though.
+      assert.notEqual(clonedToken.searchSpace.inputSequence, baseToken.searchSpace.inputSequence);
+      assert.deepEqual(clonedToken.searchSpace.inputSequence, baseToken.searchSpace.inputSequence);
+
+      assert.notEqual(clonedToken, baseToken);
+      // Perfectly deep-equal when we ignore .searchSpace.
+      assert.deepEqual({...clonedToken, searchSpace: null}, {...baseToken, searchSpace: null});
+    });
+  });
+});
\ No newline at end of file
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
index 513d99ee82..bc862ba651 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
@@ -4,6 +4,7 @@ import { ContextTracker } from '#./correction/context-tracker.js';
 import { tokenizeTransformDistribution } from '#./correction/transform-tokenization.js';
 import ModelCompositor from '#./model-compositor.js';
 import * as models from '#./models/index.js';
+import * as wordBreakers from '@keymanapp/models-wordbreakers';
 import { determineModelTokenizer } from '#./model-helpers.js';
 
 import { default as defaultBreaker } from '@keymanapp/models-wordbreakers';
@@ -13,6 +14,11 @@ import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'
 
 const tokenizer = determineModelTokenizer(new models.DummyModel({wordbreaker: defaultBreaker}));
 
+var TrieModel = models.TrieModel;
+
+var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'),
+  {wordBreaker: wordBreakers.default});
+
 describe('ContextTracker', function() {
   function toWrapperDistribution(transforms) {
     transforms = Array.isArray(transforms) ? transforms : [transforms];
@@ -420,10 +426,10 @@ describe('ContextTracker', function() {
       newContext.left.splice(0, 1);
       let rawTokens = [" ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
 
-      let baseContextMatch = ContextTracker.modelContextState(existingContext.left);
+      let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
       let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
-      assert.deepEqual(newContextMatch.state.tokens.map(token => token.raw), rawTokens);
+      assert.deepEqual(newContextMatch.state.tokens.map(token => token.exampleInput), rawTokens);
       assert.equal(newContextMatch.headTokensRemoved, 1);
       assert.equal(newContextMatch.tailTokensAdded, 0);
     });
@@ -440,10 +446,10 @@ describe('ContextTracker', function() {
       newContext.left.splice(0, 2);
       let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
 
-      let baseContextMatch = ContextTracker.modelContextState(existingContext.left);
+      let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
       let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
-      assert.deepEqual(newContextMatch.state.tokens.map(token => token.raw), rawTokens);
+      assert.deepEqual(newContextMatch.state.tokens.map(token => token.exampleInput), rawTokens);
       assert.equal(newContextMatch.headTokensRemoved, 2);
       assert.equal(newContextMatch.tailTokensAdded, 0);
     });
@@ -461,10 +467,10 @@ describe('ContextTracker', function() {
       });
       let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
 
-      let baseContextMatch = ContextTracker.modelContextState(existingContext.left);
+      let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
       let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
-      assert.deepEqual(newContextMatch.state.tokens.map(token => token.raw), rawTokens);
+      assert.deepEqual(newContextMatch.state.tokens.map(token => token.exampleInput), rawTokens);
       assert.equal(newContextMatch.headTokensRemoved, 0);
       assert.equal(newContextMatch.tailTokensAdded, 0);
     });
@@ -483,17 +489,17 @@ describe('ContextTracker', function() {
       });
       let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
 
-      let baseContextMatch = ContextTracker.modelContextState(existingContext.left);
+      let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
       let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
-      assert.deepEqual(newContextMatch.state.tokens.map(token => token.raw), rawTokens);
+      assert.deepEqual(newContextMatch.state.tokens.map(token => token.exampleInput), rawTokens);
       // We want to preserve the added whitespace when predicting a token that follows after it.
       assert.deepEqual(newContextMatch.preservationTransform, { insert: ' ', deleteLeft: 0 });
 
       // The 'wordbreak' transform
       let state = newContextMatch?.state;
-      assert.isNotEmpty(state.tokens[state.tokens.length - 2].transformDistributions);
-      assert.isEmpty(state.tokens[state.tokens.length - 1].transformDistributions);
+      assert.isNotEmpty(state.tokens[state.tokens.length - 2].searchSpace.inputSequence);
+      assert.isEmpty(state.tokens[state.tokens.length - 1].searchSpace.inputSequence);
       assert.equal(newContextMatch.headTokensRemoved, 0);
       assert.equal(newContextMatch.tailTokensAdded, 2);
     });
@@ -511,10 +517,10 @@ describe('ContextTracker', function() {
       });
       let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
 
-      let baseContextMatch = ContextTracker.modelContextState(existingContext.left);
+      let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
       let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
       assert.isOk(newContextMatch?.state);
-      assert.deepEqual(newContextMatch?.state.tokens.map(token => token.raw), rawTokens);
+      assert.deepEqual(newContextMatch?.state.tokens.map(token => token.exampleInput), rawTokens);
 
       // The 'wordbreak' transform
       assert.equal(newContextMatch.headTokensRemoved, 0);
@@ -534,16 +540,16 @@ describe('ContextTracker', function() {
       });
       let rawTokens = ["'", "a"];
 
-      let baseContextMatch = ContextTracker.modelContextState(existingContext.left);
+      let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
       let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
-      assert.deepEqual(newContextMatch.state.tokens.map(token => token.raw), rawTokens);
+      assert.deepEqual(newContextMatch.state.tokens.map(token => token.exampleInput), rawTokens);
       assert.deepEqual(newContextMatch.preservationTransform, { insert: '', deleteLeft: 0 });
 
       // The 'wordbreak' transform
       let state = newContextMatch.state;
-      assert.isNotEmpty(state.tokens[state.tokens.length - 2].transformDistributions);
-      assert.isNotEmpty(state.tokens[state.tokens.length - 1].transformDistributions);
+      assert.isNotEmpty(state.tokens[state.tokens.length - 2].searchSpace.inputSequence);
+      assert.isNotEmpty(state.tokens[state.tokens.length - 1].searchSpace.inputSequence);
 
       assert.equal(newContextMatch.headTokensRemoved, 0);
       assert.equal(newContextMatch.tailTokensAdded, 1);
@@ -563,17 +569,17 @@ describe('ContextTracker', function() {
       });
       let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
 
-      let baseContextMatch = ContextTracker.modelContextState(existingContext.left);
+      let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
       let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
-      assert.deepEqual(newContextMatch.state.tokens.map(token => token.raw), rawTokens);
+      assert.deepEqual(newContextMatch.state.tokens.map(token => token.exampleInput), rawTokens);
       // We want to preserve the added whitespace when predicting a token that follows after it.
       assert.deepEqual(newContextMatch.preservationTransform, { insert: ' ', deleteLeft: 0 });
 
       // The 'wordbreak' transform
       let state = newContextMatch.state;
-      assert.isNotEmpty(state.tokens[state.tokens.length - 2].transformDistributions);
-      assert.isEmpty(state.tokens[state.tokens.length - 1].transformDistributions);
+      assert.isNotEmpty(state.tokens[state.tokens.length - 2].searchSpace.inputSequence);
+      assert.isEmpty(state.tokens[state.tokens.length - 1].searchSpace.inputSequence);
 
       assert.equal(newContextMatch.headTokensRemoved, 2);
       assert.equal(newContextMatch.tailTokensAdded, 2);
@@ -592,21 +598,21 @@ describe('ContextTracker', function() {
       });
       let rawTokens = ["and", " ", ""];
 
-      let baseContextMatch = ContextTracker.modelContextState(existingContext.left);
+      let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
       let newContextMatch = ContextTracker.attemptMatchContext(
         newContext.left,
         baseContextMatch,
         tokenizeTransformDistribution(tokenizer, {left: "an"}, [{sample: transform, p: 1}])
       );
       assert.isNotNull(newContextMatch?.state);
-      assert.deepEqual(newContextMatch.state.tokens.map(token => token.raw), rawTokens);
+      assert.deepEqual(newContextMatch.state.tokens.map(token => token.exampleInput), rawTokens);
       // We want to preserve all text preceding the new token when applying a suggestion.
       assert.deepEqual(newContextMatch.preservationTransform, { insert: 'd ', deleteLeft: 0});
 
       // The 'wordbreak' transform
       let state = newContextMatch.state;
-      assert.isNotEmpty(state.tokens[state.tokens.length - 2].transformDistributions);
-      assert.isEmpty(state.tokens[state.tokens.length - 1].transformDistributions);
+      assert.isNotEmpty(state.tokens[state.tokens.length - 2].searchSpace.inputSequence);
+      assert.isEmpty(state.tokens[state.tokens.length - 1].searchSpace.inputSequence);
 
       assert.equal(newContextMatch.headTokensRemoved, 0);
       assert.equal(newContextMatch.tailTokensAdded, 2);
@@ -625,21 +631,21 @@ describe('ContextTracker', function() {
       });
       let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
 
-      let baseContextMatch = ContextTracker.modelContextState(existingContext.left);
+      let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
       let newContextMatch = ContextTracker.attemptMatchContext(
         newContext.left,
         baseContextMatch,
         tokenizeTransformDistribution(tokenizer, {left: "apple a day keeps the doc"}, [{sample: transform, p: 1}])
       );
       assert.isNotNull(newContextMatch?.state);
-      assert.deepEqual(newContextMatch.state.tokens.map(token => token.raw), rawTokens);
+      assert.deepEqual(newContextMatch.state.tokens.map(token => token.exampleInput), rawTokens);
       // We want to preserve all text preceding the new token when applying a suggestion.
       assert.deepEqual(newContextMatch.preservationTransform, { insert: 'tor ', deleteLeft: 0 });
 
       // The 'wordbreak' transform
       let state = newContextMatch.state;
-      assert.isNotEmpty(state.tokens[state.tokens.length - 2].transformDistributions);
-      assert.isEmpty(state.tokens[state.tokens.length - 1].transformDistributions);
+      assert.isNotEmpty(state.tokens[state.tokens.length - 2].searchSpace.inputSequence);
+      assert.isEmpty(state.tokens[state.tokens.length - 1].searchSpace.inputSequence);
 
       assert.equal(newContextMatch.headTokensRemoved, 0);
       assert.equal(newContextMatch.tailTokensAdded, 2);
@@ -650,7 +656,7 @@ describe('ContextTracker', function() {
         left: "text'"
       });
       assert.equal(baseContext.left.length, 1);
-      let baseContextMatch = ContextTracker.modelContextState(baseContext.left);
+      let baseContextMatch = ContextTracker.modelContextState(baseContext.left, plainModel);
 
       // Now the actual check.
       let newContext = models.tokenize(defaultBreaker, {
@@ -683,8 +689,8 @@ describe('ContextTracker', function() {
       });
       let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
 
-      let state = ContextTracker.modelContextState(tokenized);
-      assert.deepEqual(state.tokens.map(token => token.raw), rawTokens);
+      let state = ContextTracker.modelContextState(tokenized, plainModel);
+      assert.deepEqual(state.tokens.map(token => token.exampleInput), rawTokens);
     });
 
     it('models with final wordbreak', function() {
@@ -696,8 +702,8 @@ describe('ContextTracker', function() {
       });
       let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
 
-      let state = ContextTracker.modelContextState(tokenized);
-      assert.deepEqual(state.tokens.map(token => token.raw), rawTokens);
+      let state = ContextTracker.modelContextState(tokenized, plainModel);
+      assert.deepEqual(state.tokens.map(token => token.exampleInput), rawTokens);
     });
   });
 
@@ -747,7 +753,7 @@ describe('ContextTracker', function() {
       let reversion = compositor.acceptSuggestion(baseSuggestion, baseContext, postTransform);
 
       // Actual test assertion - was the replacement tracked?
-      assert.equal(baseContextMatch.state.tail.activeReplacementId, baseSuggestion.id);
+      assert.equal(baseContextMatch.state.tail.appliedSuggestionId, baseSuggestion.id);
       assert.equal(reversion.id, -baseSuggestion.id);
 
       // Next step - on the followup context, is the replacement still active?
@@ -755,10 +761,10 @@ describe('ContextTracker', function() {
       let postContextMatch = compositor.contextTracker.analyzeState(model, postContext);
 
       // Penultimate token corresponds to whitespace, which does not have a 'raw' representation.
-      assert.equal(postContextMatch.state.tokens[postContextMatch.state.tokens.length - 2].raw, ' ');
+      assert.equal(postContextMatch.state.tokens[postContextMatch.state.tokens.length - 2].exampleInput, ' ');
 
       // Final token is empty (follows a wordbreak)
-      assert.equal(postContextMatch.state.tail.raw, '');
+      assert.equal(postContextMatch.state.tail.exampleInput, '');
     });
   });
 });
\ No newline at end of file
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/worker-model-compositor.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/worker-model-compositor.js
index 30e7bf5188..829ce8f99a 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/worker-model-compositor.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/worker-model-compositor.js
@@ -1064,7 +1064,7 @@ describe('ModelCompositor', function() {
       assert.equal(compositor.contextTracker.count, 3);
 
       // The replacement should be marked on the context-tracking token.
-      assert.isOk(suggestionContextState.tail.replacement);
+      assert.isAtLeast(suggestionContextState.tail.appliedSuggestionId, 0);
 
       let appliedContext = models.applyTransform(baseSuggestion.transform, baseContext);
       compositor.applyReversion(reversion, appliedContext);
@@ -1074,7 +1074,7 @@ describe('ModelCompositor', function() {
       assert.equal(compositor.contextTracker.item(1), suggestionContextState);
 
       // The replacement should no longer be marked for the context-tracking token.
-      assert.isNotOk(suggestionContextState.tail.replacement);
+      assert.equal(suggestionContextState.tail.appliedSuggestionId, -1);
     });
   });
 });

From e0c553d3065ff927c505b544045489b340c6038b Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Wed, 30 Jul 2025 09:21:42 -0500
Subject: [PATCH 18/53] change(web): appliedSuggestionId should be undefined,
 not -1, when none are applied

---
 .../worker-thread/src/main/correction/context-token.ts        | 4 ++--
 .../worker-thread/src/main/model-compositor.ts                | 2 +-
 .../src/tests/mocha/cases/edit-distance/context-token.js      | 4 ++--
 .../src/tests/mocha/cases/worker-model-compositor.js          | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts
index d886434932..d0c4d40961 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts
@@ -32,9 +32,9 @@ export class ContextToken {
   /**
    * The ID of the suggestion applied to the current token, if any.
    *
-   * Is set to -1 when no such suggestion exists.
+   * Should be set to undefined when no such suggestion exists.
    */
-  appliedSuggestionId: number = -1;
+  appliedSuggestionId?: number;
 
   /**
    * Constructs a new, empty instance for use with the specified LexicalModel.
diff --git a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts
index 031931d566..3dbe14c3da 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts
@@ -317,7 +317,7 @@ export class ModelCompositor {
       this.contextTracker.popNewest();
     }
 
-    this.contextTracker.newest.tail.appliedSuggestionId = -1;
+    this.contextTracker.newest.tail.appliedSuggestionId = undefined;
 
     // Will need to be modified a bit if/when phrase-level suggestions are implemented.
     // Those will be tracked on the first token of the phrase, which won't be the tail
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-token.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-token.js
index 4486ca3ded..78df8e9c2b 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-token.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-token.js
@@ -22,7 +22,7 @@ describe('ContextToken', function() {
       assert.isEmpty(token.exampleInput);
       assert.isFalse(token.isWhitespace);
       assert.isEmpty(token.suggestions);
-      assert.equal(token.appliedSuggestionId, -1);
+      assert.isUndefined(token.appliedSuggestionId);
 
       // While searchSpace has no inputs, it _can_ match lexicon entries (via insertions).
       let searchIterator = token.searchSpace.getBestMatches(new ExecutionTimer(Number.POSITIVE_INFINITY, Number.POSITIVE_INFINITY));
@@ -52,7 +52,7 @@ describe('ContextToken', function() {
 
       // Is only set with a different value later, outside the constructor.
       assert.isEmpty(token.suggestions);
-      assert.equal(token.appliedSuggestionId, -1);
+      assert.isUndefined(token.appliedSuggestionId);
     });
 
     it("(token: ContextToken", () => {
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/worker-model-compositor.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/worker-model-compositor.js
index 829ce8f99a..6d96bc68a9 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/worker-model-compositor.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/worker-model-compositor.js
@@ -1074,7 +1074,7 @@ describe('ModelCompositor', function() {
       assert.equal(compositor.contextTracker.item(1), suggestionContextState);
 
       // The replacement should no longer be marked for the context-tracking token.
-      assert.equal(suggestionContextState.tail.appliedSuggestionId, -1);
+      assert.equal(suggestionContextState.tail.appliedSuggestionId);
     });
   });
 });

From 063054a592eade23b01331b2771e548cd8fd7f26 Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Wed, 30 Jul 2025 12:07:01 -0500
Subject: [PATCH 19/53] refactor(web): relocate textToCharTransforms method

---
 .../src/main/correction/context-token.ts      | 23 ++++++++++++++++++-
 .../src/main/correction/context-tracker.ts    | 22 ------------------
 2 files changed, 22 insertions(+), 23 deletions(-)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts
index d0c4d40961..6bff048aee 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts
@@ -1,6 +1,6 @@
 import { buildMergedTransform } from "@keymanapp/models-templates";
-import { textToCharTransforms } from "./context-tracker.js";
 import { SearchSpace } from "./distance-modeler.js";
+import { KMWString } from "@keymanapp/web-utils";
 
 import { LexicalModelTypes } from '@keymanapp/common-types';
 import Distribution = LexicalModelTypes.Distribution;
@@ -8,6 +8,27 @@ import LexicalModel = LexicalModelTypes.LexicalModel;
 import Suggestion = LexicalModelTypes.Suggestion;
 import Transform = LexicalModelTypes.Transform;
 
+function textToCharTransforms(text: string, transformId?: number) {
+  let perCharTransforms: Transform[] = [];
+
+  for(let i=0; i < KMWString.length(text); i++) {
+    let char = KMWString.charAt(text, i); // is SMP-aware
+
+    let transform: Transform = {
+      insert: char,
+      deleteLeft: 0
+    };
+
+    if(transformId) {
+      transform.id = transformId
+    }
+
+    perCharTransforms.push(transform);
+  }
+
+  return perCharTransforms;
+}
+
 export class ContextToken {
   /**
    * Indicates whether or not the token is considered whitespace.
diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
index 625d41e672..8418b2077e 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
@@ -1,5 +1,4 @@
 import { applyTransform, buildMergedTransform, Token } from '@keymanapp/models-templates';
-import { KMWString } from '@keymanapp/web-utils';
 
 import { ClassicalDistanceCalculation, EditOperation } from './classical-calculation.js';
 import TransformUtils from '../transformUtils.js';
@@ -13,27 +12,6 @@ import Suggestion = LexicalModelTypes.Suggestion;
 import Transform = LexicalModelTypes.Transform;
 import { ContextToken } from './context-token.js';
 
-export function textToCharTransforms(text: string, transformId?: number) {
-  let perCharTransforms: Transform[] = [];
-
-  for(let i=0; i < KMWString.length(text); i++) {
-    let char = KMWString.charAt(text, i); // is SMP-aware
-
-    let transform: Transform = {
-      insert: char,
-      deleteLeft: 0
-    };
-
-    if(transformId) {
-      transform.id = transformId
-    }
-
-    perCharTransforms.push(transform);
-  }
-
-  return perCharTransforms;
-}
-
 /**
  * Determines the proper 'last match' index for a tokenized sequence based on its edit path.
  *

From 7bb51bd53cd7c9364325ba5ae7c3ea4b08174999 Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Wed, 30 Jul 2025 10:49:05 -0500
Subject: [PATCH 20/53] refactor(web):  refactor tracked-context tokenization
 and usage pattern

This PR's changes aim to meet our current needs for the tracking of context tokenization (word boundaries) across edits... all while moving much closer to the design specified within the Correction-Search + Context-Tracking design doc.

Also places the class within its own separate source file.
---
 .../main/correction/context-tokenization.ts   | 29 ++++++
 .../src/main/correction/context-tracker.ts    | 89 +++++--------------
 .../src/main/model-compositor.ts              | 12 +--
 .../worker-thread/src/main/predict-helpers.ts |  6 +-
 .../cases/edit-distance/context-tracker.js    | 50 +++++------
 .../mocha/cases/worker-model-compositor.js    |  4 +-
 6 files changed, 89 insertions(+), 101 deletions(-)
 create mode 100644 web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
new file mode 100644
index 0000000000..6621e2ec06
--- /dev/null
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
@@ -0,0 +1,29 @@
+import { TrackedContextStateAlignment } from './context-tracker.js';
+import { ContextToken } from './context-token.js';
+
+export class ContextTokenization {
+  readonly tokens: ContextToken[];
+  readonly alignment?: TrackedContextStateAlignment;
+
+  constructor(tokens: ContextToken[], alignment?: TrackedContextStateAlignment) {
+    this.tokens = [].concat(tokens);
+    this.alignment = alignment;
+  }
+
+  get tail(): ContextToken {
+    return this.tokens[this.tokens.length - 1];
+  }
+
+  get exampleInput(): string[] {
+    const sequence: string[] = [];
+
+    for(const token of this.tokens) {
+      // Hide any tokens representing wordbreaks.  (Thinking ahead to phrase-level possibilities)
+      if(token.exampleInput !== null) {
+        sequence.push(token.exampleInput);
+      }
+    }
+
+    return sequence;
+  }
+}
\ No newline at end of file
diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
index 8418b2077e..409f455d35 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
@@ -11,6 +11,7 @@ import LexicalModel = LexicalModelTypes.LexicalModel;
 import Suggestion = LexicalModelTypes.Suggestion;
 import Transform = LexicalModelTypes.Transform;
 import { ContextToken } from './context-token.js';
+import { ContextTokenization } from './context-tokenization.js';
 
 /**
  * Determines the proper 'last match' index for a tokenized sequence based on its edit path.
@@ -50,7 +51,8 @@ export class TrackedContextState {
   taggedContext: Context;
   model: LexicalModel;
 
-  tokens: ContextToken[];
+  tokenization: ContextTokenization;
+
   /**
    * How many tokens were removed from the start of the best-matching ancestor.
    * Useful for restoring older states, e.g., when the user moves the caret backwards, we can recover the context at that position.
@@ -63,52 +65,18 @@ export class TrackedContextState {
     if(obj instanceof TrackedContextState) {
       let source = obj;
       // Be sure to deep-copy the tokens!  Pointer-aliasing is bad here.
-      this.tokens = source.tokens.map((token) => new ContextToken(token));
+      this.tokenization = new ContextTokenization(source.tokenization.tokens.map((token) => new ContextToken(token)));
 
       this.indexOffset = 0;
       this.model = obj.model;
       this.taggedContext = obj.taggedContext;
     } else {
       let lexicalModel = obj;
-      this.tokens = [];
+      this.tokenization = null;
       this.indexOffset = Number.MIN_SAFE_INTEGER;
       this.model = lexicalModel;
     }
   }
-
-  get head(): ContextToken {
-    return this.tokens[0];
-  }
-
-  get tail(): ContextToken {
-    return this.tokens[this.tokens.length - 1];
-  }
-
-  set tail(token: ContextToken) {
-    this.tokens[this.tokens.length - 1] = token;
-  }
-
-  popHead() {
-    this.tokens.splice(0, 1);
-    this.indexOffset -= 1;
-  }
-
-  pushTail(token: ContextToken) {
-    this.tokens.push(token);
-  }
-
-  toRawTokenization() {
-    let sequence: string[] = [];
-
-    for(let token of this.tokens) {
-      // Hide any tokens representing wordbreaks.  (Thinking ahead to phrase-level possibilities)
-      if(token.exampleInput !== null) {
-        sequence.push(token.exampleInput);
-      }
-    }
-
-    return sequence;
-  }
 }
 
 class CircularArray<Item> {
@@ -237,7 +205,7 @@ interface ContextMatchResult {
  * Represents token-count values resulting from an alignment attempt between two
  * different modeled context states.
  */
-type TrackedContextStateAlignment = {
+export type TrackedContextStateAlignment = {
   /**
    * Denotes whether or not alignment is possible between two contexts.
    */
@@ -570,7 +538,7 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
     transformSequenceDistribution?: Distribution<Transform[]>
   ): ContextMatchResult {
     // Map the previous tokenized state to an edit-distance friendly version.
-    let matchContext: string[] = matchState.toRawTokenization();
+    let matchContext: string[] = matchState.tokenization.exampleInput;
 
     const alignmentResults = this.attemptTokenizedAlignment(tokenizedContext.map((token) => token.text), matchContext);
 
@@ -601,13 +569,10 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
     }
 
     // If mutations HAVE happened, we have work to do.
-    let state = matchState;
+    const tokenization = matchState.tokenization.tokens.map((token) => new ContextToken(token));
 
     if(leadTokenShift < 0) {
-      state = new TrackedContextState(state);
-      for(let i = 0; i > leadTokenShift; i--) {
-        state.popHead();
-      }
+      tokenization.splice(0, -leadTokenShift);
     } else if(leadTokenShift > 0) {
       // TODO:  insert token(s) at the start to match the text that's back within the
       // sliding context window.
@@ -618,6 +583,8 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
 
     // If no TAIL mutations have happened, we're safe to return now.
     if(tailEditLength == 0 && tailTokenShift == 0) {
+      const state = new TrackedContextState(matchState);
+      state.tokenization = new ContextTokenization(tokenization, alignmentResults);
       return {
         state: state,
         baseState: matchState,
@@ -665,7 +632,7 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
       const matchingIndex = i + matchingTailUpdateIndex;
 
       const incomingToken = tokenizedContext[incomingIndex];
-      const matchedToken = matchState.tokens[matchingIndex];
+      const matchedToken = matchState.tokenization.tokens[matchingIndex];
 
       let primaryInput = hasDistribution ? tokenizedPrimaryInput[i] : null;
       const isBackspace = primaryInput && TransformUtils.isBackspace(primaryInput);
@@ -673,7 +640,6 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
       const isLastToken = incomingIndex == tokenizedContext.length - 1;
 
       if(isLastToken) {
-        state = new TrackedContextState(state);
         // If this token's transform component is not part of the final token,
         // it's something we'll want to preserve even when applying suggestions
         // for the final token.
@@ -693,15 +659,11 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
         token.searchSpace.addInput(tokenDistribution.map((seq) => seq[tailIndex]));
       }
 
-      state.tokens[incomingIndex] = token;
+      tokenization[incomingIndex] = token;
       tailIndex++;
     }
 
     if(tailTokenShift < 0) {
-      if(state == matchState) {
-        state = new TrackedContextState(state);
-      }
-
       // delete tail tokens
       for(let i = 0; i > tailTokenShift; i--) {
         // If ALL that remains are deletes, we're good to go.
@@ -709,13 +671,9 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
         // This may not be the token at the index, but since all that remains are deletes,
         // we'll have deleted the correct total number from the end once all iterations
         // are done.
-        state.tokens.pop();
+        tokenization.pop();
       }
     } else {
-      if(state == matchState) {
-        state = new TrackedContextState(state);
-      }
-
       for(let i = tailEditLength; i < tailEditLength + tailTokenShift; i++) {
         // create tail tokens
         const incomingIndex = i + incomingTailUpdateIndex;
@@ -738,11 +696,7 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
           preservationTransform = preservationTransform && primaryInput ? buildMergedTransform(preservationTransform, primaryInput) : (primaryInput ?? preservationTransform);
         }
 
-        if(state == matchState) {
-          state = new TrackedContextState(state);
-        }
-
-        let pushedToken = new ContextToken(state.model);
+        let pushedToken = new ContextToken(matchState.model);
 
         // TODO:  assumes that there was no shift in wordbreaking from the
         // prior context to the current one.  This may actually be a major
@@ -778,12 +732,15 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
         pushedToken.isWhitespace = incomingToken.isWhitespace;
 
         // Auto-replaces the search space to correspond with the new token.
-        state.pushTail(pushedToken);
+        tokenization.push(pushedToken);
 
         tailIndex++;
       }
     }
 
+    const state = new TrackedContextState(matchState);
+    state.tokenization = new ContextTokenization(tokenization, alignmentResults);
+
     return {
       state,
       baseState: matchState,
@@ -809,16 +766,18 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
 
     // And now build the final context state object, which includes whitespace 'tokens'.
     let state = new TrackedContextState(lexicalModel);
+    const tokenization: ContextToken[] = [];
 
     while(baseTokens.length > 0) {
-      state.pushTail(baseTokens.splice(0, 1)[0]);
+      tokenization.push(baseTokens.splice(0, 1)[0]);
     }
 
-    if(state.tokens.length == 0) {
+    if(tokenization.length == 0) {
       let token = new ContextToken(lexicalModel);
-      state.pushTail(token);
+      tokenization.push(token);
     }
 
+    state.tokenization = new ContextTokenization(tokenization);
     return state;
   }
 
diff --git a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts
index 3dbe14c3da..cf30c16a46 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts
@@ -181,7 +181,7 @@ export class ModelCompositor {
     // Store the suggestions on the final token of the current context state (if it exists).
     // Or, once phrase-level suggestions are possible, on whichever token serves as each prediction's root.
     if(postContextState) {
-      postContextState.tail.suggestions = suggestions;
+      postContextState.tokenization.tail.suggestions = suggestions;
     }
 
     return suggestions;
@@ -263,7 +263,7 @@ export class ModelCompositor {
         contextState = this.contextTracker.analyzeState(this.lexicalModel, context).state;
       }
 
-      contextState.tail.appliedSuggestionId = suggestion.id;
+      contextState.tokenization.tail.appliedSuggestionId = suggestion.id;
       let acceptedContext = models.applyTransform(suggestion.transform, context);
       if(suggestion.appendedTransform) {
         acceptedContext = models.applyTransform(suggestion.appendedTransform, acceptedContext);
@@ -302,7 +302,7 @@ export class ModelCompositor {
     for(let c = this.contextTracker.count - 1; c >= 0; c--) {
       let contextState = this.contextTracker.item(c);
 
-      if(contextState.tail.appliedSuggestionId == -reversion.id) {
+      if(contextState.tokenization.tail.appliedSuggestionId == -reversion.id) {
         contextMatchFound = true;
         break;
       }
@@ -313,16 +313,16 @@ export class ModelCompositor {
     }
 
     // Remove all contexts more recent than the one we're reverting to.
-    while(this.contextTracker.newest.tail.appliedSuggestionId != -reversion.id) {
+    while(this.contextTracker.newest.tokenization.tail.appliedSuggestionId != -reversion.id) {
       this.contextTracker.popNewest();
     }
 
-    this.contextTracker.newest.tail.appliedSuggestionId = undefined;
+    this.contextTracker.newest.tokenization.tail.appliedSuggestionId = undefined;
 
     // Will need to be modified a bit if/when phrase-level suggestions are implemented.
     // Those will be tracked on the first token of the phrase, which won't be the tail
     // if they cover multiple tokens.
-    let suggestions = this.contextTracker.newest.tail.suggestions;
+    let suggestions = this.contextTracker.newest.tokenization.tail.suggestions;
 
     suggestions.forEach(function(suggestion) {
       // A reversion's transform ID is the additive inverse of its original suggestion;
diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts
index bc1c731939..30ab7eb9c9 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts
@@ -202,7 +202,7 @@ export async function correctAndEnumerate(
   // let's just note that right now, there will only ever be one.
   //
   // The 'eventual' logic will be significantly more complex, though still manageable.
-  const searchSpace = postContextState.tail.searchSpace;
+  const searchSpace = postContextState.tokenization.tail.searchSpace;
 
   // No matter the prediction, once we know the root of the prediction, we'll always 'replace' the
   // same amount of text.  We can handle this before the big 'prediction root' loop.
@@ -210,9 +210,9 @@ export async function correctAndEnumerate(
 
   // The amount of text to 'replace' depends upon whatever sort of context change occurs
   // from the received input.
-  const postContextTokens = postContextState.tokens;
+  const postContextTokens = postContextState.tokenization.tokens;
   // Only use of `contextState`.
-  let contextLengthDelta = postContextTokens.length - contextState.tokens.length;
+  let contextLengthDelta = postContextTokens.length - contextState.tokenization.tokens.length;
   // If the context now has more tokens, the token we'll be 'predicting' didn't originally exist.
   if(contextChangeAnalysis.preservationTransform) {
     // As the word/token being corrected/predicted didn't originally exist, there's no
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
index bc862ba651..f3df01d3d9 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
@@ -429,7 +429,7 @@ describe('ContextTracker', function() {
       let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
       let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
-      assert.deepEqual(newContextMatch.state.tokens.map(token => token.exampleInput), rawTokens);
+      assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       assert.equal(newContextMatch.headTokensRemoved, 1);
       assert.equal(newContextMatch.tailTokensAdded, 0);
     });
@@ -449,7 +449,7 @@ describe('ContextTracker', function() {
       let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
       let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
-      assert.deepEqual(newContextMatch.state.tokens.map(token => token.exampleInput), rawTokens);
+      assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       assert.equal(newContextMatch.headTokensRemoved, 2);
       assert.equal(newContextMatch.tailTokensAdded, 0);
     });
@@ -470,7 +470,7 @@ describe('ContextTracker', function() {
       let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
       let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
-      assert.deepEqual(newContextMatch.state.tokens.map(token => token.exampleInput), rawTokens);
+      assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       assert.equal(newContextMatch.headTokensRemoved, 0);
       assert.equal(newContextMatch.tailTokensAdded, 0);
     });
@@ -492,14 +492,14 @@ describe('ContextTracker', function() {
       let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
       let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
-      assert.deepEqual(newContextMatch.state.tokens.map(token => token.exampleInput), rawTokens);
+      assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       // We want to preserve the added whitespace when predicting a token that follows after it.
       assert.deepEqual(newContextMatch.preservationTransform, { insert: ' ', deleteLeft: 0 });
 
       // The 'wordbreak' transform
       let state = newContextMatch?.state;
-      assert.isNotEmpty(state.tokens[state.tokens.length - 2].searchSpace.inputSequence);
-      assert.isEmpty(state.tokens[state.tokens.length - 1].searchSpace.inputSequence);
+      assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence);
+      assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence);
       assert.equal(newContextMatch.headTokensRemoved, 0);
       assert.equal(newContextMatch.tailTokensAdded, 2);
     });
@@ -520,7 +520,7 @@ describe('ContextTracker', function() {
       let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
       let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
       assert.isOk(newContextMatch?.state);
-      assert.deepEqual(newContextMatch?.state.tokens.map(token => token.exampleInput), rawTokens);
+      assert.deepEqual(newContextMatch?.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
 
       // The 'wordbreak' transform
       assert.equal(newContextMatch.headTokensRemoved, 0);
@@ -543,13 +543,13 @@ describe('ContextTracker', function() {
       let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
       let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
-      assert.deepEqual(newContextMatch.state.tokens.map(token => token.exampleInput), rawTokens);
+      assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       assert.deepEqual(newContextMatch.preservationTransform, { insert: '', deleteLeft: 0 });
 
       // The 'wordbreak' transform
       let state = newContextMatch.state;
-      assert.isNotEmpty(state.tokens[state.tokens.length - 2].searchSpace.inputSequence);
-      assert.isNotEmpty(state.tokens[state.tokens.length - 1].searchSpace.inputSequence);
+      assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence);
+      assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence);
 
       assert.equal(newContextMatch.headTokensRemoved, 0);
       assert.equal(newContextMatch.tailTokensAdded, 1);
@@ -572,14 +572,14 @@ describe('ContextTracker', function() {
       let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
       let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
-      assert.deepEqual(newContextMatch.state.tokens.map(token => token.exampleInput), rawTokens);
+      assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       // We want to preserve the added whitespace when predicting a token that follows after it.
       assert.deepEqual(newContextMatch.preservationTransform, { insert: ' ', deleteLeft: 0 });
 
       // The 'wordbreak' transform
       let state = newContextMatch.state;
-      assert.isNotEmpty(state.tokens[state.tokens.length - 2].searchSpace.inputSequence);
-      assert.isEmpty(state.tokens[state.tokens.length - 1].searchSpace.inputSequence);
+      assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence);
+      assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence);
 
       assert.equal(newContextMatch.headTokensRemoved, 2);
       assert.equal(newContextMatch.tailTokensAdded, 2);
@@ -605,14 +605,14 @@ describe('ContextTracker', function() {
         tokenizeTransformDistribution(tokenizer, {left: "an"}, [{sample: transform, p: 1}])
       );
       assert.isNotNull(newContextMatch?.state);
-      assert.deepEqual(newContextMatch.state.tokens.map(token => token.exampleInput), rawTokens);
+      assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       // We want to preserve all text preceding the new token when applying a suggestion.
       assert.deepEqual(newContextMatch.preservationTransform, { insert: 'd ', deleteLeft: 0});
 
       // The 'wordbreak' transform
       let state = newContextMatch.state;
-      assert.isNotEmpty(state.tokens[state.tokens.length - 2].searchSpace.inputSequence);
-      assert.isEmpty(state.tokens[state.tokens.length - 1].searchSpace.inputSequence);
+      assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence);
+      assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence);
 
       assert.equal(newContextMatch.headTokensRemoved, 0);
       assert.equal(newContextMatch.tailTokensAdded, 2);
@@ -638,14 +638,14 @@ describe('ContextTracker', function() {
         tokenizeTransformDistribution(tokenizer, {left: "apple a day keeps the doc"}, [{sample: transform, p: 1}])
       );
       assert.isNotNull(newContextMatch?.state);
-      assert.deepEqual(newContextMatch.state.tokens.map(token => token.exampleInput), rawTokens);
+      assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       // We want to preserve all text preceding the new token when applying a suggestion.
       assert.deepEqual(newContextMatch.preservationTransform, { insert: 'tor ', deleteLeft: 0 });
 
       // The 'wordbreak' transform
       let state = newContextMatch.state;
-      assert.isNotEmpty(state.tokens[state.tokens.length - 2].searchSpace.inputSequence);
-      assert.isEmpty(state.tokens[state.tokens.length - 1].searchSpace.inputSequence);
+      assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence);
+      assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence);
 
       assert.equal(newContextMatch.headTokensRemoved, 0);
       assert.equal(newContextMatch.tailTokensAdded, 2);
@@ -690,7 +690,7 @@ describe('ContextTracker', function() {
       let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
 
       let state = ContextTracker.modelContextState(tokenized, plainModel);
-      assert.deepEqual(state.tokens.map(token => token.exampleInput), rawTokens);
+      assert.deepEqual(state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
     });
 
     it('models with final wordbreak', function() {
@@ -703,7 +703,7 @@ describe('ContextTracker', function() {
       let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
 
       let state = ContextTracker.modelContextState(tokenized, plainModel);
-      assert.deepEqual(state.tokens.map(token => token.exampleInput), rawTokens);
+      assert.deepEqual(state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
     });
   });
 
@@ -745,7 +745,7 @@ describe('ContextTracker', function() {
       let compositor = new ModelCompositor(model);
       let baseContextMatch = compositor.contextTracker.analyzeState(model, baseContext);
 
-      baseContextMatch.state.tail.replacements = [{
+      baseContextMatch.state.tokenization.tail.replacements = [{
         suggestion: baseSuggestion,
         tokenWidth: 1
       }];
@@ -753,7 +753,7 @@ describe('ContextTracker', function() {
       let reversion = compositor.acceptSuggestion(baseSuggestion, baseContext, postTransform);
 
       // Actual test assertion - was the replacement tracked?
-      assert.equal(baseContextMatch.state.tail.appliedSuggestionId, baseSuggestion.id);
+      assert.equal(baseContextMatch.state.tokenization.tail.appliedSuggestionId, baseSuggestion.id);
       assert.equal(reversion.id, -baseSuggestion.id);
 
       // Next step - on the followup context, is the replacement still active?
@@ -761,10 +761,10 @@ describe('ContextTracker', function() {
       let postContextMatch = compositor.contextTracker.analyzeState(model, postContext);
 
       // Penultimate token corresponds to whitespace, which does not have a 'raw' representation.
-      assert.equal(postContextMatch.state.tokens[postContextMatch.state.tokens.length - 2].exampleInput, ' ');
+      assert.equal(postContextMatch.state.tokenization.tokens[postContextMatch.state.tokenization.tokens.length - 2].exampleInput, ' ');
 
       // Final token is empty (follows a wordbreak)
-      assert.equal(postContextMatch.state.tail.exampleInput, '');
+      assert.equal(postContextMatch.state.tokenization.tail.exampleInput, '');
     });
   });
 });
\ No newline at end of file
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/worker-model-compositor.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/worker-model-compositor.js
index 6d96bc68a9..2f310693a9 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/worker-model-compositor.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/worker-model-compositor.js
@@ -1064,7 +1064,7 @@ describe('ModelCompositor', function() {
       assert.equal(compositor.contextTracker.count, 3);
 
       // The replacement should be marked on the context-tracking token.
-      assert.isAtLeast(suggestionContextState.tail.appliedSuggestionId, 0);
+      assert.isAtLeast(suggestionContextState.tokenization.tail.appliedSuggestionId, 0);
 
       let appliedContext = models.applyTransform(baseSuggestion.transform, baseContext);
       compositor.applyReversion(reversion, appliedContext);
@@ -1074,7 +1074,7 @@ describe('ModelCompositor', function() {
       assert.equal(compositor.contextTracker.item(1), suggestionContextState);
 
       // The replacement should no longer be marked for the context-tracking token.
-      assert.equal(suggestionContextState.tail.appliedSuggestionId);
+      assert.equal(suggestionContextState.tokenization.tail.appliedSuggestionId);
     });
   });
 });

From bf897016c373dcc147ac3bbf9f23290422cceab8 Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Fri, 1 Aug 2025 12:05:10 -0500
Subject: [PATCH 21/53] change(web): add unit tests & clone constructor

---
 .../main/correction/context-tokenization.ts   | 17 +++-
 .../src/main/correction/context-tracker.ts    |  2 +
 .../edit-distance/context-tokenization.js     | 84 +++++++++++++++++++
 3 files changed, 99 insertions(+), 4 deletions(-)
 create mode 100644 web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
index 6621e2ec06..9762c9d87d 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
@@ -1,13 +1,22 @@
-import { TrackedContextStateAlignment } from './context-tracker.js';
 import { ContextToken } from './context-token.js';
+import { TrackedContextStateAlignment } from './context-tracker.js';
 
 export class ContextTokenization {
   readonly tokens: ContextToken[];
   readonly alignment?: TrackedContextStateAlignment;
 
-  constructor(tokens: ContextToken[], alignment?: TrackedContextStateAlignment) {
-    this.tokens = [].concat(tokens);
-    this.alignment = alignment;
+  constructor(priorToClone: ContextTokenization);
+  constructor(tokens: ContextToken[], alignment?: TrackedContextStateAlignment);
+  constructor(param1: ContextToken[] | ContextTokenization, alignment?: TrackedContextStateAlignment) {
+    if(!(param1 instanceof ContextTokenization)) {
+      const tokens = param1;
+      this.tokens = [].concat(tokens);
+      this.alignment = alignment;
+    } else {
+      const priorToClone = param1;
+      this.tokens = priorToClone.tokens.map((entry) => new ContextToken(entry));
+      this.alignment = {...priorToClone.alignment};
+    }
   }
 
   get tail(): ContextToken {
diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
index 409f455d35..42fea93fe7 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
@@ -655,6 +655,8 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
         token = new ContextToken(matchState.model, incomingToken.text);
         token.searchSpace.inputSequence.forEach((entry) => entry[0].sample.id = primaryInput.id);
       } else {
+        // Assumption:  there have been no intervening keystrokes since the last well-aligned context.
+        // (May not be valid with epic/dict-breaker or with complex, word-boundary crossing transforms)
         token = new ContextToken(matchedToken);
         token.searchSpace.addInput(tokenDistribution.map((seq) => seq[tailIndex]));
       }
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js
new file mode 100644
index 0000000000..8ba132ed56
--- /dev/null
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js
@@ -0,0 +1,84 @@
+import { assert } from 'chai';
+
+import { ContextToken } from '#./correction/context-token.js';
+import { ContextTokenization } from '#./correction/context-tokenization.js';
+
+import * as models from '#./models/index.js';
+import { default as defaultBreaker } from '@keymanapp/models-wordbreakers';
+import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs';
+
+var TrieModel = models.TrieModel;
+
+var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'),
+  {wordBreaker: defaultBreaker});
+
+function toToken(text) {
+  let isWhitespace = text == ' ';
+  let token = new ContextToken(plainModel, text);
+  token.isWhitespace = isWhitespace;
+  return token;
+}
+
+describe('ContextTokenization', function() {
+  describe("<constructor>", () => {
+    it("constructs from just a token array", () => {
+      const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
+      let tokenization = new ContextTokenization(rawTextTokens.map((text => toToken(text))));
+
+      assert.deepEqual(tokenization.tokens.map((entry) => entry.exampleInput), rawTextTokens);
+      assert.deepEqual(tokenization.tokens.map((entry) => entry.isWhitespace), rawTextTokens.map((entry) => entry == ' '));
+      assert.isNotOk(tokenization.alignment);
+      assert.equal(tokenization.tail.exampleInput, 'day');
+      assert.isFalse(tokenization.tail.isWhitespace);
+      assert.isUndefined(tokenization.tail.appliedSuggestionId);
+    });
+
+    it("constructs from a token array + alignment data", () => {
+      const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
+      let alignment = {
+        canAlign: true,
+        leadTokenShift: 0,
+        matchLength: 6,
+        tailEditLength: 1,
+        tailTokenShift: 0
+      };
+
+      let tokenization = new ContextTokenization(rawTextTokens.map((text => toToken(text))), alignment);
+
+      assert.deepEqual(tokenization.tokens.map((entry) => entry.exampleInput), rawTextTokens);
+      assert.deepEqual(tokenization.tokens.map((entry) => entry.isWhitespace), rawTextTokens.map((entry) => entry == ' '));
+      assert.isOk(tokenization.alignment);
+      assert.deepEqual(tokenization.alignment, alignment);
+      assert.equal(tokenization.tail.exampleInput, 'day');
+      assert.isFalse(tokenization.tail.isWhitespace);
+      assert.isUndefined(tokenization.tail.appliedSuggestionId);
+    });
+
+    it('clones', () => {
+      const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
+
+      let baseTokenization = new ContextTokenization(rawTextTokens.map((text => toToken(text))), {
+        canAlign: true,
+        leadTokenShift: 0,
+        matchLength: 6,
+        tailEditLength: 1,
+        tailTokenShift: 0
+      });
+
+      let cloned = new ContextTokenization(baseTokenization);
+
+      assert.notDeepEqual(cloned, baseTokenization);
+      assert.notDeepEqual(cloned.tokens, baseTokenization.tokens);
+      assert.deepEqual(cloned.tokens.map((token) => token.searchSpace.inputSequence),
+        baseTokenization.tokens.map((token) => token.searchSpace.inputSequence));
+      assert.deepEqual(cloned.alignment, baseTokenization.alignment);
+    });
+  });
+
+  it('exampleInput', () => {
+    const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
+    let tokenization = new ContextTokenization(rawTextTokens.map((text => toToken(text))));
+
+    assert.deepEqual(tokenization.exampleInput, rawTextTokens);
+  });
+});
\ No newline at end of file

From 91928a38fdc7c3279d4e3bf21cb75053f00374e6 Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Wed, 30 Jul 2025 11:57:43 -0500
Subject: [PATCH 22/53] refactor(web): relocate alignment helpers to a separate
 file

---
 .../src/main/correction/alignment-helpers.ts  | 114 +++++++++++++++++
 .../src/main/correction/context-tracker.ts    | 120 +-----------------
 .../edit-distance/context-tokenization.js     |  76 ++++++++++-
 .../cases/edit-distance/context-tracker.js    |  73 -----------
 4 files changed, 193 insertions(+), 190 deletions(-)
 create mode 100644 web/src/engine/predictive-text/worker-thread/src/main/correction/alignment-helpers.ts

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/alignment-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/alignment-helpers.ts
new file mode 100644
index 0000000000..54679e200e
--- /dev/null
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/alignment-helpers.ts
@@ -0,0 +1,114 @@
+import { ClassicalDistanceCalculation, EditOperation } from "./classical-calculation.js";
+
+/**
+ * Determines the proper 'last match' index for a tokenized sequence based on its edit path.
+ *
+ * In particular, this method is designed to handle the following case:
+ * ['to', 'apple', ' ', ''] => ['to', 'apply', ' ', 'n']
+ *
+ * Edit path for this example case:
+ * ['match', 'substitute', 'match', 'substitute']
+ *
+ * In cases such as these, the whitespace match should be considered 'edited'. While the ' '
+ * is unedited, it follows the edited 'apple' => 'apply', so it must have been deleted and
+ * then re-inserted.  As a result, 'to' is the true "last matched" token.
+ * @param editPath
+ * @returns
+ */
+export function getEditPathLastMatch(editPath: EditOperation[]) {
+  const editLength = editPath.length;
+  // Special handling: appending whitespace to whitespace with the default wordbreaker.
+  // The default wordbreaker currently adds an empty token after whitespace; this would
+  // show up with 'substitute', 'match' at the end of the edit path.  (This should remain.)
+  if(editLength >= 2 && editPath[editLength - 2] == 'substitute' && editPath[editLength - 1] == 'match') {
+    return editPath.lastIndexOf('match', editLength - 2);
+  } else {
+    return editPath.lastIndexOf('match');
+  }
+}
+
+/**
+ * Aligns two tokens on a character-by-character basis as needed for higher, token-level alignment
+ * operations.
+ * @param incomingToken The incoming token value
+ * @param matchingToken The pre-existing token value to use for comparison and alignment
+ * @param forNearCaret  If `false`, disallows any substitutions and activates a leading-edge alignment
+ * validation mode.
+ * @returns
+ */
+export function isSubstitutionAlignable(
+  incomingToken: string,
+  matchingToken: string,
+  forNearCaret?: boolean
+): boolean {
+  // 1 - Determine the edit path for the word.
+  let subEditPath = ClassicalDistanceCalculation.computeDistance(
+    [...matchingToken].map(value => ({key: value})),
+    [...incomingToken].map(value => ({key: value})),
+    // Diagonal width to consider must be at least 2, as adding a single
+    // whitespace after a token tends to add two tokens: one for whitespace,
+    // one for the empty token to follow it.
+    3
+  ).editPath();
+
+  const firstInsert = subEditPath.indexOf('insert');
+  const firstDelete = subEditPath.indexOf('delete');
+
+  // 2 - deletions and insertions should be mutually exclusive.
+  // A fixed, unedited word can't slide across both 'left' and 'right' boundaries at the same time.
+  if(firstInsert != -1 && firstDelete != -1) {
+    return false;
+  };
+
+  // 3 - checks exclusive to leading-edge conditions
+  if(!forNearCaret) {
+    const firstSubstitute = subEditPath.indexOf('substitute');
+    const firstMatch      = subEditPath.indexOf('match');
+    if(firstSubstitute > -1) {
+      return false;
+    } else if(firstMatch > -1) {
+      // Should not have inserts on both sides of matched text!
+      if(firstInsert > -1 && firstInsert < firstMatch && subEditPath.lastIndexOf('insert') > firstMatch) {
+        return false;
+      } else if(firstDelete > -1 && firstDelete < firstMatch && subEditPath.lastIndexOf('delete') > firstMatch) {
+        return false;
+      }
+    }
+
+    // Further checks below are oriented for text/tokens at the caret.
+    return true;
+  }
+
+  // 4 - check the stats for total edits of each type and validate that edits don't overly exceed
+  // original characters.
+  const editCount = {
+    matchMove: 0,
+    rawEdit: 0
+  };
+
+  subEditPath.forEach((entry) => {
+    switch(entry) {
+      case 'transpose-end':
+      case 'transpose-start':
+      case 'match':
+        editCount.matchMove++;
+        break;
+      case 'insert':
+      case 'transpose-insert':
+      case 'delete':
+      case 'transpose-delete':
+      case 'substitute':
+        editCount.rawEdit++;
+    }
+  });
+
+  // We shouldn't have more raw substitutions, inserts, and deletes than matches + transposes,
+  // though allowing +1 as a fudge factor.
+  // The 'a' => 'à' pattern can be a reasonably common Keyman keyboard rule and
+  // is one substitution, zero matches in NFC.
+  if(editCount.matchMove + 1 < editCount.rawEdit) {
+    return false;
+  }
+
+  return true;
+}
\ No newline at end of file
diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
index 42fea93fe7..c6eea80fa8 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
@@ -1,6 +1,7 @@
 import { applyTransform, buildMergedTransform, Token } from '@keymanapp/models-templates';
 
-import { ClassicalDistanceCalculation, EditOperation } from './classical-calculation.js';
+import { getEditPathLastMatch, isSubstitutionAlignable } from './alignment-helpers.js';
+import { ClassicalDistanceCalculation } from './classical-calculation.js';
 import TransformUtils from '../transformUtils.js';
 import { determineModelTokenizer } from '../model-helpers.js';
 import { tokenizeTransform, tokenizeTransformDistribution } from './transform-tokenization.js';
@@ -13,33 +14,6 @@ import Transform = LexicalModelTypes.Transform;
 import { ContextToken } from './context-token.js';
 import { ContextTokenization } from './context-tokenization.js';
 
-/**
- * Determines the proper 'last match' index for a tokenized sequence based on its edit path.
- *
- * In particular, this method is designed to handle the following case:
- * ['to', 'apple', ' ', ''] => ['to', 'apply', ' ', 'n']
- *
- * Edit path for this example case:
- * ['match', 'substitute', 'match', 'substitute']
- *
- * In cases such as these, the whitespace match should be considered 'edited'. While the ' '
- * is unedited, it follows the edited 'apple' => 'apply', so it must have been deleted and
- * then re-inserted.  As a result, 'to' is the true "last matched" token.
- * @param editPath
- * @returns
- */
-export function getEditPathLastMatch(editPath: EditOperation[]) {
-  const editLength = editPath.length;
-  // Special handling: appending whitespace to whitespace with the default wordbreaker.
-  // The default wordbreaker currently adds an empty token after whitespace; this would
-  // show up with 'substitute', 'match' at the end of the edit path.  (This should remain.)
-  if(editLength >= 2 && editPath[editLength - 2] == 'substitute' && editPath[editLength - 1] == 'match') {
-    return editPath.lastIndexOf('match', editLength - 2);
-  } else {
-    return editPath.lastIndexOf('match');
-  }
-}
-
 export class TrackedContextSuggestion {
   suggestion: Suggestion;
   tokenWidth: number;
@@ -242,92 +216,6 @@ export type TrackedContextStateAlignment = {
 };
 
 export class ContextTracker extends CircularArray<TrackedContextState> {
-  /**
-   * Aligns two tokens on a character-by-character basis as needed for higher, token-level alignment
-   * operations.
-   * @param incomingToken The incoming token value
-   * @param matchingToken The pre-existing token value to use for comparison and alignment
-   * @param forNearCaret  If `false`, disallows any substitutions and activates a leading-edge alignment
-   * validation mode.
-   * @returns
-   */
-  static isSubstitutionAlignable(
-    incomingToken: string,
-    matchingToken: string,
-    forNearCaret?: boolean
-  ): boolean {
-    // 1 - Determine the edit path for the word.
-    let subEditPath = ClassicalDistanceCalculation.computeDistance(
-      [...matchingToken].map(value => ({key: value})),
-      [...incomingToken].map(value => ({key: value})),
-      // Diagonal width to consider must be at least 2, as adding a single
-      // whitespace after a token tends to add two tokens: one for whitespace,
-      // one for the empty token to follow it.
-      3
-    ).editPath();
-
-    const firstInsert = subEditPath.indexOf('insert');
-    const firstDelete = subEditPath.indexOf('delete');
-
-    // 2 - deletions and insertions should be mutually exclusive.
-    // A fixed, unedited word can't slide across both 'left' and 'right' boundaries at the same time.
-    if(firstInsert != -1 && firstDelete != -1) {
-      return false;
-    };
-
-    // 3 - checks exclusive to leading-edge conditions
-    if(!forNearCaret) {
-      const firstSubstitute = subEditPath.indexOf('substitute');
-      const firstMatch      = subEditPath.indexOf('match');
-      if(firstSubstitute > -1) {
-        return false;
-      } else if(firstMatch > -1) {
-        // Should not have inserts on both sides of matched text!
-        if(firstInsert > -1 && firstInsert < firstMatch && subEditPath.lastIndexOf('insert') > firstMatch) {
-          return false;
-        } else if(firstDelete > -1 && firstDelete < firstMatch && subEditPath.lastIndexOf('delete') > firstMatch) {
-          return false;
-        }
-      }
-
-      // Further checks below are oriented for text/tokens at the caret.
-      return true;
-    }
-
-    // 4 - check the stats for total edits of each type and validate that edits don't overly exceed
-    // original characters.
-    const editCount = {
-      matchMove: 0,
-      rawEdit: 0
-    };
-
-    subEditPath.forEach((entry) => {
-      switch(entry) {
-        case 'transpose-end':
-        case 'transpose-start':
-        case 'match':
-          editCount.matchMove++;
-          break;
-        case 'insert':
-        case 'transpose-insert':
-        case 'delete':
-        case 'transpose-delete':
-        case 'substitute':
-          editCount.rawEdit++;
-      }
-    });
-
-    // We shouldn't have more raw substitutions, inserts, and deletes than matches + transposes,
-    // though allowing +1 as a fudge factor.
-    // The 'a' => 'à' pattern can be a reasonably common Keyman keyboard rule and
-    // is one substitution, zero matches in NFC.
-    if(editCount.matchMove + 1 < editCount.rawEdit) {
-      return false;
-    }
-
-    return true;
-  }
-
   static attemptTokenizedAlignment(
     incomingTokenization: string[],
     tokenizationToMatch: string[]
@@ -353,7 +241,7 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
       for(let i = 0; i < editPath.length; i++) {
         if(editPath[i] == 'substitute') {
           subCount++;
-          if(!this.isSubstitutionAlignable(incomingTokenization[i], tokenizationToMatch[i], true)) {
+          if(!isSubstitutionAlignable(incomingTokenization[i], tokenizationToMatch[i], true)) {
             return {
               canAlign: false
             };
@@ -486,7 +374,7 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
           const matchingSub = tokenizationToMatch[i + (leadTokensRemoved < 0 ? leadTokensRemoved : 0)];
 
           // Double-check the word - does the 'substituted' word itself align?
-          if(!this.isSubstitutionAlignable(incomingSub, matchingSub)) {
+          if(!isSubstitutionAlignable(incomingSub, matchingSub)) {
             return {
               canAlign: false
             };
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js
index 8ba132ed56..77b2e04994 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js
@@ -1,4 +1,5 @@
 import { assert } from 'chai';
+import { isSubstitutionAlignable } from "#./correction/context-alignment.js";
 
 import { ContextToken } from '#./correction/context-token.js';
 import { ContextTokenization } from '#./correction/context-tokenization.js';
@@ -19,6 +20,79 @@ function toToken(text) {
   return token;
 }
 
+describe('isSubstitutionAlignable', () => {
+  it(`returns true:  'ca' => 'can'`, () => {
+    assert.isTrue(isSubstitutionAlignable('can', 'ca'));
+  });
+
+  // Leading word in context window starts sliding out of said window.
+  it(`returns true:  'can' => 'an'`, () => {
+    assert.isTrue(isSubstitutionAlignable('an', 'can'));
+  });
+
+  // Same edits on both sides:  not valid.
+  it(`returns false: 'apple' => 'grapples'`, () => {
+    assert.isFalse(isSubstitutionAlignable('grapples', 'apple'));
+  });
+
+  // Edits on one side:  valid.
+  it(`returns true: 'apple' => 'grapple'`, () => {
+    assert.isTrue(isSubstitutionAlignable('grapple', 'apple'));
+  });
+
+  // Edits on one side:  valid.
+  it(`returns true: 'apple' => 'grapple'`, () => {
+    assert.isTrue(isSubstitutionAlignable('apples', 'apple'));
+  });
+
+  // Same edits on both sides:  not valid.
+  it(`returns false: 'grapples' => 'apple'`, () => {
+    assert.isFalse(isSubstitutionAlignable('apple', 'grapples'));
+  });
+
+  // Substitution:  not valid when not permitted via parameter.
+  it(`returns false:  'apple' => 'banana'`, () => {
+    // edit path:  'insert' ('b' of banana), 'match' (on leading a), rest are 'substitute'.
+    assert.isFalse(isSubstitutionAlignable('banana', 'apple'));
+  });
+
+  // Substitution:  not valid if too much is substituted, even if allowed via parameter.
+  it(`returns false:  'apple' => 'banana' (subs allowed)`, () => {
+    // edit path:  'insert' ('b' of banana), 'match' (on leading a), rest are 'substitute'.
+    // 1 match vs 4 substitute = no bueno.  It'd require too niche of a keyboard rule.
+    assert.isFalse(isSubstitutionAlignable('banana', 'apple', true));
+  });
+
+  it(`returns true: 'a' => 'à' (subs allowed)`, () => {
+    assert.isTrue(isSubstitutionAlignable('à', 'a', true));
+  });
+
+  // Leading substitution:  valid if enough of the remaining word matches.
+  // Could totally happen from a legit Keyman keyboard rule.
+  it(`returns true: 'can' => 'van' (subs allowed)`, () => {
+    assert.isTrue(isSubstitutionAlignable('van', 'can', true));
+  });
+
+  // Trailing substitution:  invalid if not allowed.
+  it(`returns false: 'can' => 'cap' (subs not allowed)`, () => {
+    assert.isFalse(isSubstitutionAlignable('cap', 'can'));
+  });
+
+  // Trailing substitution:  valid.
+  it(`returns false: 'can' => 'cap' (subs allowed)`, () => {
+    assert.isTrue(isSubstitutionAlignable('cap', 'can', true));
+  });
+
+  it(`returns true:  'clasts' => 'clasps' (subs allowed)`, () => {
+    assert.isTrue(isSubstitutionAlignable('clasps', 'clasts', true));
+  });
+
+  // random deletion at the start + later substitution = still permitted
+  it(`returns false:  'clasts' => 'lasps' (subs allowed)`, () => {
+    assert.isTrue(isSubstitutionAlignable('lasps', 'clasts', true));
+  });
+});
+
 describe('ContextTokenization', function() {
   describe("<constructor>", () => {
     it("constructs from just a token array", () => {
@@ -81,4 +155,4 @@ describe('ContextTokenization', function() {
 
     assert.deepEqual(tokenization.exampleInput, rawTextTokens);
   });
-});
\ No newline at end of file
+});
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
index f3df01d3d9..e85a605714 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
@@ -28,79 +28,6 @@ describe('ContextTracker', function() {
     }];
   }
 
-  describe('isSubstitutionAlignable', () => {
-    it(`returns true:  'ca' => 'can'`, () => {
-      assert.isTrue(ContextTracker.isSubstitutionAlignable('can', 'ca'));
-    });
-
-    // Leading word in context window starts sliding out of said window.
-    it(`returns true:  'can' => 'an'`, () => {
-      assert.isTrue(ContextTracker.isSubstitutionAlignable('an', 'can'));
-    });
-
-    // Same edits on both sides:  not valid.
-    it(`returns false: 'apple' => 'grapples'`, () => {
-      assert.isFalse(ContextTracker.isSubstitutionAlignable('grapples', 'apple'));
-    });
-
-    // Edits on one side:  valid.
-    it(`returns true: 'apple' => 'grapple'`, () => {
-      assert.isTrue(ContextTracker.isSubstitutionAlignable('grapple', 'apple'));
-    });
-
-    // Edits on one side:  valid.
-    it(`returns true: 'apple' => 'grapple'`, () => {
-      assert.isTrue(ContextTracker.isSubstitutionAlignable('apples', 'apple'));
-    });
-
-    // Same edits on both sides:  not valid.
-    it(`returns false: 'grapples' => 'apple'`, () => {
-      assert.isFalse(ContextTracker.isSubstitutionAlignable('apple', 'grapples'));
-    });
-
-    // Substitution:  not valid when not permitted via parameter.
-    it(`returns false:  'apple' => 'banana'`, () => {
-      // edit path:  'insert' ('b' of banana), 'match' (on leading a), rest are 'substitute'.
-      assert.isFalse(ContextTracker.isSubstitutionAlignable('banana', 'apple'));
-    });
-
-    // Substitution:  not valid if too much is substituted, even if allowed via parameter.
-    it(`returns false:  'apple' => 'banana' (subs allowed)`, () => {
-      // edit path:  'insert' ('b' of banana), 'match' (on leading a), rest are 'substitute'.
-      // 1 match vs 4 substitute = no bueno.  It'd require too niche of a keyboard rule.
-      assert.isFalse(ContextTracker.isSubstitutionAlignable('banana', 'apple', true));
-    });
-
-    it(`returns true: 'a' => 'à' (subs allowed)`, () => {
-      assert.isTrue(ContextTracker.isSubstitutionAlignable('à', 'a', true));
-    });
-
-    // Leading substitution:  valid if enough of the remaining word matches.
-    // Could totally happen from a legit Keyman keyboard rule.
-    it(`returns true: 'can' => 'van' (subs allowed)`, () => {
-      assert.isTrue(ContextTracker.isSubstitutionAlignable('van', 'can', true));
-    });
-
-    // Trailing substitution:  invalid if not allowed.
-    it(`returns false: 'can' => 'cap' (subs not allowed)`, () => {
-      assert.isFalse(ContextTracker.isSubstitutionAlignable('cap', 'can'));
-    });
-
-    // Trailing substitution:  valid.
-    it(`returns false: 'can' => 'cap' (subs allowed)`, () => {
-      assert.isTrue(ContextTracker.isSubstitutionAlignable('cap', 'can', true));
-    });
-
-    it(`returns true:  'clasts' => 'clasps' (subs allowed)`, () => {
-      assert.isTrue(ContextTracker.isSubstitutionAlignable('clasps', 'clasts', true));
-    });
-
-    // random deletion at the start + later substitution = still permitted
-    it(`returns false:  'clasts' => 'lasps' (subs allowed)`, () => {
-      assert.isTrue(ContextTracker.isSubstitutionAlignable('lasps', 'clasts', true));
-    });
-  });
-
   describe('attemptTokenizedAlignment', () => {
     it("properly matches and aligns when contexts match", () => {
       const baseContext = [

From 8153dc417e545a29c064fae6133d70d80c6dafe7 Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Wed, 30 Jul 2025 11:49:46 -0500
Subject: [PATCH 23/53] refactor(web): relocates tokenization-alignment method
 to ContextTokenization class

---
 .../main/correction/context-tokenization.ts   | 258 +++++++++++-
 .../src/main/correction/context-tracker.ts    | 250 +-----------
 .../edit-distance/context-tokenization.js     | 366 +++++++++++++++++-
 .../cases/edit-distance/context-tracker.js    | 312 ---------------
 4 files changed, 616 insertions(+), 570 deletions(-)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
index 9762c9d87d..79633c971c 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
@@ -1,13 +1,54 @@
 import { ContextToken } from './context-token.js';
-import { TrackedContextStateAlignment } from './context-tracker.js';
+import { ClassicalDistanceCalculation } from './classical-calculation.js';
+import { getEditPathLastMatch, isSubstitutionAlignable } from './alignment-helpers.js';
+
+/**
+ * Represents token-count values resulting from an alignment attempt between two
+ * different modeled context states.
+ */
+export type ContextStateAlignment = {
+  /**
+   * Denotes whether or not alignment is possible between two contexts.
+   */
+  canAlign: false
+} | {
+  /**
+   * Denotes whether or not alignment is possible between two contexts.
+   */
+  canAlign: true,
+  /**
+   * Notes the number of tokens added to the head of the 'incoming'/'new' context
+   * of the contexts being aligned.  If negative, the incoming context deleted
+   * a token found in the 'original' / base context.
+   *
+   * For the alignment, [base context index] + leadTokenShift = [incoming context index].
+   */
+  leadTokenShift: number,
+  /**
+   * The count of tokens perfectly aligned, with no need for edits, for two successfully-
+   * alignable contexts.
+   */
+  matchLength: number,
+  /**
+   * The count of tokens at the tail perfectly aligned (existing in both contexts) but
+   * edited for two successfully-alignable contexts.  These tokens directly follow those
+   * that need no edits.
+   */
+  tailEditLength: number,
+  /**
+   * The count of new tokens added at the end of the incoming context for two aligned contexts.
+   * If negative, the incoming context deleted a previously-existing token from the original.
+   */
+  tailTokenShift: number
+};
 
 export class ContextTokenization {
   readonly tokens: ContextToken[];
-  readonly alignment?: TrackedContextStateAlignment;
+  readonly alignment?: ContextStateAlignment;
 
   constructor(priorToClone: ContextTokenization);
-  constructor(tokens: ContextToken[], alignment?: TrackedContextStateAlignment);
-  constructor(param1: ContextToken[] | ContextTokenization, alignment?: TrackedContextStateAlignment) {
+  constructor(tokens: ContextToken[], alignment?: ContextStateAlignment);
+  constructor(param1: ContextToken[] | ContextTokenization, alignment?: ContextStateAlignment) {
     if(!(param1 instanceof ContextTokenization)) {
       const tokens = param1;
       this.tokens = [].concat(tokens);
@@ -35,4 +76,213 @@ export class ContextTokenization {
 
     return sequence;
   }
+
+  /**
+   * Determines the alignment between a new, incoming tokenization source and the
+   * tokenization modeled by the current instance.
+   * @param incomingTokenization Raw strings corresponding to the tokenization of the incoming context
+   * @returns Alignment data that details if and how the incoming tokenization aligns with
+   * the tokenization modeled by this instance.
+   */
+  computeAlignment(incomingTokenization: string[]): ContextStateAlignment {
+    // Map the tokenized state to an edit-distance friendly version.
+    const tokenizationToMatch = this.exampleInput;
+
+    // Inverted order, since 'match' existed before our new context.
+    let mapping = ClassicalDistanceCalculation.computeDistance(
+      tokenizationToMatch.map(value => ({key: value})),
+      incomingTokenization.map(value => ({key: value})),
+      // Diagonal width to consider must be at least 2, as adding a single
+      // whitespace after a token tends to add two tokens: one for whitespace,
+      // one for the empty token to follow it.
+      3
+    );
+
+    let editPath = mapping.editPath();
+    // Special case:  new context bootstrapping - first token often substitutes.
+    // The text length is small enough that no words should be able to rotate out the start of the context.
+    // Special handling needed in case of no 'match'; the rest of the method assumes at least one 'match'.
+    if(editPath.length <= 3 && (editPath[0] == 'substitute' || editPath[0] == 'match')) {
+      let matchCount = 0;
+      let subCount = 0;
+      for(let i = 0; i < editPath.length; i++) {
+        if(editPath[i] == 'substitute') {
+          subCount++;
+          if(!isSubstitutionAlignable(incomingTokenization[i], tokenizationToMatch[i], true)) {
+            return {
+              canAlign: false
+            };
+          }
+        } else if(editPath[i] == 'match') {
+          // If a substitution is already recorded, treat the 'match' as a substitution.
+          if(subCount > 0) {
+            subCount++;
+          } else {
+            matchCount++;
+          }
+        }
+      }
+
+      const insertCount = editPath.filter((entry) => entry == 'insert').length;
+      const deleteCount = editPath.filter((entry) => entry == 'delete').length;
+
+      return {
+        canAlign: true,
+        matchLength: matchCount,
+        leadTokenShift: 0,
+        tailEditLength: subCount,
+        tailTokenShift: insertCount - deleteCount
+      }
+    }
+
+    // From here on assumes that at least one 'match' exists on the path.
+    // It all works great... once the context is long enough for at least one stable token.
+    const firstMatch = editPath.indexOf('match');
+    const lastMatch = getEditPathLastMatch(editPath);
+    if(firstMatch == -1) {
+      // If there are no matches, there's no alignment.
+      return {
+        canAlign: false
+      };
+    }
+
+    // Transpositions are not allowed at the token level during context alignment.
+    if(editPath.find((entry) => entry.indexOf('transpose') > -1)) {
+      return {
+        canAlign: false
+      };
+    }
+
+    let matchLength = lastMatch - firstMatch + 1;
+    let tailInsertLength = 0;
+    let tailDeleteLength = 0;
+    for(let i = lastMatch; i < editPath.length; i++) {
+      if(editPath[i] == 'insert') {
+        tailInsertLength++;
+      } else if(editPath[i] == 'delete') {
+        tailDeleteLength++;
+      }
+    }
+    if(tailInsertLength > 0 && tailDeleteLength > 0) {
+      // Something's gone weird if this happens; that should appear as a substitution instead.
+      // Otherwise, we have a VERY niche edit scenario.
+      return {
+        canAlign: false
+      };
+    }
+    const tailSubstituteLength = (editPath.length - 1 - lastMatch) - tailInsertLength - tailDeleteLength;
+
+    // Assertion:  for a long context, the bulk of the edit path should be a
+    // continuous block of 'match' entries.  If there's anything else in
+    // the middle, we have a context mismatch.
+    if(firstMatch > -1) {
+      for(let i = firstMatch+1; i < lastMatch; i++) {
+        if(editPath[i] != 'match') {
+          return {
+            canAlign: false
+          };
+        }
+      }
+    }
+
+    // If we have a perfect match with a pre-existing context, no mutations have
+    // happened; we have a 100% perfect match.
+    if(firstMatch == 0 && lastMatch == editPath.length - 1) {
+      return {
+        canAlign: true,
+        leadTokenShift: 0,
+        matchLength,
+        tailEditLength: tailSubstituteLength,
+        tailTokenShift: tailInsertLength - tailDeleteLength
+      };
+    }
+
+    // The edit path calc tries to put substitutes first, before inserts.
+    // We don't want that on the leading edge.
+    const lastEarlyInsert = editPath.lastIndexOf('insert', firstMatch);
+    const firstSubstitute = editPath.indexOf('substitute');
+    if(firstSubstitute > -1 && firstSubstitute < firstMatch && firstSubstitute < lastEarlyInsert) {
+      editPath[firstSubstitute] = 'insert';
+      editPath[lastEarlyInsert] = 'substitute';
+    }
+
+    // If mutations HAVE happened, we need to double-check the context-state alignment.
+    let priorEdit: typeof editPath[0];
+    let leadTokensRemoved = 0;
+    let leadSubstitutions = 0;
+
+    // The `i` index below aligns based upon the index within the `tokenizationToMatch` sequence
+    // and how it would have to be edited to align to the `incomingTokenization` sequence.
+    for(let i = 0; i < firstMatch; i++) {
+      switch(editPath[i]) {
+        case 'delete':
+          // All deletions should appear at the sliding window edge; if a deletion appears
+          // after the edge, but before the first match, something's wrong.
+          if(priorEdit && priorEdit != 'delete') {
+            return {
+              canAlign: false
+            };
+          }
+          leadTokensRemoved++;
+          break;
+        case 'substitute':
+          // We only allow for one leading token to be substituted.
+          //
+          // Any extras in the front would be pure inserts, not substitutions, due to
+          // the sliding context window and its implications.
+          if(leadSubstitutions++ > 0) {
+            return {
+              canAlign: false
+            };
+          }
+
+          // Find the word before and after substitution.
+          const incomingSub = incomingTokenization[i - (leadTokensRemoved > 0 ? leadTokensRemoved : 0)];
+          const matchingSub = tokenizationToMatch[i + (leadTokensRemoved < 0 ? leadTokensRemoved : 0)];
+
+          // Double-check the word - does the 'substituted' word itself align?
+          if(!isSubstitutionAlignable(incomingSub, matchingSub)) {
+            return {
+              canAlign: false
+            };
+          }
+
+          // There's no major need to drop parts of a token being 'slid' out of the context window.
+          // We'll leave it intact and treat it as a 'match'
+          matchLength++;
+          break;
+        case 'insert':
+          // Only allow an insert at the leading edge, as with 'delete's.
+          if(priorEdit && priorEdit != 'insert') {
+            return {
+              canAlign: false
+            };
+          }
+          // In case of backspaces, it's also possible to 'insert' a 'new'
+          // token - an old one that's slid back into view.
+          leadTokensRemoved--;
+          break;
+        default:
+          // No 'match' can exist before the first found index for a 'match'.
+          // No 'transpose-' edits should exist within this section, either.
+          return {
+            canAlign: false
+          };
+      }
+      priorEdit = editPath[i];
+    }
+
+    // If we need some form of tail-token substitution verification, add that here.
+
+    return {
+      canAlign: true,
+      // leadTokensRemoved represents the number of tokens that must be removed from the base context
+      // when aligning the contexts.  Externally, it's more helpful to think in terms of the count added
+      // to the incoming context.
+      leadTokenShift: -leadTokensRemoved + 0, // add 0 in case of a 'negative zero', which affects unit tests.
+      matchLength,
+      tailEditLength: tailSubstituteLength,
+      tailTokenShift: tailInsertLength - tailDeleteLength
+    };
+  }
 }
\ No newline at end of file
diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
index c6eea80fa8..e3307e8e8d 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
@@ -1,7 +1,5 @@
 import { applyTransform, buildMergedTransform, Token } from '@keymanapp/models-templates';
 
-import { getEditPathLastMatch, isSubstitutionAlignable } from './alignment-helpers.js';
-import { ClassicalDistanceCalculation } from './classical-calculation.js';
 import TransformUtils from '../transformUtils.js';
 import { determineModelTokenizer } from '../model-helpers.js';
 import { tokenizeTransform, tokenizeTransformDistribution } from './transform-tokenization.js';
@@ -175,260 +173,14 @@ interface ContextMatchResult {
   tailTokensAdded: number;
 }
 
-/**
- * Represents token-count values resulting from an alignment attempt between two
- * different modeled context states.
- */
-export type TrackedContextStateAlignment = {
-  /**
-   * Denotes whether or not alignment is possible between two contexts.
-   */
-  canAlign: false
-} | {
-  /**
-   * Denotes whether or not alignment is possible between two contexts.
-   */
-  canAlign: true,
-  /**
-   * Notes the number of tokens added to the head of the 'incoming'/'new' context
-   * of the contexts being aligned.  If negative, the incoming context deleted
-   * a token found in the 'original' / base context.
-   *
-   * For the alignment, [base context index] + leadTokenShift = [incoming context index].
-   */
-  leadTokenShift: number,
-  /**
-   * The count of tokens perfectly aligned, with no need for edits, for two successfully-
-   * alignable contexts.
-   */
-  matchLength: number,
-  /**
-   * The count of tokens at the tail perfectly aligned (existing in both contexts) but
-   * edited for two successfully-alignable contexts.  These tokens directly follow those
-   * that need no edits.
-   */
-  tailEditLength: number,
-  /**
-   * The count of new tokens added at the end of the incoming context for two aligned contexts.
-   * If negative, the incoming context deleted a previously-existing token from the original.
-   */
-  tailTokenShift: number
-};
-
 export class ContextTracker extends CircularArray<TrackedContextState> {
-  static attemptTokenizedAlignment(
-    incomingTokenization: string[],
-    tokenizationToMatch: string[]
-  ): TrackedContextStateAlignment {
-
-    // Inverted order, since 'match' existed before our new context.
-    let mapping = ClassicalDistanceCalculation.computeDistance(
-      tokenizationToMatch.map(value => ({key: value})),
-      incomingTokenization.map(value => ({key: value})),
-      // Diagonal width to consider must be at least 2, as adding a single
-      // whitespace after a token tends to add two tokens: one for whitespace,
-      // one for the empty token to follow it.
-      3
-    );
-
-    let editPath = mapping.editPath();
-    // Special case:  new context bootstrapping - first token often substitutes.
-    // The text length is small enough that no words should be able to rotate out the start of the context.
-    // Special handling needed in case of no 'match'; the rest of the method assumes at least one 'match'.
-    if(editPath.length <= 3 && (editPath[0] == 'substitute' || editPath[0] == 'match')) {
-      let matchCount = 0;
-      let subCount = 0;
-      for(let i = 0; i < editPath.length; i++) {
-        if(editPath[i] == 'substitute') {
-          subCount++;
-          if(!isSubstitutionAlignable(incomingTokenization[i], tokenizationToMatch[i], true)) {
-            return {
-              canAlign: false
-            };
-          }
-        } else if(editPath[i] == 'match') {
-          // If a substitution is already recorded, treat the 'match' as a substitution.
-          if(subCount > 0) {
-            subCount++;
-          } else {
-            matchCount++;
-          }
-        }
-      }
-
-      const insertCount = editPath.filter((entry) => entry == 'insert').length;
-      const deleteCount = editPath.filter((entry) => entry == 'delete').length;
-
-      return {
-        canAlign: true,
-        matchLength: matchCount,
-        leadTokenShift: 0,
-        tailEditLength: subCount,
-        tailTokenShift: insertCount - deleteCount
-      }
-    }
-
-    // From here on assumes that at least one 'match' exists on the path.
-    // It all works great... once the context is long enough for at least one stable token.
-    const firstMatch = editPath.indexOf('match');
-    const lastMatch = getEditPathLastMatch(editPath);
-    if(firstMatch == -1) {
-      // If there are no matches, there's no alignment.
-      return {
-        canAlign: false
-      };
-    }
-
-    // Transpositions are not allowed at the token level during context alignment.
-    if(editPath.find((entry) => entry.indexOf('transpose') > -1)) {
-      return {
-        canAlign: false
-      };
-    }
-
-    let matchLength = lastMatch - firstMatch + 1;
-    let tailInsertLength = 0;
-    let tailDeleteLength = 0;
-    for(let i = lastMatch; i < editPath.length; i++) {
-      if(editPath[i] == 'insert') {
-        tailInsertLength++;
-      } else if(editPath[i] == 'delete') {
-        tailDeleteLength++;
-      }
-    }
-    if(tailInsertLength > 0 && tailDeleteLength > 0) {
-      // Something's gone weird if this happens; that should appear as a substitution instead.
-      // Otherwise, we have a VERY niche edit scenario.
-      return {
-        canAlign: false
-      };
-    }
-    const tailSubstituteLength = (editPath.length - 1 - lastMatch) - tailInsertLength - tailDeleteLength;
-
-    // Assertion:  for a long context, the bulk of the edit path should be a
-    // continuous block of 'match' entries.  If there's anything else in
-    // the middle, we have a context mismatch.
-    if(firstMatch > -1) {
-      for(let i = firstMatch+1; i < lastMatch; i++) {
-        if(editPath[i] != 'match') {
-          return {
-            canAlign: false
-          };
-        }
-      }
-    }
-
-    // If we have a perfect match with a pre-existing context, no mutations have
-    // happened; we have a 100% perfect match.
-    if(firstMatch == 0 && lastMatch == editPath.length - 1) {
-      return {
-        canAlign: true,
-        leadTokenShift: 0,
-        matchLength,
-        tailEditLength: tailSubstituteLength,
-        tailTokenShift: tailInsertLength - tailDeleteLength
-      };
-    }
-
-    // The edit path calc tries to put substitutes first, before inserts.
-    // We don't want that on the leading edge.
-    const lastEarlyInsert = editPath.lastIndexOf('insert', firstMatch);
-    const firstSubstitute = editPath.indexOf('substitute');
-    if(firstSubstitute > -1 && firstSubstitute < firstMatch && firstSubstitute < lastEarlyInsert) {
-      editPath[firstSubstitute] = 'insert';
-      editPath[lastEarlyInsert] = 'substitute';
-    }
-
-    // If mutations HAVE happened, we need to double-check the context-state alignment.
-    let priorEdit: typeof editPath[0];
-    let leadTokensRemoved = 0;
-    let leadSubstitutions = 0;
-
-    // The `i` index below aligns based upon the index within the `tokenizationToMatch` sequence
-    // and how it would have to be edited to align to the `incomingTokenization` sequence.
-    for(let i = 0; i < firstMatch; i++) {
-      switch(editPath[i]) {
-        case 'delete':
-          // All deletions should appear at the sliding window edge; if a deletion appears
-          // after the edge, but before the first match, something's wrong.
-          if(priorEdit && priorEdit != 'delete') {
-            return {
-              canAlign: false
-            };
-          }
-          leadTokensRemoved++;
-          break;
-        case 'substitute':
-          // We only allow for one leading token to be substituted.
-          //
-          // Any extras in the front would be pure inserts, not substitutions, due to
-          // the sliding context window and its implications.
-          if(leadSubstitutions++ > 0) {
-            return {
-              canAlign: false
-            };
-          }
-
-          // Find the word before and after substitution.
-          const incomingSub = incomingTokenization[i - (leadTokensRemoved > 0 ? leadTokensRemoved : 0)];
-          const matchingSub = tokenizationToMatch[i + (leadTokensRemoved < 0 ? leadTokensRemoved : 0)];
-
-          // Double-check the word - does the 'substituted' word itself align?
-          if(!isSubstitutionAlignable(incomingSub, matchingSub)) {
-            return {
-              canAlign: false
-            };
-          }
-
-          // There's no major need to drop parts of a token being 'slid' out of the context window.
-          // We'll leave it intact and treat it as a 'match'
-          matchLength++;
-          break;
-        case 'insert':
-          // Only allow an insert at the leading edge, as with 'delete's.
-          if(priorEdit && priorEdit != 'insert') {
-            return {
-              canAlign: false
-            };
-          }
-          // In case of backspaces, it's also possible to 'insert' a 'new'
-          // token - an old one that's slid back into view.
-          leadTokensRemoved--;
-          break;
-        default:
-          // No 'match' can exist before the first found index for a 'match'.
-          // No 'transpose-' edits should exist within this section, either.
-          return {
-            canAlign: false
-          };
-      }
-      priorEdit = editPath[i];
-    }
-
-    // If we need some form of tail-token substitution verification, add that here.
-
-    return {
-      canAlign: true,
-      // leadTokensRemoved represents the number of tokens that must be removed from the base context
-      // when aligning the contexts.  Externally, it's more helpful to think in terms of the count added
-      // to the incoming context.
-      leadTokenShift: -leadTokensRemoved + 0, // add 0 in case of a 'negative zero', which affects unit tests.
-      matchLength,
-      tailEditLength: tailSubstituteLength,
-      tailTokenShift: tailInsertLength - tailDeleteLength
-    };
-  }
-
   static attemptMatchContext(
     tokenizedContext: Token[],
     matchState: TrackedContextState,
     // the distribution should be tokenized already.
     transformSequenceDistribution?: Distribution<Transform[]>
   ): ContextMatchResult {
-    // Map the previous tokenized state to an edit-distance friendly version.
-    let matchContext: string[] = matchState.tokenization.exampleInput;
-
-    const alignmentResults = this.attemptTokenizedAlignment(tokenizedContext.map((token) => token.text), matchContext);
+    const alignmentResults = matchState.tokenization.computeAlignment(tokenizedContext.map((token) => token.text));
 
     if(!alignmentResults.canAlign) {
       return null;
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js
index 77b2e04994..cb42d5cb22 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js
@@ -1,18 +1,22 @@
 import { assert } from 'chai';
 import { isSubstitutionAlignable } from "#./correction/context-alignment.js";
-
+import { ContextTokenization } from "#./correction/context-tokenization.js";
 import { ContextToken } from '#./correction/context-token.js';
-import { ContextTokenization } from '#./correction/context-tokenization.js';
 
-import * as models from '#./models/index.js';
-import { default as defaultBreaker } from '@keymanapp/models-wordbreakers';
 import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs';
+import { TrieModel } from '#./models/index.js';
+import { default as defaultBreaker } from '@keymanapp/models-wordbreakers';
 
-var TrieModel = models.TrieModel;
+// TODO:  consider mocking out the need for SearchSpace stuff?
 
 var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'),
   {wordBreaker: defaultBreaker});
 
+function buildBaseTokenization(textTokens) {
+  const tokens = textTokens.map((entry) => new ContextToken(plainModel, entry));
+  return new ContextTokenization(tokens);
+}
+
 function toToken(text) {
   let isWhitespace = text == ' ';
   let token = new ContextToken(plainModel, text);
@@ -155,4 +159,356 @@ describe('ContextTokenization', function() {
 
     assert.deepEqual(tokenization.exampleInput, rawTextTokens);
   });
+
+  describe('computeAlignment', () => {
+    it("properly matches and aligns when contexts match", () => {
+      const baseContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [...baseContext];
+
+      const baseTokenization = buildBaseTokenization(baseContext);
+      const computedAlignment = baseTokenization.computeAlignment(newContext);
+
+      assert.deepEqual(computedAlignment, {
+        canAlign: true,
+        leadTokenShift: 0,
+        matchLength: 5,
+        tailEditLength: 0,
+        tailTokenShift: 0
+      });
+    });
+
+    it("detects unalignable contexts - no matching tokens", () => {
+      const baseContext = [
+        'swift', 'tan', 'wolf', 'leaped', 'across'
+      ];
+      const newContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+
+      const baseTokenization = buildBaseTokenization(baseContext);
+      const computedAlignment = baseTokenization.computeAlignment(newContext);
+
+      assert.deepEqual(computedAlignment, {canAlign: false});
+    });
+
+    it("detects unalignable contexts - too many mismatching tokens", () => {
+      const baseContext = [
+        'swift', 'tan', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+
+      const baseTokenization = buildBaseTokenization(baseContext);
+      const computedAlignment = baseTokenization.computeAlignment(newContext);
+
+      assert.deepEqual(computedAlignment, {canAlign: false});
+    });
+
+    it("fails alignment for leading-edge word substitutions", () => {
+      const baseContext = [
+        'swift', 'brown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+
+      const baseTokenization = buildBaseTokenization(baseContext);
+      const computedAlignment = baseTokenization.computeAlignment(newContext);
+
+      assert.deepEqual(computedAlignment, {canAlign: false});
+    });
+
+    it("fails alignment for small leading-edge word substitutions", () => {
+      const baseContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'sick', 'brown', 'fox', 'jumped', 'over'
+      ];
+
+      const baseTokenization = buildBaseTokenization(baseContext);
+      const computedAlignment = baseTokenization.computeAlignment(newContext);
+
+      assert.deepEqual(computedAlignment, {canAlign: false});
+    });
+
+    it("properly matches and aligns when lead token is modified", () => {
+      const baseContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'uick', 'brown', 'fox', 'jumped', 'over'
+      ];
+
+      const baseTokenization = buildBaseTokenization(baseContext);
+      const computedAlignment = baseTokenization.computeAlignment(newContext);
+
+      assert.deepEqual(computedAlignment, {
+        canAlign: true,
+        leadTokenShift: 0,
+        matchLength: 5,
+        tailEditLength: 0,
+        tailTokenShift: 0
+      });
+    });
+
+    it("properly matches and aligns when lead token is removed", () => {
+      const baseContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'brown', 'fox', 'jumped', 'over'
+      ];
+
+      const baseTokenization = buildBaseTokenization(baseContext);
+      const computedAlignment = baseTokenization.computeAlignment(newContext);
+
+      assert.deepEqual(computedAlignment, {
+        canAlign: true,
+        leadTokenShift: -1,
+        matchLength: 4,
+        tailEditLength: 0,
+        tailTokenShift: 0
+      });
+    });
+
+    it("properly matches and aligns when lead token is added", () => {
+      const baseContext = [
+        'brown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+
+      const baseTokenization = buildBaseTokenization(baseContext);
+      const computedAlignment = baseTokenization.computeAlignment(newContext);
+
+      assert.deepEqual(computedAlignment, {
+        canAlign: true,
+        leadTokenShift: 1,
+        matchLength: 4,
+        tailEditLength: 0,
+        tailTokenShift: 0
+      });
+    });
+
+    it("properly matches and aligns when lead tokens are removed and modified", () => {
+      const baseContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'ox', 'jumped', 'over'
+      ];
+
+      const baseTokenization = buildBaseTokenization(baseContext);
+      const computedAlignment = baseTokenization.computeAlignment(newContext);
+
+      assert.deepEqual(computedAlignment, {
+        canAlign: true,
+        leadTokenShift: -2,
+        matchLength: 3,
+        tailEditLength: 0,
+        tailTokenShift: 0
+      });
+    });
+
+    it("properly matches and aligns when lead tokens are added and modified", () => {
+      const baseContext = [
+        'rown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+
+      const baseTokenization = buildBaseTokenization(baseContext);
+      const computedAlignment = baseTokenization.computeAlignment(newContext);
+
+      assert.deepEqual(computedAlignment, {
+        canAlign: true,
+        leadTokenShift: 1,
+        matchLength: 4,
+        tailEditLength: 0,
+        tailTokenShift: 0
+      });
+    });
+
+    it("properly matches and aligns when lead token is removed and tail token is added", () => {
+      const baseContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'brown', 'fox', 'jumped', 'over', 'the'
+      ];
+
+      const baseTokenization = buildBaseTokenization(baseContext);
+      const computedAlignment = baseTokenization.computeAlignment(newContext);
+
+      assert.deepEqual(computedAlignment, {
+        canAlign: true,
+        leadTokenShift: -1,
+        matchLength: 4,
+        tailEditLength: 0,
+        tailTokenShift: 1
+      });
+    });
+
+    it("properly matches and aligns when lead token and tail token are modified", () => {
+      const baseContext = [
+        'quick', 'brown', 'fox', 'jumped', 'ove'
+      ];
+      const newContext = [
+        'uick', 'brown', 'fox', 'jumped', 'over'
+      ];
+
+      const baseTokenization = buildBaseTokenization(baseContext);
+      const computedAlignment = baseTokenization.computeAlignment(newContext);
+
+      assert.deepEqual(computedAlignment, {
+        canAlign: true,
+        leadTokenShift: 0,
+        matchLength: 4, // we treat 'quick' and 'uick' as the same
+        tailEditLength: 1,
+        tailTokenShift: 0
+      });
+    });
+
+    it("properly matches and aligns when lead token and tail token are modified + new token appended", () => {
+      const baseContext = [
+        'quick', 'brown', 'fox', 'jumped', 'ove'
+      ];
+      const newContext = [
+        'uick', 'brown', 'fox', 'jumped', 'over', 't'
+      ];
+
+      const baseTokenization = buildBaseTokenization(baseContext);
+      const computedAlignment = baseTokenization.computeAlignment(newContext);
+
+      assert.deepEqual(computedAlignment, {
+        canAlign: true,
+        leadTokenShift: 0,
+        matchLength: 4, // we treat 'quick' and 'uick' as the same
+        tailEditLength: 1,
+        tailTokenShift: 1
+      });
+    });
+
+    it("properly handles context window sliding backward", () => {
+      const baseContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'e', 'quick', 'brown', 'fox', 'jumped', 'ove'
+      ];
+
+      const baseTokenization = buildBaseTokenization(baseContext);
+      const computedAlignment = baseTokenization.computeAlignment(newContext);
+
+      assert.deepEqual(computedAlignment, {
+        canAlign: true,
+        leadTokenShift: 1,
+        matchLength: 4, // we treat 'quick' and 'uick' as the same
+        tailEditLength: 1,
+        tailTokenShift: 0
+      });
+    });
+
+    it("properly handles context window sliding far backward", () => {
+      const baseContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'the', 'quick', 'brown', 'fox', 'jumped'
+      ];
+
+      const baseTokenization = buildBaseTokenization(baseContext);
+      const computedAlignment = baseTokenization.computeAlignment(newContext);
+
+      assert.deepEqual(computedAlignment, {
+        canAlign: true,
+        leadTokenShift: 1,
+        matchLength: 4, // we treat 'quick' and 'uick' as the same
+        tailEditLength: 0,
+        tailTokenShift: -1
+      });
+    });
+
+    it("properly handles context window sliding farther backward", () => {
+      const baseContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'the', 'quick', 'brown', 'fox', 'jumpe'
+      ];
+
+      const baseTokenization = buildBaseTokenization(baseContext);
+      const computedAlignment = baseTokenization.computeAlignment(newContext);
+
+      assert.deepEqual(computedAlignment, {
+        canAlign: true,
+        leadTokenShift: 1,
+        matchLength: 3, // we treat 'quick' and 'uick' as the same
+        tailEditLength: 1,
+        tailTokenShift: -1
+      });
+    });
+
+    it("fails alignment for mid-head deletion", () => {
+      const baseContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'quick', 'fox', 'jumped', 'over'
+      ];
+
+      const baseTokenization = buildBaseTokenization(baseContext);
+      const computedAlignment = baseTokenization.computeAlignment(newContext);
+
+      assert.deepEqual(computedAlignment, {canAlign: false});
+    });
+
+    it("fails alignment for mid-head insertion", () => {
+      const baseContext = [
+        'quick', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+
+      const baseTokenization = buildBaseTokenization(baseContext);
+      const computedAlignment = baseTokenization.computeAlignment(newContext);
+
+      assert.deepEqual(computedAlignment, {canAlign: false});
+    });
+
+    it("fails alignment for mid-tail deletion", () => {
+      const baseContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'quick', 'brown', 'fox', 'over'
+      ];
+
+      const baseTokenization = buildBaseTokenization(baseContext);
+      const computedAlignment = baseTokenization.computeAlignment(newContext);
+
+      assert.deepEqual(computedAlignment, {canAlign: false});
+    });
+
+    it("fails alignment for mid-tail insertion", () => {
+      const baseContext = [
+        'quick', 'brown', 'fox', 'jumped', 'over'
+      ];
+      const newContext = [
+        'quick', 'brown', 'fox', 'jumped', 'far', 'over'
+      ];
+
+      const baseTokenization = buildBaseTokenization(baseContext);
+      const computedAlignment = baseTokenization.computeAlignment(newContext);
+
+      assert.deepEqual(computedAlignment, {canAlign: false});
+    });
+  });
 });
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
index e85a605714..624f6283a2 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
@@ -28,318 +28,6 @@ describe('ContextTracker', function() {
     }];
   }
 
-  describe('attemptTokenizedAlignment', () => {
-    it("properly matches and aligns when contexts match", () => {
-      const baseContext = [
-        'quick', 'brown', 'fox', 'jumped', 'over'
-      ];
-      const newContext = [...baseContext];
-
-      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
-      assert.deepEqual(computedAlignment, {
-        canAlign: true,
-        leadTokenShift: 0,
-        matchLength: 5,
-        tailEditLength: 0,
-        tailTokenShift: 0
-      });
-    });
-
-    it("detects unalignable contexts - no matching tokens", () => {
-      const baseContext = [
-        'swift', 'tan', 'wolf', 'leaped', 'across'
-      ];
-      const newContext = [
-        'quick', 'brown', 'fox', 'jumped', 'over'
-      ];
-
-      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
-      assert.deepEqual(computedAlignment, {canAlign: false});
-    });
-
-    it("detects unalignable contexts - too many mismatching tokens", () => {
-      const baseContext = [
-        'swift', 'tan', 'fox', 'jumped', 'over'
-      ];
-      const newContext = [
-        'quick', 'brown', 'fox', 'jumped', 'over'
-      ];
-
-      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
-      assert.deepEqual(computedAlignment, {canAlign: false});
-    });
-
-    it("fails alignment for leading-edge word substitutions", () => {
-      const baseContext = [
-        'swift', 'brown', 'fox', 'jumped', 'over'
-      ];
-      const newContext = [
-        'quick', 'brown', 'fox', 'jumped', 'over'
-      ];
-
-      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
-      assert.deepEqual(computedAlignment, {canAlign: false});
-    });
-
-    it("fails alignment for small leading-edge word substitutions", () => {
-      const baseContext = [
-        'quick', 'brown', 'fox', 'jumped', 'over'
-      ];
-      const newContext = [
-        'sick', 'brown', 'fox', 'jumped', 'over'
-      ];
-
-      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
-      assert.deepEqual(computedAlignment, {canAlign: false});
-    });
-
-    it("properly matches and aligns when lead token is modified", () => {
-      const baseContext = [
-        'quick', 'brown', 'fox', 'jumped', 'over'
-      ];
-      const newContext = [
-        'uick', 'brown', 'fox', 'jumped', 'over'
-      ];
-
-      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
-      assert.deepEqual(computedAlignment, {
-        canAlign: true,
-        leadTokenShift: 0,
-        matchLength: 5,
-        tailEditLength: 0,
-        tailTokenShift: 0
-      });
-    });
-
-    it("properly matches and aligns when lead token is removed", () => {
-      const baseContext = [
-        'quick', 'brown', 'fox', 'jumped', 'over'
-      ];
-      const newContext = [
-        'brown', 'fox', 'jumped', 'over'
-      ];
-
-      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
-      assert.deepEqual(computedAlignment, {
-        canAlign: true,
-        leadTokenShift: -1,
-        matchLength: 4,
-        tailEditLength: 0,
-        tailTokenShift: 0
-      });
-    });
-
-    it("properly matches and aligns when lead token is added", () => {
-      const baseContext = [
-        'brown', 'fox', 'jumped', 'over'
-      ];
-      const newContext = [
-        'quick', 'brown', 'fox', 'jumped', 'over'
-      ];
-
-      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
-      assert.deepEqual(computedAlignment, {
-        canAlign: true,
-        leadTokenShift: 1,
-        matchLength: 4,
-        tailEditLength: 0,
-        tailTokenShift: 0
-      });
-    });
-
-    it("properly matches and aligns when lead tokens are removed and modified", () => {
-      const baseContext = [
-        'quick', 'brown', 'fox', 'jumped', 'over'
-      ];
-      const newContext = [
-        'ox', 'jumped', 'over'
-      ];
-
-      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
-      assert.deepEqual(computedAlignment, {
-        canAlign: true,
-        leadTokenShift: -2,
-        matchLength: 3,
-        tailEditLength: 0,
-        tailTokenShift: 0
-      });
-    });
-
-    it("properly matches and aligns when lead tokens are added and modified", () => {
-      const baseContext = [
-        'rown', 'fox', 'jumped', 'over'
-      ];
-      const newContext = [
-        'quick', 'brown', 'fox', 'jumped', 'over'
-      ];
-
-      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
-      assert.deepEqual(computedAlignment, {
-        canAlign: true,
-        leadTokenShift: 1,
-        matchLength: 4,
-        tailEditLength: 0,
-        tailTokenShift: 0
-      });
-    });
-
-    it("properly matches and aligns when lead token is removed and tail token is added", () => {
-      const baseContext = [
-        'quick', 'brown', 'fox', 'jumped', 'over'
-      ];
-      const newContext = [
-        'brown', 'fox', 'jumped', 'over', 'the'
-      ];
-
-      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
-      assert.deepEqual(computedAlignment, {
-        canAlign: true,
-        leadTokenShift: -1,
-        matchLength: 4,
-        tailEditLength: 0,
-        tailTokenShift: 1
-      });
-    });
-
-    it("properly matches and aligns when lead token and tail token are modified", () => {
-      const baseContext = [
-        'quick', 'brown', 'fox', 'jumped', 'ove'
-      ];
-      const newContext = [
-        'uick', 'brown', 'fox', 'jumped', 'over'
-      ];
-
-      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
-      assert.deepEqual(computedAlignment, {
-        canAlign: true,
-        leadTokenShift: 0,
-        matchLength: 4, // we treat 'quick' and 'uick' as the same
-        tailEditLength: 1,
-        tailTokenShift: 0
-      });
-    });
-
-    it("properly matches and aligns when lead token and tail token are modified + new token appended", () => {
-      const baseContext = [
-        'quick', 'brown', 'fox', 'jumped', 'ove'
-      ];
-      const newContext = [
-        'uick', 'brown', 'fox', 'jumped', 'over', 't'
-      ];
-
-      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
-      assert.deepEqual(computedAlignment, {
-        canAlign: true,
-        leadTokenShift: 0,
-        matchLength: 4, // we treat 'quick' and 'uick' as the same
-        tailEditLength: 1,
-        tailTokenShift: 1
-      });
-    });
-
-    it("properly handles context window sliding backward", () => {
-      const baseContext = [
-        'quick', 'brown', 'fox', 'jumped', 'over'
-      ];
-      const newContext = [
-        'e', 'quick', 'brown', 'fox', 'jumped', 'ove'
-      ];
-
-      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
-      assert.deepEqual(computedAlignment, {
-        canAlign: true,
-        leadTokenShift: 1,
-        matchLength: 4, // we treat 'quick' and 'uick' as the same
-        tailEditLength: 1,
-        tailTokenShift: 0
-      });
-    });
-
-    it("properly handles context window sliding far backward", () => {
-      const baseContext = [
-        'quick', 'brown', 'fox', 'jumped', 'over'
-      ];
-      const newContext = [
-        'the', 'quick', 'brown', 'fox', 'jumped'
-      ];
-
-      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
-      assert.deepEqual(computedAlignment, {
-        canAlign: true,
-        leadTokenShift: 1,
-        matchLength: 4, // we treat 'quick' and 'uick' as the same
-        tailEditLength: 0,
-        tailTokenShift: -1
-      });
-    });
-
-    it("properly handles context window sliding farther backward", () => {
-      const baseContext = [
-        'quick', 'brown', 'fox', 'jumped', 'over'
-      ];
-      const newContext = [
-        'the', 'quick', 'brown', 'fox', 'jumpe'
-      ];
-
-      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
-      assert.deepEqual(computedAlignment, {
-        canAlign: true,
-        leadTokenShift: 1,
-        matchLength: 3, // we treat 'quick' and 'uick' as the same
-        tailEditLength: 1,
-        tailTokenShift: -1
-      });
-    });
-
-    it("fails alignment for mid-head deletion", () => {
-      const baseContext = [
-        'quick', 'brown', 'fox', 'jumped', 'over'
-      ];
-      const newContext = [
-        'quick', 'fox', 'jumped', 'over'
-      ];
-
-      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
-      assert.deepEqual(computedAlignment, {canAlign: false});
-    });
-
-    it("fails alignment for mid-head insertion", () => {
-      const baseContext = [
-        'quick', 'fox', 'jumped', 'over'
-      ];
-      const newContext = [
-        'quick', 'brown', 'fox', 'jumped', 'over'
-      ];
-
-      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
-      assert.deepEqual(computedAlignment, {canAlign: false});
-    });
-
-    it("fails alignment for mid-tail deletion", () => {
-      const baseContext = [
-        'quick', 'brown', 'fox', 'jumped', 'over'
-      ];
-      const newContext = [
-        'quick', 'brown', 'fox', 'over'
-      ];
-
-      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
-      assert.deepEqual(computedAlignment, {canAlign: false});
-    });
-
-    it("fails alignment for mid-tail insertion", () => {
-      const baseContext = [
-        'quick', 'brown', 'fox', 'jumped', 'over'
-      ];
-      const newContext = [
-        'quick', 'brown', 'fox', 'jumped', 'far', 'over'
-      ];
-
-      const computedAlignment = ContextTracker.attemptTokenizedAlignment(newContext, baseContext);
-      assert.deepEqual(computedAlignment, {canAlign: false});
-    });
-  });
-
   describe('attemptMatchContext', function() {
     it("properly matches and aligns when lead token is removed", function() {
       let existingContext = models.tokenize(defaultBreaker, {

From 5bd5ccfca43fa72c1e117d3997622b5835715a4a Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Wed, 30 Jul 2025 15:36:15 -0500
Subject: [PATCH 24/53] refactor(web): refactor tracked-context state and usage
 pattern

Note that this will temporarily cause predictive-text to throw out older context states (that may be valid rewind targets) in favor of newer ones (that are known to be invalid rewind targets) due to the current context-state caching logic.  That'll be resolved later, in an upcoming PR - and preferably, this one shouldn't merge without that one.
---
 .../src/main/correction/context-state.ts      |  89 +++++++++++
 .../src/main/correction/context-tracker.ts    |  84 ++++-------
 .../worker-thread/src/main/predict-helpers.ts |   5 +-
 .../cases/edit-distance/context-tracker.js    | 139 +++++++++---------
 4 files changed, 185 insertions(+), 132 deletions(-)
 create mode 100644 web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts
new file mode 100644
index 0000000000..244f4f362b
--- /dev/null
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts
@@ -0,0 +1,89 @@
+import { ContextTokenization } from './context-tokenization.js';
+
+import { LexicalModelTypes } from '@keymanapp/common-types';
+import Context = LexicalModelTypes.Context;
+import Distribution = LexicalModelTypes.Distribution;
+import LexicalModel = LexicalModelTypes.LexicalModel;
+import Suggestion = LexicalModelTypes.Suggestion;
+import Transform = LexicalModelTypes.Transform;
+
+/**
+ * Represents a state of the active context at some point in time along with the
+ * results of all related, reusable predictive-text operations.
+ */
+export class ContextState {
+  /**
+   * The context window in view for the represented Context state,
+   * as passed between the predictive-text worker and its host.
+   */
+  context: Context;
+
+  /**
+   * The active lexical model operating upon the Context.
+   */
+  model: LexicalModel;
+
+  /**
+   * Denotes the most likely tokenization for the represented Context.
+   */
+  tokenization: ContextTokenization;
+
+  /**
+   * Denotes the keystroke-sourced Transform that was last applied to a
+   * prior ContextState.
+   *
+   * Note:  if this specific ContextState resulted from applying a
+   * Suggestion, this may not match text seen in the current Context!
+   */
+  appliedInput?: Transform;
+
+  /**
+   * Denotes all keystroke data contributing to ContextTokens seen in
+   * .tokenization.  For each contributing context transition, its ID
+   * may be used to retrieve the original fat-finger distribution for
+   * potential keystroke effects.
+   */
+  inputTransforms: Map<number, Distribution<Transform>>;
+
+  /**
+   * The full set of Suggestions produced for the transition to this context state.
+   */
+  suggestions: Suggestion[];
+
+  /**
+   * If set, denotes the suggestion ID for the suggestion (from .suggestions) that
+   * was applied for the final transition to this context state.
+   */
+  appliedSuggestionId?: number;
+
+  /**
+   * Indicates whether or not the applied suggestion (if it exists) was applied
+   * directly by the user.
+   *
+   * - `true` if directly applied via banner interaction or other explicitly-intended
+   * behaviors
+   * - `false` if indirectly applied (say, by triggering whitespace/punctuation input)
+   * - `undefined` if no suggestion has been applied.
+   */
+  isManuallyApplied?: boolean;
+
+  constructor(stateToClone: ContextState);
+  constructor(context: Context, model: LexicalModel);
+  constructor(param1: Context | ContextState, model?: LexicalModel) {
+    if(!(param1 instanceof ContextState)) {
+      const context = param1;
+      this.context = context;
+      this.model = model;
+    } else {
+      const stateToClone = param1;
+
+      Object.assign(this, stateToClone);
+      this.inputTransforms = new Map(stateToClone.inputTransforms);
+      this.tokenization = new ContextTokenization(stateToClone.tokenization);
+
+      // A shallow copy of the array is fine, but we'd be best off
+      // not aliasing the array itself.
+      this.suggestions = [].concat(stateToClone.suggestions);
+    }
+  }
+}
\ No newline at end of file
diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
index e3307e8e8d..f08cc8fac4 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
@@ -1,4 +1,4 @@
-import { applyTransform, buildMergedTransform, Token } from '@keymanapp/models-templates';
+import { applyTransform, buildMergedTransform } from '@keymanapp/models-templates';
 
 import TransformUtils from '../transformUtils.js';
 import { determineModelTokenizer } from '../model-helpers.js';
@@ -7,49 +7,10 @@ import { LexicalModelTypes } from '@keymanapp/common-types';
 import Context = LexicalModelTypes.Context;
 import Distribution = LexicalModelTypes.Distribution;
 import LexicalModel = LexicalModelTypes.LexicalModel;
-import Suggestion = LexicalModelTypes.Suggestion;
 import Transform = LexicalModelTypes.Transform;
 import { ContextToken } from './context-token.js';
 import { ContextTokenization } from './context-tokenization.js';
-
-export class TrackedContextSuggestion {
-  suggestion: Suggestion;
-  tokenWidth: number;
-}
-
-export class TrackedContextState {
-  // Stores the post-transform Context.  Useful as a debugging reference, but also used to
-  // pre-validate context state matches in case of discarded changes from multitaps.
-  taggedContext: Context;
-  model: LexicalModel;
-
-  tokenization: ContextTokenization;
-
-  /**
-   * How many tokens were removed from the start of the best-matching ancestor.
-   * Useful for restoring older states, e.g., when the user moves the caret backwards, we can recover the context at that position.
-   */
-  indexOffset: number;
-
-  constructor(source: TrackedContextState);
-  constructor(model: LexicalModel);
-  constructor(obj: TrackedContextState | LexicalModel) {
-    if(obj instanceof TrackedContextState) {
-      let source = obj;
-      // Be sure to deep-copy the tokens!  Pointer-aliasing is bad here.
-      this.tokenization = new ContextTokenization(source.tokenization.tokens.map((token) => new ContextToken(token)));
-
-      this.indexOffset = 0;
-      this.model = obj.model;
-      this.taggedContext = obj.taggedContext;
-    } else {
-      let lexicalModel = obj;
-      this.tokenization = null;
-      this.indexOffset = Number.MIN_SAFE_INTEGER;
-      this.model = lexicalModel;
-    }
-  }
-}
+import { ContextState } from './context-state.js';
 
 class CircularArray<Item> {
   static readonly DEFAULT_ARRAY_SIZE = 5;
@@ -146,13 +107,13 @@ interface ContextMatchResult {
   /**
    * Represents the current state of the context after applying incoming keystroke data.
    */
-  state: TrackedContextState;
+  state: ContextState;
 
   /**
    * Represents the previously-cached context state that best matches `state` if available.
    * May be `null` if no such state could be found within the context-state cache.
    */
-  baseState: TrackedContextState;
+  baseState: ContextState;
 
   /**
    * Indicates the portion of the incoming keystroke data, if any, that applies to
@@ -173,13 +134,16 @@ interface ContextMatchResult {
   tailTokensAdded: number;
 }
 
-export class ContextTracker extends CircularArray<TrackedContextState> {
+export class ContextTracker extends CircularArray<ContextState> {
   static attemptMatchContext(
-    tokenizedContext: Token[],
-    matchState: TrackedContextState,
+    context: Context,
+    lexicalModel: LexicalModel,
+    matchState: ContextState,
     // the distribution should be tokenized already.
     transformSequenceDistribution?: Distribution<Transform[]>
   ): ContextMatchResult {
+    const tokenizedContext = determineModelTokenizer(lexicalModel)(context).left;
+
     const alignmentResults = matchState.tokenization.computeAlignment(tokenizedContext.map((token) => token.text));
 
     if(!alignmentResults.canAlign) {
@@ -223,7 +187,7 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
 
     // If no TAIL mutations have happened, we're safe to return now.
     if(tailEditLength == 0 && tailTokenShift == 0) {
-      const state = new TrackedContextState(matchState);
+      const state = new ContextState(context, lexicalModel);
       state.tokenization = new ContextTokenization(tokenization, alignmentResults);
       return {
         state: state,
@@ -292,7 +256,7 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
       let token: ContextToken;
 
       if(isBackspace) {
-        token = new ContextToken(matchState.model, incomingToken.text);
+        token = new ContextToken(lexicalModel, incomingToken.text);
         token.searchSpace.inputSequence.forEach((entry) => entry[0].sample.id = primaryInput.id);
       } else {
         // Assumption:  there have been no intervening keystrokes since the last well-aligned context.
@@ -338,7 +302,7 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
           preservationTransform = preservationTransform && primaryInput ? buildMergedTransform(preservationTransform, primaryInput) : (primaryInput ?? preservationTransform);
         }
 
-        let pushedToken = new ContextToken(matchState.model);
+        let pushedToken = new ContextToken(lexicalModel);
 
         // TODO:  assumes that there was no shift in wordbreaking from the
         // prior context to the current one.  This may actually be a major
@@ -380,7 +344,7 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
       }
     }
 
-    const state = new TrackedContextState(matchState);
+    const state = new ContextState(context, lexicalModel);
     state.tokenization = new ContextTokenization(tokenization, alignmentResults);
 
     return {
@@ -392,10 +356,13 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
     };
   }
 
+  // Aim:  relocate to ContextState in some form.
   private static modelContextState(
-    tokenizedContext: Token[],
+    context: Context,
     lexicalModel: LexicalModel
-  ): TrackedContextState {
+  ): ContextState {
+    const tokenizedContext = determineModelTokenizer(lexicalModel)(context).left;
+
     let baseTokens = tokenizedContext.map(function(entry) {
       let token = new ContextToken(lexicalModel, entry.text);
 
@@ -407,7 +374,7 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
     });
 
     // And now build the final context state object, which includes whitespace 'tokens'.
-    let state = new TrackedContextState(lexicalModel);
+    let state = new ContextState(context, lexicalModel);
     const tokenization: ContextToken[] = [];
 
     while(baseTokens.length > 0) {
@@ -423,6 +390,7 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
     return state;
   }
 
+  // Aim:  relocate to ContextState in some form... or ContextTransition?
   /**
    * Compares the current, post-input context against the most recently-seen contexts from previous prediction calls, returning
    * the most information-rich `TrackedContextState` possible.  If a match is found, the state will be annotated with the
@@ -485,7 +453,7 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
         // Skip intermediate multitap-produced contexts.
         // When multitapping, we skip all contexts from prior taps within the same interaction,
         // but not any contexts from before the multitap started.
-        const priorTaggedContext = priorMatchState.taggedContext;
+        const priorTaggedContext = priorMatchState.context;
         if(priorTaggedContext && transformDistribution && transformDistribution.length > 0) {
           // Using the potential `matchState` + the incoming transform, do the results line up for
           // our observed context?  If not, skip it.
@@ -502,7 +470,7 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
           continue;
         }
 
-        let result = ContextTracker.attemptMatchContext(tokenizedContext.left, this.item(i), tokenizedDistribution);
+        let result = ContextTracker.attemptMatchContext(context, model, this.item(i), tokenizedDistribution);
 
         if(result?.state) {
           // Keep it reasonably current!  And it's probably fine to have it more than once
@@ -513,7 +481,7 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
             this.enqueue(priorMatchState);
           }
 
-          result.state.taggedContext = context;
+          result.state.context = context;
           if(result.state != this.item(i)) {
             this.enqueue(result.state);
           }
@@ -527,8 +495,8 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
     //
     // Assumption:  as a caret needs to move to context before any actual transform distributions occur,
     // this state is only reached on caret moves; thus, transformDistribution is actually just a single null transform.
-    let state = ContextTracker.modelContextState(tokenizedContext.left, model);
-    state.taggedContext = context;
+    let state = ContextTracker.modelContextState(context, model);
+    state.context = context;
     this.enqueue(state);
     return { state, baseState: null, headTokensRemoved: 0, tailTokensAdded: 0 };
   }
diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts
index 30ab7eb9c9..12c2547f4f 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts
@@ -3,7 +3,8 @@ import { KMWString } from '@keymanapp/web-utils';
 
 import TransformUtils from './transformUtils.js';
 import { determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js';
-import { ContextTracker, TrackedContextState } from './correction/context-tracker.js';
+import { ContextTracker } from './correction/context-tracker.js';
+import { ContextState } from './correction/context-state.js';
 import { ExecutionTimer } from './correction/execution-timer.js';
 import ModelCompositor from './model-compositor.js';
 import { LexicalModelTypes } from '@keymanapp/common-types';
@@ -108,7 +109,7 @@ export async function correctAndEnumerate(
    *
    * Otherwise, is `null`.
    */
-  postContextState?: TrackedContextState;
+  postContextState?: ContextState;
 
   /**
    * The suggestions generated based on the user's input state.
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
index 624f6283a2..de21766c0e 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
@@ -30,19 +30,20 @@ describe('ContextTracker', function() {
 
   describe('attemptMatchContext', function() {
     it("properly matches and aligns when lead token is removed", function() {
-      let existingContext = models.tokenize(defaultBreaker, {
+      let existingContext = {
         left: "an apple a day keeps the doctor"
-      });
+      };
       let transform = {
         insert: '',
         deleteLeft: 0
-      }
-      let newContext = deepCopy(existingContext);
-      newContext.left.splice(0, 1);
+      };
+      let newContext = {
+        left: " apple a day keeps the doctor"
+      };
       let rawTokens = [" ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
 
-      let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
-      let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
+      let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel);
+      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseContextMatch, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
       assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       assert.equal(newContextMatch.headTokensRemoved, 1);
@@ -50,19 +51,20 @@ describe('ContextTracker', function() {
     });
 
     it("properly matches and aligns when lead token + following whitespace are removed", function() {
-      let existingContext = models.tokenize(defaultBreaker, {
+      let existingContext = {
         left: "an apple a day keeps the doctor"
-      });
+      };
       let transform = {
         insert: '',
         deleteLeft: 0
-      }
-      let newContext = deepCopy(existingContext);
-      newContext.left.splice(0, 2);
+      };
+      let newContext = {
+        left: "apple a day keeps the doctor"
+      };
       let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
 
-      let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
-      let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
+      let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel);
+      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseContextMatch, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
       assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       assert.equal(newContextMatch.headTokensRemoved, 2);
@@ -70,20 +72,20 @@ describe('ContextTracker', function() {
     });
 
     it("properly matches and aligns when final token is edited", function() {
-      let existingContext = models.tokenize(defaultBreaker, {
+      let existingContext = {
         left: "an apple a day keeps the docto"
-      });
+      };
       let transform = {
         insert: 'r',
         deleteLeft: 0
       }
-      let newContext = models.tokenize(defaultBreaker, {
+      let newContext = {
         left: "an apple a day keeps the doctor"
-      });
+      };
       let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
 
-      let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
-      let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
+      let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel);
+      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseContextMatch, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
       assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       assert.equal(newContextMatch.headTokensRemoved, 0);
@@ -92,20 +94,20 @@ describe('ContextTracker', function() {
 
     // Needs improved context-state management (due to 2x tokens)
     it("properly matches and aligns when a 'wordbreak' is added", function() {
-      let existingContext = models.tokenize(defaultBreaker, {
+      let existingContext = {
         left: "an apple a day keeps the doctor"
-      });
+      };
       let transform = {
         insert: ' ',
         deleteLeft: 0
       }
-      let newContext = models.tokenize(defaultBreaker, {
+      let newContext = {
         left: "an apple a day keeps the doctor "
-      });
+      };
       let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
 
-      let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
-      let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
+      let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel);
+      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseContextMatch, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
       assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       // We want to preserve the added whitespace when predicting a token that follows after it.
@@ -120,20 +122,20 @@ describe('ContextTracker', function() {
     });
 
     it("properly matches and aligns when a 'wordbreak' is removed via backspace", function() {
-      let existingContext = models.tokenize(defaultBreaker, {
+      let existingContext = {
         left: "an apple a day keeps the doctor "
-      });
+      };
       let transform = {
         insert: '',
         deleteLeft: 1
       }
-      let newContext = models.tokenize(defaultBreaker, {
+      let newContext = {
         left: "an apple a day keeps the doctor"
-      });
+      };
       let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
 
-      let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
-      let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
+      let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel);
+      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseContextMatch, toWrapperDistribution(transform));
       assert.isOk(newContextMatch?.state);
       assert.deepEqual(newContextMatch?.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
 
@@ -143,20 +145,20 @@ describe('ContextTracker', function() {
     });
 
     it("properly matches and aligns when an implied 'wordbreak' occurs (as when following \"'\")", function() {
-      let existingContext = models.tokenize(defaultBreaker, {
+      let existingContext = {
         left: "'"
-      });
+      };
       let transform = {
         insert: 'a',
         deleteLeft: 0
       }
-      let newContext = models.tokenize(defaultBreaker, {
+      let newContext = {
         left: "'a"
-      });
+      };
       let rawTokens = ["'", "a"];
 
-      let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
-      let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
+      let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel);
+      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseContextMatch, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
       assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       assert.deepEqual(newContextMatch.preservationTransform, { insert: '', deleteLeft: 0 });
@@ -172,20 +174,20 @@ describe('ContextTracker', function() {
 
     // Needs improved context-state management (due to 2x tokens)
     it("properly matches and aligns when lead token is removed AND a 'wordbreak' is added'", function() {
-      let existingContext = models.tokenize(defaultBreaker, {
+      let existingContext = {
         left: "an apple a day keeps the doctor"
-      });
+      };
       let transform = {
         insert: ' ',
         deleteLeft: 0
       }
-      let newContext = models.tokenize(defaultBreaker, {
+      let newContext = {
         left: "apple a day keeps the doctor "
-      });
+      };
       let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
 
-      let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
-      let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
+      let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel);
+      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseContextMatch, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
       assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       // We want to preserve the added whitespace when predicting a token that follows after it.
@@ -201,21 +203,22 @@ describe('ContextTracker', function() {
     });
 
     it("properly matches and aligns when initial token is modified AND a 'wordbreak' is added'", function() {
-      let existingContext = models.tokenize(defaultBreaker, {
+      let existingContext = {
         left: "an"
-      });
+      };
       let transform = {
         insert: 'd ',
         deleteLeft: 0
       }
-      let newContext = models.tokenize(defaultBreaker, {
+      let newContext = {
         left: "and "
-      });
+      };
       let rawTokens = ["and", " ", ""];
 
-      let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
+      let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel);
       let newContextMatch = ContextTracker.attemptMatchContext(
-        newContext.left,
+        newContext,
+        plainModel,
         baseContextMatch,
         tokenizeTransformDistribution(tokenizer, {left: "an"}, [{sample: transform, p: 1}])
       );
@@ -234,21 +237,22 @@ describe('ContextTracker', function() {
     });
 
     it("properly matches and aligns when tail token is modified AND a 'wordbreak' is added'", function() {
-      let existingContext = models.tokenize(defaultBreaker, {
+      let existingContext = {
         left: "apple a day keeps the doc"
-      });
+      };
       let transform = {
         insert: 'tor ',
         deleteLeft: 0
       }
-      let newContext = models.tokenize(defaultBreaker, {
+      let newContext = {
         left: "apple a day keeps the doctor "
-      });
+      };
       let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
 
-      let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
+      let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel);
       let newContextMatch = ContextTracker.attemptMatchContext(
-        newContext.left,
+        newContext,
+        plainModel,
         baseContextMatch,
         tokenizeTransformDistribution(tokenizer, {left: "apple a day keeps the doc"}, [{sample: transform, p: 1}])
       );
@@ -271,7 +275,7 @@ describe('ContextTracker', function() {
         left: "text'"
       });
       assert.equal(baseContext.left.length, 1);
-      let baseContextMatch = ContextTracker.modelContextState(baseContext.left, plainModel);
+      let baseContextMatch = ContextTracker.modelContextState({left: "text'"}, plainModel);
 
       // Now the actual check.
       let newContext = models.tokenize(defaultBreaker, {
@@ -286,7 +290,8 @@ describe('ContextTracker', function() {
         deleteLeft: 0
       }
       let problemContextMatch = ContextTracker.attemptMatchContext(
-        newContext.left,
+        {left: "text'\""},
+        plainModel,
         baseContextMatch,
         tokenizeTransformDistribution(tokenizer, {left: "text'"}, [{sample: transform, p: 1}])
       );
@@ -296,28 +301,18 @@ describe('ContextTracker', function() {
 
   describe('modelContextState', function() {
     it('models without final wordbreak', function() {
-      let tokenized = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"].map((entry) => {
-        return {
-          text: entry,
-          isWhitespace: entry == " "
-        };
-      });
+      let context = { left: "an apple a day keeps the doctor" };
       let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
 
-      let state = ContextTracker.modelContextState(tokenized, plainModel);
+      let state = ContextTracker.modelContextState(context, plainModel);
       assert.deepEqual(state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
     });
 
     it('models with final wordbreak', function() {
-      let tokenized = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""].map((entry) => {
-        return {
-          text: entry,
-          isWhitespace: entry == " "
-        };
-      });
+      let context = { left: "an apple a day keeps the doctor " };
       let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
 
-      let state = ContextTracker.modelContextState(tokenized, plainModel);
+      let state = ContextTracker.modelContextState(context, plainModel);
       assert.deepEqual(state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
     });
   });

From f3390b61bf04b531ff0b0bec577e88bd8fdf982b Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Thu, 31 Jul 2025 08:47:41 -0500
Subject: [PATCH 25/53] refactor(web): refactor modelContextState as
 ContextState.initFromReset

---
 .../src/main/correction/context-state.ts      | 52 ++++++++++++++++-
 .../src/main/correction/context-tracker.ts    | 43 ++------------
 .../cases/edit-distance/context-tracker.js    | 58 ++++++++++++-------
 3 files changed, 90 insertions(+), 63 deletions(-)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts
index 244f4f362b..3c7643b23e 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts
@@ -6,6 +6,8 @@ import Distribution = LexicalModelTypes.Distribution;
 import LexicalModel = LexicalModelTypes.LexicalModel;
 import Suggestion = LexicalModelTypes.Suggestion;
 import Transform = LexicalModelTypes.Transform;
+import { ContextToken } from './context-token.js';
+import { determineModelTokenizer } from '#./model-helpers.js';
 
 /**
  * Represents a state of the active context at some point in time along with the
@@ -16,12 +18,12 @@ export class ContextState {
    * The context window in view for the represented Context state,
    * as passed between the predictive-text worker and its host.
    */
-  context: Context;
+  private _context: Context;
 
   /**
    * The active lexical model operating upon the Context.
    */
-  model: LexicalModel;
+  readonly model: LexicalModel;
 
   /**
    * Denotes the most likely tokenization for the represented Context.
@@ -72,7 +74,7 @@ export class ContextState {
   constructor(param1: Context | ContextState, model?: LexicalModel) {
     if(!(param1 instanceof ContextState)) {
       const context = param1;
-      this.context = context;
+      this._context = context;
       this.model = model;
     } else {
       const stateToClone = param1;
@@ -86,4 +88,48 @@ export class ContextState {
       this.suggestions = [].concat(stateToClone.suggestions);
     }
   }
+
+  /**
+   * The context window in view for the represented Context state,
+   * as passed between the predictive-text worker and its host.
+   */
+  get context(): Context {
+    return this._context;
+  }
+
+  /**
+   * Initializes the ContextState instance for use when no valid prior
+   * information is available - typically, immediately after engine
+   * initialization or a context reset.
+   */
+  initFromReset() {
+    const context = this.context;
+    const lexicalModel = this.model;
+
+    const tokenizedContext = determineModelTokenizer(lexicalModel)(context).left;
+
+    let baseTokens = tokenizedContext.map(function(entry) {
+      let token = new ContextToken(lexicalModel, entry.text);
+
+      if(entry.isWhitespace) {
+        token.isWhitespace = true;
+      }
+
+      return token;
+    });
+
+    // And now build the final context state object, which includes whitespace 'tokens'.);
+    const tokenization: ContextToken[] = [];
+
+    while(baseTokens.length > 0) {
+      tokenization.push(baseTokens.splice(0, 1)[0]);
+    }
+
+    if(tokenization.length == 0) {
+      let token = new ContextToken(lexicalModel);
+      tokenization.push(token);
+    }
+
+    this.tokenization = new ContextTokenization(tokenization);
+  }
 }
\ No newline at end of file
diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
index f08cc8fac4..2fcfdac161 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
@@ -135,12 +135,14 @@ interface ContextMatchResult {
 }
 
 export class ContextTracker extends CircularArray<ContextState> {
+  // Aim:  relocate to ContextTransition in some form?
+  // Or can we split it up in some manner across the different types?
   static attemptMatchContext(
     context: Context,
     lexicalModel: LexicalModel,
     matchState: ContextState,
     // the distribution should be tokenized already.
-    transformSequenceDistribution?: Distribution<Transform[]>
+    transformSequenceDistribution?: Distribution<Transform[]> // transform distribution is needed here.
   ): ContextMatchResult {
     const tokenizedContext = determineModelTokenizer(lexicalModel)(context).left;
 
@@ -356,40 +358,6 @@ export class ContextTracker extends CircularArray<ContextState> {
     };
   }
 
-  // Aim:  relocate to ContextState in some form.
-  private static modelContextState(
-    context: Context,
-    lexicalModel: LexicalModel
-  ): ContextState {
-    const tokenizedContext = determineModelTokenizer(lexicalModel)(context).left;
-
-    let baseTokens = tokenizedContext.map(function(entry) {
-      let token = new ContextToken(lexicalModel, entry.text);
-
-      if(entry.isWhitespace) {
-        token.isWhitespace = true;
-      }
-
-      return token;
-    });
-
-    // And now build the final context state object, which includes whitespace 'tokens'.
-    let state = new ContextState(context, lexicalModel);
-    const tokenization: ContextToken[] = [];
-
-    while(baseTokens.length > 0) {
-      tokenization.push(baseTokens.splice(0, 1)[0]);
-    }
-
-    if(tokenization.length == 0) {
-      let token = new ContextToken(lexicalModel);
-      tokenization.push(token);
-    }
-
-    state.tokenization = new ContextTokenization(tokenization);
-    return state;
-  }
-
   // Aim:  relocate to ContextState in some form... or ContextTransition?
   /**
    * Compares the current, post-input context against the most recently-seen contexts from previous prediction calls, returning
@@ -481,7 +449,6 @@ export class ContextTracker extends CircularArray<ContextState> {
             this.enqueue(priorMatchState);
           }
 
-          result.state.context = context;
           if(result.state != this.item(i)) {
             this.enqueue(result.state);
           }
@@ -495,8 +462,8 @@ export class ContextTracker extends CircularArray<ContextState> {
     //
     // Assumption:  as a caret needs to move to context before any actual transform distributions occur,
     // this state is only reached on caret moves; thus, transformDistribution is actually just a single null transform.
-    let state = ContextTracker.modelContextState(context, model);
-    state.context = context;
+    let state = new ContextState(context, model);
+    state.initFromReset();
     this.enqueue(state);
     return { state, baseState: null, headTokensRemoved: 0, tailTokensAdded: 0 };
   }
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
index de21766c0e..3bc068135f 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
@@ -11,6 +11,7 @@ import { default as defaultBreaker } from '@keymanapp/models-wordbreakers';
 import { deepCopy } from '@keymanapp/web-utils';
 
 import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs';
+import { ContextState } from '#./correction/context-state.js';
 
 const tokenizer = determineModelTokenizer(new models.DummyModel({wordbreaker: defaultBreaker}));
 
@@ -42,8 +43,9 @@ describe('ContextTracker', function() {
       };
       let rawTokens = [" ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
 
-      let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel);
-      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseContextMatch, toWrapperDistribution(transform));
+      let baseState = new ContextState(existingContext, plainModel);
+      baseState.initFromReset();
+      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
       assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       assert.equal(newContextMatch.headTokensRemoved, 1);
@@ -63,8 +65,9 @@ describe('ContextTracker', function() {
       };
       let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
 
-      let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel);
-      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseContextMatch, toWrapperDistribution(transform));
+      let baseState = new ContextState(existingContext, plainModel);
+      baseState.initFromReset();
+      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
       assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       assert.equal(newContextMatch.headTokensRemoved, 2);
@@ -84,8 +87,9 @@ describe('ContextTracker', function() {
       };
       let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
 
-      let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel);
-      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseContextMatch, toWrapperDistribution(transform));
+      let baseState = new ContextState(existingContext, plainModel);
+      baseState.initFromReset();
+      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
       assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       assert.equal(newContextMatch.headTokensRemoved, 0);
@@ -106,8 +110,9 @@ describe('ContextTracker', function() {
       };
       let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
 
-      let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel);
-      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseContextMatch, toWrapperDistribution(transform));
+      let baseState = new ContextState(existingContext, plainModel);
+      baseState.initFromReset();
+      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
       assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       // We want to preserve the added whitespace when predicting a token that follows after it.
@@ -134,8 +139,9 @@ describe('ContextTracker', function() {
       };
       let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
 
-      let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel);
-      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseContextMatch, toWrapperDistribution(transform));
+      let baseState = new ContextState(existingContext, plainModel);
+      baseState.initFromReset();
+      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
       assert.isOk(newContextMatch?.state);
       assert.deepEqual(newContextMatch?.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
 
@@ -157,8 +163,9 @@ describe('ContextTracker', function() {
       };
       let rawTokens = ["'", "a"];
 
-      let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel);
-      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseContextMatch, toWrapperDistribution(transform));
+      let baseState = new ContextState(existingContext, plainModel);
+      baseState.initFromReset();
+      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
       assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       assert.deepEqual(newContextMatch.preservationTransform, { insert: '', deleteLeft: 0 });
@@ -186,8 +193,9 @@ describe('ContextTracker', function() {
       };
       let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
 
-      let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel);
-      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseContextMatch, toWrapperDistribution(transform));
+      let baseState = new ContextState(existingContext, plainModel);
+      baseState.initFromReset();
+      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
       assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       // We want to preserve the added whitespace when predicting a token that follows after it.
@@ -215,11 +223,12 @@ describe('ContextTracker', function() {
       };
       let rawTokens = ["and", " ", ""];
 
-      let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel);
+      let baseState = new ContextState(existingContext, plainModel);
+      baseState.initFromReset();
       let newContextMatch = ContextTracker.attemptMatchContext(
         newContext,
         plainModel,
-        baseContextMatch,
+        baseState,
         tokenizeTransformDistribution(tokenizer, {left: "an"}, [{sample: transform, p: 1}])
       );
       assert.isNotNull(newContextMatch?.state);
@@ -249,11 +258,12 @@ describe('ContextTracker', function() {
       };
       let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
 
-      let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel);
+      let baseState = new ContextState(existingContext, plainModel);
+      baseState.initFromReset();
       let newContextMatch = ContextTracker.attemptMatchContext(
         newContext,
         plainModel,
-        baseContextMatch,
+        baseState,
         tokenizeTransformDistribution(tokenizer, {left: "apple a day keeps the doc"}, [{sample: transform, p: 1}])
       );
       assert.isNotNull(newContextMatch?.state);
@@ -275,7 +285,9 @@ describe('ContextTracker', function() {
         left: "text'"
       });
       assert.equal(baseContext.left.length, 1);
-      let baseContextMatch = ContextTracker.modelContextState({left: "text'"}, plainModel);
+
+      let baseState = new ContextState({ left: "text'" }, plainModel);
+      baseState.initFromReset();
 
       // Now the actual check.
       let newContext = models.tokenize(defaultBreaker, {
@@ -292,7 +304,7 @@ describe('ContextTracker', function() {
       let problemContextMatch = ContextTracker.attemptMatchContext(
         {left: "text'\""},
         plainModel,
-        baseContextMatch,
+        baseState,
         tokenizeTransformDistribution(tokenizer, {left: "text'"}, [{sample: transform, p: 1}])
       );
       assert.isNull(problemContextMatch);
@@ -304,7 +316,8 @@ describe('ContextTracker', function() {
       let context = { left: "an apple a day keeps the doctor" };
       let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
 
-      let state = ContextTracker.modelContextState(context, plainModel);
+      let state = new ContextState(context, plainModel);
+      state.initFromReset();
       assert.deepEqual(state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
     });
 
@@ -312,7 +325,8 @@ describe('ContextTracker', function() {
       let context = { left: "an apple a day keeps the doctor " };
       let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
 
-      let state = ContextTracker.modelContextState(context, plainModel);
+      let state = new ContextState(context, plainModel);
+      state.initFromReset();
       assert.deepEqual(state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
     });
   });

From ee5d09727424e76d4273a038a0340e0fe8be3600 Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Fri, 1 Aug 2025 12:53:56 -0500
Subject: [PATCH 26/53] feat(web): add unit tests for
 ContextState.initFromReset

---
 .../cases/edit-distance/context-state.js      | 68 +++++++++++++++++++
 1 file changed, 68 insertions(+)
 create mode 100644 web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-state.js

diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-state.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-state.js
new file mode 100644
index 0000000000..3ac00011ff
--- /dev/null
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-state.js
@@ -0,0 +1,68 @@
+import { assert } from 'chai';
+
+import { ContextState } from '#./correction/context-state.js';
+import * as models from '#./models/index.js';
+
+import { default as defaultBreaker } from '@keymanapp/models-wordbreakers';
+
+import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs';
+
+var TrieModel = models.TrieModel;
+
+var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'),
+  {wordBreaker: defaultBreaker});
+
+describe('ContextState', () => {
+  it('<constructor>', () => {
+    let context = { left: '', right: '' };
+    let state = new ContextState(context, plainModel);
+
+    assert.equal(state.context, context);
+    assert.equal(state.model, plainModel);
+    assert.isNotOk(state.tokenization);
+    assert.isUndefined(state.isManuallyApplied);
+    assert.isNotOk(state.suggestions);
+    assert.isNotOk(state.appliedSuggestionId);
+  });
+
+  describe('initFromReset', () => {
+    it('<empty context>', () => {
+      let context = { left: '', right: '' };
+      let state = new ContextState(context, plainModel);
+
+      assert.isNotOk(state.tokenization);
+      assert.equal(state.context, context);
+
+      state.initFromReset();
+      assert.isOk(state.tokenization);
+      assert.equal(state.tokenization.tokens.length, 1);
+      assert.equal(state.tokenization.tail.exampleInput, '');
+    });
+
+    it('with initial text (without ending whitespace', () => {
+      let context = { left: 'the quick brown fox', right: '' };
+      let state = new ContextState(context, plainModel);
+
+      assert.isNotOk(state.tokenization);
+      assert.equal(state.context, context);
+
+      state.initFromReset();
+      assert.isOk(state.tokenization);
+      assert.equal(state.tokenization.tokens.length, 7);
+      assert.deepEqual(state.tokenization.exampleInput, ['the', ' ', 'quick', ' ', 'brown', ' ', 'fox']);
+    });
+
+    it('with initial text (with ending whitespace', () => {
+      let context = { left: 'the quick brown fox ', right: '' };
+      let state = new ContextState(context, plainModel);
+
+      assert.isNotOk(state.tokenization);
+      assert.equal(state.context, context);
+
+      state.initFromReset();
+      assert.isOk(state.tokenization);
+      assert.equal(state.tokenization.tokens.length, 9);
+      assert.deepEqual(state.tokenization.exampleInput, ['the', ' ', 'quick', ' ', 'brown', ' ', 'fox', ' ', '']);
+    });
+  });
+});
\ No newline at end of file

From 3a56d244641422ee7bcd53ccd7959d65b811e7a7 Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Fri, 1 Aug 2025 08:32:53 -0500
Subject: [PATCH 27/53] refactor(web): refactor tokenization of fat-finger
 distribution transforms into dedicated method

---
 .../src/main/correction/context-tracker.ts    | 31 +++--------
 .../main/correction/transform-tokenization.ts | 52 +++++++++++++++++++
 .../cases/edit-distance/context-tracker.js    | 11 ++--
 3 files changed, 63 insertions(+), 31 deletions(-)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
index 2fcfdac161..035bcd99c0 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
@@ -2,7 +2,7 @@ import { applyTransform, buildMergedTransform } from '@keymanapp/models-template
 
 import TransformUtils from '../transformUtils.js';
 import { determineModelTokenizer } from '../model-helpers.js';
-import { tokenizeTransform, tokenizeTransformDistribution } from './transform-tokenization.js';
+import { tokenizeAndFilterDistribution } from './transform-tokenization.js';
 import { LexicalModelTypes } from '@keymanapp/common-types';
 import Context = LexicalModelTypes.Context;
 import Distribution = LexicalModelTypes.Distribution;
@@ -142,9 +142,10 @@ export class ContextTracker extends CircularArray<ContextState> {
     lexicalModel: LexicalModel,
     matchState: ContextState,
     // the distribution should be tokenized already.
-    transformSequenceDistribution?: Distribution<Transform[]> // transform distribution is needed here.
+    transformDistribution?: Distribution<Transform> // transform distribution is needed here.
   ): ContextMatchResult {
     const tokenizedContext = determineModelTokenizer(lexicalModel)(context).left;
+    const transformSequenceDistribution = tokenizeAndFilterDistribution(context, lexicalModel, transformDistribution);
 
     const alignmentResults = matchState.tokenization.computeAlignment(tokenizedContext.map((token) => token.text));
 
@@ -164,7 +165,7 @@ export class ContextTracker extends CircularArray<ContextState> {
     // If we have a perfect match with a pre-existing context, no mutations have
     // happened; just re-use the old context state.
     if(tailEditLength == 0 && leadTokenShift == 0 && tailTokenShift == 0) {
-      return { state: matchState, baseState: matchState, headTokensRemoved: 0, tailTokensAdded: 0 };
+      return { state: matchState, baseState: matchState, headTokensRemoved: 0, tailTokensAdded: 0 };;
     } else {
       // If we didn't get any input, we really should perfectly match
       // a previous context state.  If such a state is out of our cache,
@@ -381,37 +382,17 @@ export class ContextTracker extends CircularArray<ContextState> {
       throw "This lexical model does not provide adequate data for correction algorithms and context reuse";
     }
 
-    let tokenize = determineModelTokenizer(model);
-
     if(transformDistribution?.length == 0) {
       transformDistribution = null;
     }
     const inputTransform = transformDistribution?.[0];
-    let transformTokenLength = 0;
-    let tokenizedDistribution: Distribution<Transform[]> = null;
     if(inputTransform) {
       // These two methods apply transforms internally; do not mutate context here.
       // This particularly matters for the 'distribution' variant.
-
-      // What if a pre-whitespace token has a final substitution as PART of an edit?
-      // Say, ['apple', ' ', ''] => ['apply', ' ', 'n']
-      // For now... we can't really handle that case well - modeling the 'e' => 'y' part.
-      // Will likely require improvements to tokenizeTransform(), which doesn't yet handle
-      // deleteLeft tokenization for transforms spanning tokens & whitespace.
-      //
-      // See: #14361.
-      // There's a good shot attemptTokenizedAlignment would be useful for it.
-      transformTokenLength = tokenizeTransform(tokenize, context, inputTransform.sample).length;
-      tokenizedDistribution = tokenizeTransformDistribution(tokenize, context, transformDistribution);
-
-      // Now we update the context used for context-state management based upon our input.
       context = applyTransform(inputTransform.sample, context);
-
-      // While we lack phrase-based / phrase-oriented prediction support, we'll just extract the
-      // set that matches the token length that results from our input.
-      tokenizedDistribution = tokenizedDistribution.filter((entry) => entry.sample.length == transformTokenLength);
     }
 
+    const tokenize = determineModelTokenizer(model);
     const tokenizedContext = tokenize(context);
 
     if(tokenizedContext.left.length > 0) {
@@ -438,7 +419,7 @@ export class ContextTracker extends CircularArray<ContextState> {
           continue;
         }
 
-        let result = ContextTracker.attemptMatchContext(context, model, this.item(i), tokenizedDistribution);
+        let result = ContextTracker.attemptMatchContext(context, model, this.item(i), transformDistribution);
 
         if(result?.state) {
           // Keep it reasonably current!  And it's probably fine to have it more than once
diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/transform-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/transform-tokenization.ts
index 291245014c..62c157ee24 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/transform-tokenization.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/transform-tokenization.ts
@@ -1,8 +1,10 @@
 import { LexicalModelTypes } from '@keymanapp/common-types';
 import Context = LexicalModelTypes.Context;
 import Distribution = LexicalModelTypes.Distribution;
+import LexicalModel = LexicalModelTypes.LexicalModel;
 import Transform = LexicalModelTypes.Transform;
 import { applyTransform, type Tokenization } from "@keymanapp/models-templates";
+import { determineModelTokenizer } from '#./model-helpers.js';
 
 /**
  * Determines a tokenization-aware sequence of (`Transform`) edits, one per
@@ -85,4 +87,54 @@ export function tokenizeTransformDistribution(
       p: transform.p
     };
   });
+}
+
+/**
+ * Given an incoming distribution of Transforms, this method applies
+ * `tokenizeTransform` for each, mapping each transform to its tokenized form in
+ * the returned distribution.
+ *
+ * It then filters out all incoming Transforms that do not result in the same
+ * number of tokens as the "primary input" when applied, as the context-tracker
+ * and predictive-text engine cannot handle word-breaking divergence well at
+ * this time.
+ * @param context
+ * @param model
+ * @param transformDistribution
+ * @returns
+ */
+export function tokenizeAndFilterDistribution(
+  context: Context,
+  model: LexicalModel,
+  transformDistribution?: Distribution<Transform>
+) {
+  let tokenize = determineModelTokenizer(model);
+  const inputTransform = transformDistribution?.[0];
+
+  let transformTokenLength = 0;
+  let tokenizedDistribution: Distribution<Transform[]> = null;
+  if(inputTransform) {
+    // These two methods apply transforms internally; do not mutate context here.
+    // This particularly matters for the 'distribution' variant.
+
+    // What if a pre-whitespace token has a final substitution as PART of an edit?
+    // Say, ['apple', ' ', ''] => ['apply', ' ', 'n']
+    // For now... we can't really handle that case well - modeling the 'e' => 'y' part.
+    // Will likely require improvements to tokenizeTransform(), which doesn't yet handle
+    // deleteLeft tokenization for transforms spanning tokens & whitespace.
+    //
+    // See: #14361.
+    // There's a good shot attemptTokenizedAlignment would be useful for it.
+    transformTokenLength = tokenizeTransform(tokenize, context, inputTransform.sample).length;
+    tokenizedDistribution = tokenizeTransformDistribution(tokenize, context, transformDistribution);
+
+    // Now we update the context used for context-state management based upon our input.
+    context = applyTransform(inputTransform.sample, context);
+
+    // While we lack phrase-based / phrase-oriented prediction support, we'll just extract the
+    // set that matches the token length that results from our input.
+    tokenizedDistribution = tokenizedDistribution.filter((entry) => entry.sample.length == transformTokenLength);
+  }
+
+  return tokenizedDistribution;
 }
\ No newline at end of file
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
index 3bc068135f..4e2a8c142e 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
@@ -21,10 +21,9 @@ var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'),
   {wordBreaker: wordBreakers.default});
 
 describe('ContextTracker', function() {
-  function toWrapperDistribution(transforms) {
-    transforms = Array.isArray(transforms) ? transforms : [transforms];
+  function toWrapperDistribution(transform) {
     return [{
-      sample: transforms,
+      sample: transform,
       p: 1.0
     }];
   }
@@ -229,7 +228,7 @@ describe('ContextTracker', function() {
         newContext,
         plainModel,
         baseState,
-        tokenizeTransformDistribution(tokenizer, {left: "an"}, [{sample: transform, p: 1}])
+        [{sample: transform, p: 1}]
       );
       assert.isNotNull(newContextMatch?.state);
       assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
@@ -264,7 +263,7 @@ describe('ContextTracker', function() {
         newContext,
         plainModel,
         baseState,
-        tokenizeTransformDistribution(tokenizer, {left: "apple a day keeps the doc"}, [{sample: transform, p: 1}])
+        [{sample: transform, p: 1}]
       );
       assert.isNotNull(newContextMatch?.state);
       assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
@@ -305,7 +304,7 @@ describe('ContextTracker', function() {
         {left: "text'\""},
         plainModel,
         baseState,
-        tokenizeTransformDistribution(tokenizer, {left: "text'"}, [{sample: transform, p: 1}])
+        [{sample: transform, p: 1}]
       );
       assert.isNull(problemContextMatch);
     });

From daf2ad5481aa5977f490cf66667b1ca34a0865b5 Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Thu, 31 Jul 2025 13:16:16 -0500
Subject: [PATCH 28/53] refactor(web): refactor tracked-context
 state-transition return object

---
 .../src/main/correction/context-tracker.ts    | 74 +++++----------
 .../src/main/correction/context-transition.ts | 90 +++++++++++++++++++
 .../src/main/model-compositor.ts              |  2 +-
 .../worker-thread/src/main/predict-helpers.ts |  6 +-
 .../cases/edit-distance/context-tracker.js    | 90 +++++++++----------
 5 files changed, 160 insertions(+), 102 deletions(-)
 create mode 100644 web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
index 035bcd99c0..7a07618fed 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
@@ -11,6 +11,7 @@ import Transform = LexicalModelTypes.Transform;
 import { ContextToken } from './context-token.js';
 import { ContextTokenization } from './context-tokenization.js';
 import { ContextState } from './context-state.js';
+import { ContextTransition } from './context-transition.js';
 
 class CircularArray<Item> {
   static readonly DEFAULT_ARRAY_SIZE = 5;
@@ -103,37 +104,6 @@ class CircularArray<Item> {
   }
 }
 
-interface ContextMatchResult {
-  /**
-   * Represents the current state of the context after applying incoming keystroke data.
-   */
-  state: ContextState;
-
-  /**
-   * Represents the previously-cached context state that best matches `state` if available.
-   * May be `null` if no such state could be found within the context-state cache.
-   */
-  baseState: ContextState;
-
-  /**
-   * Indicates the portion of the incoming keystroke data, if any, that applies to
-   * tokens before the last pre-caret token and thus should not be replaced by predictions
-   * based upon `state`.  If the provided context state + the incoming transform do not
-   * adequately match the current context, the match attempt will fail with a `null` result.
-   *
-   * Should generally be non-null if the token before the caret did not previously exist.
-   *
-   * The result may be null if it does not match the prior context state or if bookkeeping
-   * based upon it is problematic - say, if wordbreaking effects shift due to new input,
-   * causing a mismatch with the prior state's tokenization.
-   * (Refer to #12494 for an example case.)
-   */
-  preservationTransform?: Transform;
-
-  headTokensRemoved: number;
-  tailTokensAdded: number;
-}
-
 export class ContextTracker extends CircularArray<ContextState> {
   // Aim:  relocate to ContextTransition in some form?
   // Or can we split it up in some manner across the different types?
@@ -143,8 +113,10 @@ export class ContextTracker extends CircularArray<ContextState> {
     matchState: ContextState,
     // the distribution should be tokenized already.
     transformDistribution?: Distribution<Transform> // transform distribution is needed here.
-  ): ContextMatchResult {
+  ): ContextTransition {
     const tokenizedContext = determineModelTokenizer(lexicalModel)(context).left;
+    const baseTransition = new ContextTransition(matchState, matchState.appliedInput?.id);
+
     const transformSequenceDistribution = tokenizeAndFilterDistribution(context, lexicalModel, transformDistribution);
 
     const alignmentResults = matchState.tokenization.computeAlignment(tokenizedContext.map((token) => token.text));
@@ -165,7 +137,8 @@ export class ContextTracker extends CircularArray<ContextState> {
     // If we have a perfect match with a pre-existing context, no mutations have
     // happened; just re-use the old context state.
     if(tailEditLength == 0 && leadTokenShift == 0 && tailTokenShift == 0) {
-      return { state: matchState, baseState: matchState, headTokensRemoved: 0, tailTokensAdded: 0 };;
+      baseTransition.replaceFinal(new ContextState(matchState), transformDistribution);
+      return baseTransition;
     } else {
       // If we didn't get any input, we really should perfectly match
       // a previous context state.  If such a state is out of our cache,
@@ -192,12 +165,8 @@ export class ContextTracker extends CircularArray<ContextState> {
     if(tailEditLength == 0 && tailTokenShift == 0) {
       const state = new ContextState(context, lexicalModel);
       state.tokenization = new ContextTokenization(tokenization, alignmentResults);
-      return {
-        state: state,
-        baseState: matchState,
-        headTokensRemoved: -leadTokenShift,
-        tailTokensAdded: tailTokenShift
-      }
+      baseTransition.replaceFinal(state, transformDistribution);
+      return baseTransition;
     }
 
     // ***
@@ -349,14 +318,8 @@ export class ContextTracker extends CircularArray<ContextState> {
 
     const state = new ContextState(context, lexicalModel);
     state.tokenization = new ContextTokenization(tokenization, alignmentResults);
-
-    return {
-      state,
-      baseState: matchState,
-      preservationTransform,
-      headTokensRemoved: alignmentResults.leadTokenShift < 0 ? -alignmentResults.leadTokenShift : 0,
-      tailTokensAdded: alignmentResults.tailTokenShift
-    };
+    baseTransition.replaceFinal(state, transformDistribution, preservationTransform);
+    return baseTransition;
   }
 
   // Aim:  relocate to ContextState in some form... or ContextTransition?
@@ -375,7 +338,7 @@ export class ContextTracker extends CircularArray<ContextState> {
     context: Context,
     transformDistribution?: Distribution<Transform>,
     preserveMatchState?: boolean
-  ): ContextMatchResult {
+  ): ContextTransition {
     if(!model.traverseFromRoot) {
       // Assumption:  LexicalModel provides a valid traverseFromRoot function.  (Is technically optional)
       // Without it, no 'corrections' may be made; the model can only be used to predict, not correct.
@@ -385,6 +348,7 @@ export class ContextTracker extends CircularArray<ContextState> {
     if(transformDistribution?.length == 0) {
       transformDistribution = null;
     }
+
     const inputTransform = transformDistribution?.[0];
     if(inputTransform) {
       // These two methods apply transforms internally; do not mutate context here.
@@ -421,17 +385,17 @@ export class ContextTracker extends CircularArray<ContextState> {
 
         let result = ContextTracker.attemptMatchContext(context, model, this.item(i), transformDistribution);
 
-        if(result?.state) {
+        if(result?.final) {
           // Keep it reasonably current!  And it's probably fine to have it more than once
           // in the history.  However, if it's the most current already, there's no need
           // to refresh it.
-          if(this.newest != result.state && this.newest != priorMatchState) {
+          if(this.newest != result.final && this.newest != priorMatchState) {
             // Already has a taggedContext.
             this.enqueue(priorMatchState);
           }
 
-          if(result.state != this.item(i)) {
-            this.enqueue(result.state);
+          if(result.final != this.item(i)) {
+            this.enqueue(result.final);
           }
           return result;
         }
@@ -446,7 +410,11 @@ export class ContextTracker extends CircularArray<ContextState> {
     let state = new ContextState(context, model);
     state.initFromReset();
     this.enqueue(state);
-    return { state, baseState: null, headTokensRemoved: 0, tailTokensAdded: 0 };
+    const transition = new ContextTransition(state, /* TODO:  we need a clear value here in the future! */ null);
+    // Hacky, but holds the course for now.  This should only really happen from context resets, which can
+    // then use a different path.
+    transition.replaceFinal(state, []);
+    return transition;
   }
 
   clearCache() {
diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts
new file mode 100644
index 0000000000..abc0653db7
--- /dev/null
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts
@@ -0,0 +1,90 @@
+import { ContextState } from './context-state.js';
+
+import { LexicalModelTypes } from '@keymanapp/common-types';
+import Distribution = LexicalModelTypes.Distribution;
+import Transform = LexicalModelTypes.Transform;
+
+export class ContextTransition {
+  private states: [ContextState, ContextState];
+  private baseIndex = 0;
+
+  inputDistribution?: Distribution<Transform>;
+  // The transform ID in play.
+  private _transitionId?: number;
+
+  /**
+   * Indicates the portion of the incoming keystroke data, if any, that applies to
+   * tokens before the last pre-caret token and thus should not be replaced by predictions
+   * based upon `state`.  If the provided context state + the incoming transform do not
+   * adequately match the current context, the match attempt will fail with a `null` result.
+   *
+   * Should generally be non-null if the token before the caret did not previously exist.
+   *
+   * The result may be null if it does not match the prior context state or if bookkeeping
+   * based upon it is problematic - say, if wordbreaking effects shift due to new input,
+   * causing a mismatch with the prior state's tokenization.
+   * (Refer to #12494 for an example case.)
+   */
+  preservationTransform?: Transform;
+
+  constructor(context: ContextState, transitionId: number);
+  constructor(baseTransition: ContextTransition);
+  constructor(param: ContextState | ContextTransition, transitionId?: number) {
+    if(!(param instanceof ContextTransition)) {
+      const contextState = param;
+      // We're initializing a ContextTransition from a blank or reset context.
+      const baseState = contextState;
+      this.states = [baseState, null];
+      this._transitionId = transitionId;
+    } else {
+      const baseTransition = param;
+      Object.assign(this, baseTransition);
+
+      // These need to be deep-copied.
+      this.states = baseTransition.states.map((entry) => new ContextState(entry)) as [ContextState, ContextState];
+    }
+  }
+
+  get base(): ContextState {
+    return this.states[this.baseIndex];
+  }
+
+  get final(): ContextState {
+    return this.states[this.finalIndex]
+  }
+
+  private get finalIndex(): number {
+    return (this.baseIndex + 1) % 2;
+  }
+
+  get transitionId(): number {
+    return this._transitionId;
+  }
+
+  commitTransition(): ContextTransition {
+    // Preserve a deep-copy of the current object before proceeding.
+    const cloned = new ContextTransition(this);
+
+    // Commit 'final' and make it the new 'base'.
+    const finalIndex = this.baseIndex;
+    this.baseIndex = this.finalIndex;
+
+    // The old 'base' does not make a valid new 'final' - drop it.
+    this.states[finalIndex] = null;
+
+    // And drop the old transition data while we're at it.
+    this.inputDistribution = null;
+    this._transitionId = null;
+
+    return cloned;
+  }
+
+  replaceFinal(state: ContextState, inputDistribution: Distribution<Transform>, preservationTransform?: Transform) {
+    this.states[this.finalIndex] = state;
+    this.inputDistribution = inputDistribution;
+    // Long-term, this should never be null... but we need to allow it at this point
+    // in the refactoring process.
+    this._transitionId = inputDistribution?.find((entry) => entry.sample.id !== undefined)?.sample.id;
+    this.preservationTransform = preservationTransform;
+  }
+}
\ No newline at end of file
diff --git a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts
index cf30c16a46..bbee5a170c 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts
@@ -260,7 +260,7 @@ export class ModelCompositor {
     if(this.contextTracker) {
       let contextState = this.contextTracker.newest;
       if(!contextState) {
-        contextState = this.contextTracker.analyzeState(this.lexicalModel, context).state;
+        contextState = this.contextTracker.analyzeState(this.lexicalModel, context).final;
       }
 
       contextState.tokenization.tail.appliedSuggestionId = suggestion.id;
diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts
index 12c2547f4f..de9fc44840 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts
@@ -180,11 +180,11 @@ export async function correctAndEnumerate(
   // facilitates a more thorough correction-search pattern.
 
   // Token replacement benefits greatly from knowledge of the prior context state.
-  let { state: contextState } = contextTracker.analyzeState(
+  let contextState = contextTracker.analyzeState(
     lexicalModel,
     context,
     null
-  );
+  ).final;
 
   // Corrections and predictions are based upon the post-context state, though.
   const contextChangeAnalysis = contextTracker.analyzeState(
@@ -194,7 +194,7 @@ export async function correctAndEnumerate(
       ? transformDistribution
       : null
   );
-  const postContextState = contextChangeAnalysis.state;
+  const postContextState = contextChangeAnalysis.final;
 
   // TODO:  Should we filter backspaces & whitespaces out of the transform distribution?
   //        Ideally, the answer (in the future) will be no, but leaving it in right now may pose an issue.
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
index 4e2a8c142e..019a7c6e45 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
@@ -45,10 +45,10 @@ describe('ContextTracker', function() {
       let baseState = new ContextState(existingContext, plainModel);
       baseState.initFromReset();
       let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
-      assert.isNotNull(newContextMatch?.state);
-      assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
-      assert.equal(newContextMatch.headTokensRemoved, 1);
-      assert.equal(newContextMatch.tailTokensAdded, 0);
+      assert.isNotNull(newContextMatch?.final);
+      assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
+      assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, -1);
+      assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0);
     });
 
     it("properly matches and aligns when lead token + following whitespace are removed", function() {
@@ -67,10 +67,10 @@ describe('ContextTracker', function() {
       let baseState = new ContextState(existingContext, plainModel);
       baseState.initFromReset();
       let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
-      assert.isNotNull(newContextMatch?.state);
-      assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
-      assert.equal(newContextMatch.headTokensRemoved, 2);
-      assert.equal(newContextMatch.tailTokensAdded, 0);
+      assert.isNotNull(newContextMatch?.final);
+      assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
+      assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, -2);
+      assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0);
     });
 
     it("properly matches and aligns when final token is edited", function() {
@@ -89,10 +89,10 @@ describe('ContextTracker', function() {
       let baseState = new ContextState(existingContext, plainModel);
       baseState.initFromReset();
       let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
-      assert.isNotNull(newContextMatch?.state);
-      assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
-      assert.equal(newContextMatch.headTokensRemoved, 0);
-      assert.equal(newContextMatch.tailTokensAdded, 0);
+      assert.isNotNull(newContextMatch?.final);
+      assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
+      assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0);
+      assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0);
     });
 
     // Needs improved context-state management (due to 2x tokens)
@@ -112,17 +112,17 @@ describe('ContextTracker', function() {
       let baseState = new ContextState(existingContext, plainModel);
       baseState.initFromReset();
       let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
-      assert.isNotNull(newContextMatch?.state);
-      assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
+      assert.isNotNull(newContextMatch?.final);
+      assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       // We want to preserve the added whitespace when predicting a token that follows after it.
       assert.deepEqual(newContextMatch.preservationTransform, { insert: ' ', deleteLeft: 0 });
 
       // The 'wordbreak' transform
-      let state = newContextMatch?.state;
+      let state = newContextMatch?.final;
       assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence);
       assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence);
-      assert.equal(newContextMatch.headTokensRemoved, 0);
-      assert.equal(newContextMatch.tailTokensAdded, 2);
+      assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0);
+      assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 2);
     });
 
     it("properly matches and aligns when a 'wordbreak' is removed via backspace", function() {
@@ -141,12 +141,12 @@ describe('ContextTracker', function() {
       let baseState = new ContextState(existingContext, plainModel);
       baseState.initFromReset();
       let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
-      assert.isOk(newContextMatch?.state);
-      assert.deepEqual(newContextMatch?.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
+      assert.isOk(newContextMatch?.final);
+      assert.deepEqual(newContextMatch?.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
 
       // The 'wordbreak' transform
-      assert.equal(newContextMatch.headTokensRemoved, 0);
-      assert.equal(newContextMatch.tailTokensAdded, -2);
+      assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0);
+      assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, -2);
     });
 
     it("properly matches and aligns when an implied 'wordbreak' occurs (as when following \"'\")", function() {
@@ -165,17 +165,17 @@ describe('ContextTracker', function() {
       let baseState = new ContextState(existingContext, plainModel);
       baseState.initFromReset();
       let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
-      assert.isNotNull(newContextMatch?.state);
-      assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
+      assert.isNotNull(newContextMatch?.final);
+      assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       assert.deepEqual(newContextMatch.preservationTransform, { insert: '', deleteLeft: 0 });
 
       // The 'wordbreak' transform
-      let state = newContextMatch.state;
+      let state = newContextMatch.final;
       assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence);
       assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence);
 
-      assert.equal(newContextMatch.headTokensRemoved, 0);
-      assert.equal(newContextMatch.tailTokensAdded, 1);
+      assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0);
+      assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 1);
     })
 
     // Needs improved context-state management (due to 2x tokens)
@@ -195,18 +195,18 @@ describe('ContextTracker', function() {
       let baseState = new ContextState(existingContext, plainModel);
       baseState.initFromReset();
       let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
-      assert.isNotNull(newContextMatch?.state);
-      assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
+      assert.isNotNull(newContextMatch?.final);
+      assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       // We want to preserve the added whitespace when predicting a token that follows after it.
       assert.deepEqual(newContextMatch.preservationTransform, { insert: ' ', deleteLeft: 0 });
 
       // The 'wordbreak' transform
-      let state = newContextMatch.state;
+      let state = newContextMatch.final;
       assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence);
       assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence);
 
-      assert.equal(newContextMatch.headTokensRemoved, 2);
-      assert.equal(newContextMatch.tailTokensAdded, 2);
+      assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, -2);
+      assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 2);
     });
 
     it("properly matches and aligns when initial token is modified AND a 'wordbreak' is added'", function() {
@@ -230,18 +230,18 @@ describe('ContextTracker', function() {
         baseState,
         [{sample: transform, p: 1}]
       );
-      assert.isNotNull(newContextMatch?.state);
-      assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
+      assert.isNotNull(newContextMatch?.final);
+      assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       // We want to preserve all text preceding the new token when applying a suggestion.
       assert.deepEqual(newContextMatch.preservationTransform, { insert: 'd ', deleteLeft: 0});
 
       // The 'wordbreak' transform
-      let state = newContextMatch.state;
+      let state = newContextMatch.final;
       assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence);
       assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence);
 
-      assert.equal(newContextMatch.headTokensRemoved, 0);
-      assert.equal(newContextMatch.tailTokensAdded, 2);
+      assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0);
+      assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 2);
     });
 
     it("properly matches and aligns when tail token is modified AND a 'wordbreak' is added'", function() {
@@ -265,18 +265,18 @@ describe('ContextTracker', function() {
         baseState,
         [{sample: transform, p: 1}]
       );
-      assert.isNotNull(newContextMatch?.state);
-      assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
+      assert.isNotNull(newContextMatch?.final);
+      assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       // We want to preserve all text preceding the new token when applying a suggestion.
       assert.deepEqual(newContextMatch.preservationTransform, { insert: 'tor ', deleteLeft: 0 });
 
       // The 'wordbreak' transform
-      let state = newContextMatch.state;
+      let state = newContextMatch.final;
       assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence);
       assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence);
 
-      assert.equal(newContextMatch.headTokensRemoved, 0);
-      assert.equal(newContextMatch.tailTokensAdded, 2);
+      assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0);
+      assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 2);
     });
 
     it('rejects hard-to-handle case: tail token is split into three rather than two', function() {
@@ -368,7 +368,7 @@ describe('ContextTracker', function() {
       let compositor = new ModelCompositor(model);
       let baseContextMatch = compositor.contextTracker.analyzeState(model, baseContext);
 
-      baseContextMatch.state.tokenization.tail.replacements = [{
+      baseContextMatch.final.tokenization.tail.replacements = [{
         suggestion: baseSuggestion,
         tokenWidth: 1
       }];
@@ -376,7 +376,7 @@ describe('ContextTracker', function() {
       let reversion = compositor.acceptSuggestion(baseSuggestion, baseContext, postTransform);
 
       // Actual test assertion - was the replacement tracked?
-      assert.equal(baseContextMatch.state.tokenization.tail.appliedSuggestionId, baseSuggestion.id);
+      assert.equal(baseContextMatch.final.tokenization.tail.appliedSuggestionId, baseSuggestion.id);
       assert.equal(reversion.id, -baseSuggestion.id);
 
       // Next step - on the followup context, is the replacement still active?
@@ -384,10 +384,10 @@ describe('ContextTracker', function() {
       let postContextMatch = compositor.contextTracker.analyzeState(model, postContext);
 
       // Penultimate token corresponds to whitespace, which does not have a 'raw' representation.
-      assert.equal(postContextMatch.state.tokenization.tokens[postContextMatch.state.tokenization.tokens.length - 2].exampleInput, ' ');
+      assert.equal(postContextMatch.final.tokenization.tokens[postContextMatch.final.tokenization.tokens.length - 2].exampleInput, ' ');
 
       // Final token is empty (follows a wordbreak)
-      assert.equal(postContextMatch.state.tokenization.tail.exampleInput, '');
+      assert.equal(postContextMatch.final.tokenization.tail.exampleInput, '');
     });
   });
 });
\ No newline at end of file

From 7459e0edea8b23674920d5f719a4dfbce3e9c071 Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Fri, 1 Aug 2025 14:55:14 -0500
Subject: [PATCH 29/53] fix(web): fix import path for internal class used by
 unit test

---
 .../src/tests/mocha/cases/edit-distance/context-tokenization.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js
index 77b2e04994..a263c233c4 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js
@@ -1,5 +1,5 @@
 import { assert } from 'chai';
-import { isSubstitutionAlignable } from "#./correction/context-alignment.js";
+import { isSubstitutionAlignable } from "#./correction/alignment-helpers.js";
 
 import { ContextToken } from '#./correction/context-token.js';
 import { ContextTokenization } from '#./correction/context-tokenization.js';

From 1dbe62e42eafe0f50771e85e1773d5a3d4493eaf Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Mon, 4 Aug 2025 14:18:50 -0500
Subject: [PATCH 30/53] fix(web): stop unnecessarily cloning reused
 ContextState

---
 .../worker-thread/src/main/correction/context-tracker.ts        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
index 7a07618fed..4824b284dd 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
@@ -137,7 +137,7 @@ export class ContextTracker extends CircularArray<ContextState> {
     // If we have a perfect match with a pre-existing context, no mutations have
     // happened; just re-use the old context state.
     if(tailEditLength == 0 && leadTokenShift == 0 && tailTokenShift == 0) {
-      baseTransition.replaceFinal(new ContextState(matchState), transformDistribution);
+      baseTransition.replaceFinal(matchState, transformDistribution);
       return baseTransition;
     } else {
       // If we didn't get any input, we really should perfectly match

From 265f0265fd8a67e8122be5d3e5ef4389d2e0089b Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Mon, 4 Aug 2025 15:13:44 -0500
Subject: [PATCH 31/53] fix(web): prevent auto-select from keeping old
 autoaccept positions highlighted

---
 web/src/engine/osk/src/banner/suggestionBanner.ts | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/web/src/engine/osk/src/banner/suggestionBanner.ts b/web/src/engine/osk/src/banner/suggestionBanner.ts
index a4d852cb49..25143cb316 100644
--- a/web/src/engine/osk/src/banner/suggestionBanner.ts
+++ b/web/src/engine/osk/src/banner/suggestionBanner.ts
@@ -773,11 +773,10 @@ export class SuggestionBanner extends Banner {
       if(suggestions.length > i) {
         const suggestion = suggestions[i];
         d.update(suggestion, optionFormat);
-        if(this.predictionContext.selected == suggestion) {
-          d.highlight(true);
-        }
+        d.highlight(this.predictionContext.selected == suggestion)
       } else {
         d.update(null, optionFormat);
+        d.highlight(false);
       }
     }
 

From 9baeb9247fbbe7844da07d1f22412339cd1c7285 Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Tue, 5 Aug 2025 08:42:51 -0500
Subject: [PATCH 32/53] fix(web): prevent double-application of input
 transforms during tokenization

Fixes: #14456
---
 .../src/main/correction/context-tracker.ts    | 19 +++++----
 .../cases/edit-distance/context-tracker.js    | 39 ++++---------------
 2 files changed, 17 insertions(+), 41 deletions(-)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
index 035bcd99c0..2fc66d06e6 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
@@ -144,9 +144,12 @@ export class ContextTracker extends CircularArray<ContextState> {
     // the distribution should be tokenized already.
     transformDistribution?: Distribution<Transform> // transform distribution is needed here.
   ): ContextMatchResult {
-    const tokenizedContext = determineModelTokenizer(lexicalModel)(context).left;
     const transformSequenceDistribution = tokenizeAndFilterDistribution(context, lexicalModel, transformDistribution);
 
+    if(transformDistribution?.[0]) {
+      context = applyTransform(transformDistribution[0].sample, context);
+    }
+    const tokenizedContext = determineModelTokenizer(lexicalModel)(context).left;
     const alignmentResults = matchState.tokenization.computeAlignment(tokenizedContext.map((token) => token.text));
 
     if(!alignmentResults.canAlign) {
@@ -386,16 +389,12 @@ export class ContextTracker extends CircularArray<ContextState> {
       transformDistribution = null;
     }
     const inputTransform = transformDistribution?.[0];
-    if(inputTransform) {
-      // These two methods apply transforms internally; do not mutate context here.
-      // This particularly matters for the 'distribution' variant.
-      context = applyTransform(inputTransform.sample, context);
-    }
+    const postContext = inputTransform ? applyTransform(inputTransform.sample, context) : context;
 
     const tokenize = determineModelTokenizer(model);
-    const tokenizedContext = tokenize(context);
+    const tokenizedPostContext = tokenize(postContext)
 
-    if(tokenizedContext.left.length > 0) {
+    if(tokenizedPostContext.left.length > 0) {
       for(let i = this.count - 1; i >= 0; i--) {
         const priorMatchState = this.item(i);
 
@@ -412,10 +411,10 @@ export class ContextTracker extends CircularArray<ContextState> {
           //
           // `priorTaggedContext` must not be `null`!
           const doublecheckContext = applyTransform(transformDistribution[0].sample, priorTaggedContext);
-          if(doublecheckContext.left != context.left) {
+          if(doublecheckContext.left != postContext.left) {
             continue;
           }
-        } else if(priorTaggedContext?.left != context.left) {
+        } else if(priorTaggedContext?.left != postContext.left) {
           continue;
         }
 
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
index 4e2a8c142e..11695eaf0f 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
@@ -1,20 +1,15 @@
 import { assert } from 'chai';
 
 import { ContextTracker } from '#./correction/context-tracker.js';
-import { tokenizeTransformDistribution } from '#./correction/transform-tokenization.js';
 import ModelCompositor from '#./model-compositor.js';
 import * as models from '#./models/index.js';
 import * as wordBreakers from '@keymanapp/models-wordbreakers';
-import { determineModelTokenizer } from '#./model-helpers.js';
 
 import { default as defaultBreaker } from '@keymanapp/models-wordbreakers';
-import { deepCopy } from '@keymanapp/web-utils';
 
 import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs';
 import { ContextState } from '#./correction/context-state.js';
 
-const tokenizer = determineModelTokenizer(new models.DummyModel({wordbreaker: defaultBreaker}));
-
 var TrieModel = models.TrieModel;
 
 var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'),
@@ -81,14 +76,11 @@ describe('ContextTracker', function() {
         insert: 'r',
         deleteLeft: 0
       }
-      let newContext = {
-        left: "an apple a day keeps the doctor"
-      };
       let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
 
       let baseState = new ContextState(existingContext, plainModel);
       baseState.initFromReset();
-      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
+      let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
       assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       assert.equal(newContextMatch.headTokensRemoved, 0);
@@ -104,14 +96,11 @@ describe('ContextTracker', function() {
         insert: ' ',
         deleteLeft: 0
       }
-      let newContext = {
-        left: "an apple a day keeps the doctor "
-      };
       let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
 
       let baseState = new ContextState(existingContext, plainModel);
       baseState.initFromReset();
-      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
+      let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
       assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       // We want to preserve the added whitespace when predicting a token that follows after it.
@@ -133,14 +122,11 @@ describe('ContextTracker', function() {
         insert: '',
         deleteLeft: 1
       }
-      let newContext = {
-        left: "an apple a day keeps the doctor"
-      };
       let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
 
       let baseState = new ContextState(existingContext, plainModel);
       baseState.initFromReset();
-      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
+      let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform));
       assert.isOk(newContextMatch?.state);
       assert.deepEqual(newContextMatch?.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
 
@@ -157,14 +143,11 @@ describe('ContextTracker', function() {
         insert: 'a',
         deleteLeft: 0
       }
-      let newContext = {
-        left: "'a"
-      };
       let rawTokens = ["'", "a"];
 
       let baseState = new ContextState(existingContext, plainModel);
       baseState.initFromReset();
-      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
+      let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
       assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       assert.deepEqual(newContextMatch.preservationTransform, { insert: '', deleteLeft: 0 });
@@ -188,7 +171,7 @@ describe('ContextTracker', function() {
         deleteLeft: 0
       }
       let newContext = {
-        left: "apple a day keeps the doctor "
+        left: "apple a day keeps the doctor"
       };
       let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
 
@@ -217,15 +200,12 @@ describe('ContextTracker', function() {
         insert: 'd ',
         deleteLeft: 0
       }
-      let newContext = {
-        left: "and "
-      };
       let rawTokens = ["and", " ", ""];
 
       let baseState = new ContextState(existingContext, plainModel);
       baseState.initFromReset();
       let newContextMatch = ContextTracker.attemptMatchContext(
-        newContext,
+        existingContext,
         plainModel,
         baseState,
         [{sample: transform, p: 1}]
@@ -252,15 +232,12 @@ describe('ContextTracker', function() {
         insert: 'tor ',
         deleteLeft: 0
       }
-      let newContext = {
-        left: "apple a day keeps the doctor "
-      };
       let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
 
       let baseState = new ContextState(existingContext, plainModel);
       baseState.initFromReset();
       let newContextMatch = ContextTracker.attemptMatchContext(
-        newContext,
+        existingContext,
         plainModel,
         baseState,
         [{sample: transform, p: 1}]
@@ -301,7 +278,7 @@ describe('ContextTracker', function() {
         deleteLeft: 0
       }
       let problemContextMatch = ContextTracker.attemptMatchContext(
-        {left: "text'\""},
+        {left: "text'"},
         plainModel,
         baseState,
         [{sample: transform, p: 1}]

From a82993f125f3a4d9660e12f64e044f2230acc9af Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Tue, 5 Aug 2025 12:01:45 -0500
Subject: [PATCH 33/53] refactor(web): improve ContextState construction
 patterns

After proceeding further in development, the only non-initFromReset initialization pattern that appears is constructing a raw instance, then immediately providing a computed tokenization.  Thus, accepting the tokenization as an optional parameter will allow us a nice constructor spec for both init styles.
---
 .../src/main/correction/context-state.ts      | 25 ++++++++++++++++---
 .../src/main/correction/context-tracker.ts    |  1 -
 .../cases/edit-distance/context-state.js      | 19 ++------------
 .../cases/edit-distance/context-tracker.js    | 12 ---------
 4 files changed, 24 insertions(+), 33 deletions(-)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts
index 3c7643b23e..c6042b4ec5 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts
@@ -69,13 +69,32 @@ export class ContextState {
    */
   isManuallyApplied?: boolean;
 
+  /**
+   * Deep-copies a prior instance.
+   * @param stateToClone
+   */
   constructor(stateToClone: ContextState);
-  constructor(context: Context, model: LexicalModel);
-  constructor(param1: Context | ContextState, model?: LexicalModel) {
+  /**
+   * Initializes a new ContextState instance based on the active model and context.
+   *
+   * If a precomputed tokenization of the context (with prior correction-search
+   * calculation data) is not available, it will be spun up from scratch.
+   *
+   * @param context
+   * @param model
+   * @param tokenization
+   */
+  constructor(context: Context, model: LexicalModel, tokenization?: ContextTokenization);
+  constructor(param1: Context | ContextState, model?: LexicalModel, tokenization?: ContextTokenization) {
     if(!(param1 instanceof ContextState)) {
       const context = param1;
       this._context = context;
       this.model = model;
+      if(tokenization) {
+        this.tokenization = tokenization;
+      } else {
+        this.initFromReset();
+      }
     } else {
       const stateToClone = param1;
 
@@ -102,7 +121,7 @@ export class ContextState {
    * information is available - typically, immediately after engine
    * initialization or a context reset.
    */
-  initFromReset() {
+  private initFromReset() {
     const context = this.context;
     const lexicalModel = this.model;
 
diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
index 2fcfdac161..b20c829e28 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
@@ -463,7 +463,6 @@ export class ContextTracker extends CircularArray<ContextState> {
     // Assumption:  as a caret needs to move to context before any actual transform distributions occur,
     // this state is only reached on caret moves; thus, transformDistribution is actually just a single null transform.
     let state = new ContextState(context, model);
-    state.initFromReset();
     this.enqueue(state);
     return { state, baseState: null, headTokensRemoved: 0, tailTokensAdded: 0 };
   }
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-state.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-state.js
index 3ac00011ff..8ffdea5ef6 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-state.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-state.js
@@ -19,21 +19,16 @@ describe('ContextState', () => {
 
     assert.equal(state.context, context);
     assert.equal(state.model, plainModel);
-    assert.isNotOk(state.tokenization);
+    assert.isOk(state.tokenization);
     assert.isUndefined(state.isManuallyApplied);
     assert.isNotOk(state.suggestions);
     assert.isNotOk(state.appliedSuggestionId);
   });
 
-  describe('initFromReset', () => {
+  describe('initializing without prior tokenization', () => {
     it('<empty context>', () => {
       let context = { left: '', right: '' };
       let state = new ContextState(context, plainModel);
-
-      assert.isNotOk(state.tokenization);
-      assert.equal(state.context, context);
-
-      state.initFromReset();
       assert.isOk(state.tokenization);
       assert.equal(state.tokenization.tokens.length, 1);
       assert.equal(state.tokenization.tail.exampleInput, '');
@@ -42,11 +37,6 @@ describe('ContextState', () => {
     it('with initial text (without ending whitespace', () => {
       let context = { left: 'the quick brown fox', right: '' };
       let state = new ContextState(context, plainModel);
-
-      assert.isNotOk(state.tokenization);
-      assert.equal(state.context, context);
-
-      state.initFromReset();
       assert.isOk(state.tokenization);
       assert.equal(state.tokenization.tokens.length, 7);
       assert.deepEqual(state.tokenization.exampleInput, ['the', ' ', 'quick', ' ', 'brown', ' ', 'fox']);
@@ -55,11 +45,6 @@ describe('ContextState', () => {
     it('with initial text (with ending whitespace', () => {
       let context = { left: 'the quick brown fox ', right: '' };
       let state = new ContextState(context, plainModel);
-
-      assert.isNotOk(state.tokenization);
-      assert.equal(state.context, context);
-
-      state.initFromReset();
       assert.isOk(state.tokenization);
       assert.equal(state.tokenization.tokens.length, 9);
       assert.deepEqual(state.tokenization.exampleInput, ['the', ' ', 'quick', ' ', 'brown', ' ', 'fox', ' ', '']);
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
index 3bc068135f..9db2abed9e 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
@@ -44,7 +44,6 @@ describe('ContextTracker', function() {
       let rawTokens = [" ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
 
       let baseState = new ContextState(existingContext, plainModel);
-      baseState.initFromReset();
       let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
       assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
@@ -66,7 +65,6 @@ describe('ContextTracker', function() {
       let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
 
       let baseState = new ContextState(existingContext, plainModel);
-      baseState.initFromReset();
       let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
       assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
@@ -88,7 +86,6 @@ describe('ContextTracker', function() {
       let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
 
       let baseState = new ContextState(existingContext, plainModel);
-      baseState.initFromReset();
       let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
       assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
@@ -111,7 +108,6 @@ describe('ContextTracker', function() {
       let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
 
       let baseState = new ContextState(existingContext, plainModel);
-      baseState.initFromReset();
       let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
       assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
@@ -140,7 +136,6 @@ describe('ContextTracker', function() {
       let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
 
       let baseState = new ContextState(existingContext, plainModel);
-      baseState.initFromReset();
       let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
       assert.isOk(newContextMatch?.state);
       assert.deepEqual(newContextMatch?.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
@@ -164,7 +159,6 @@ describe('ContextTracker', function() {
       let rawTokens = ["'", "a"];
 
       let baseState = new ContextState(existingContext, plainModel);
-      baseState.initFromReset();
       let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
       assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
@@ -194,7 +188,6 @@ describe('ContextTracker', function() {
       let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
 
       let baseState = new ContextState(existingContext, plainModel);
-      baseState.initFromReset();
       let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.state);
       assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
@@ -224,7 +217,6 @@ describe('ContextTracker', function() {
       let rawTokens = ["and", " ", ""];
 
       let baseState = new ContextState(existingContext, plainModel);
-      baseState.initFromReset();
       let newContextMatch = ContextTracker.attemptMatchContext(
         newContext,
         plainModel,
@@ -259,7 +251,6 @@ describe('ContextTracker', function() {
       let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
 
       let baseState = new ContextState(existingContext, plainModel);
-      baseState.initFromReset();
       let newContextMatch = ContextTracker.attemptMatchContext(
         newContext,
         plainModel,
@@ -287,7 +278,6 @@ describe('ContextTracker', function() {
       assert.equal(baseContext.left.length, 1);
 
       let baseState = new ContextState({ left: "text'" }, plainModel);
-      baseState.initFromReset();
 
       // Now the actual check.
       let newContext = models.tokenize(defaultBreaker, {
@@ -317,7 +307,6 @@ describe('ContextTracker', function() {
       let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
 
       let state = new ContextState(context, plainModel);
-      state.initFromReset();
       assert.deepEqual(state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
     });
 
@@ -326,7 +315,6 @@ describe('ContextTracker', function() {
       let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
 
       let state = new ContextState(context, plainModel);
-      state.initFromReset();
       assert.deepEqual(state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
     });
   });

From ab52938d96cfb5adff2be911f26d03979049c938 Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Fri, 8 Aug 2025 09:22:01 -0500
Subject: [PATCH 34/53] docs(web): adds doc-comments for
 textToCharTransforms(), ContextToken class

---
 .../src/main/correction/context-token.ts          | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts
index 6bff048aee..198503d363 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts
@@ -8,6 +8,17 @@ import LexicalModel = LexicalModelTypes.LexicalModel;
 import Suggestion = LexicalModelTypes.Suggestion;
 import Transform = LexicalModelTypes.Transform;
 
+/**
+ * Breaks apart a raw text string into individual, single-codepoint
+ * transforms, all set with the specified transform ID.
+ *
+ * This is designed for use when initializing a new ContextToken without
+ * any prior cached data or for rewriting its probabilities after
+ * receiving backspace input.
+ * @param text
+ * @param transformId
+ * @returns
+ */
 function textToCharTransforms(text: string, transformId?: number) {
   let perCharTransforms: Transform[] = [];
 
@@ -29,6 +40,10 @@ function textToCharTransforms(text: string, transformId?: number) {
   return perCharTransforms;
 }
 
+/**
+ * Represents cached data about one token (either a word or a unit of whitespace)
+ * in the context and associated correction-search progress and results.
+ */
 export class ContextToken {
   /**
    * Indicates whether or not the token is considered whitespace.

From 9b299550600071737a6dcfafcfd12cd6fb08bda7 Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Fri, 8 Aug 2025 09:25:55 -0500
Subject: [PATCH 35/53] docs(web): documents the ContextTokenization class

---
 .../src/main/correction/context-tokenization.ts       | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
index 9762c9d87d..b6a99b5af5 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
@@ -1,6 +1,10 @@
 import { ContextToken } from './context-token.js';
 import { TrackedContextStateAlignment } from './context-tracker.js';
 
+/**
+ * This class represents the sequence of tokens (words and whitespace blocks)
+ * held within the active sliding context-window at a single point in time.
+ */
 export class ContextTokenization {
   readonly tokens: ContextToken[];
   readonly alignment?: TrackedContextStateAlignment;
@@ -19,10 +23,17 @@ export class ContextTokenization {
     }
   }
 
+  /**
+   * Returns the token adjacent to the text insertion point.
+   */
   get tail(): ContextToken {
     return this.tokens[this.tokens.length - 1];
   }
 
+  /**
+   * Returns a plain-text string representing the most probable representation for all
+   * tokens represented by this tokenization instance.
+   */
   get exampleInput(): string[] {
     const sequence: string[] = [];
 

From c7e84cdff794a974e30bf766990411375cb3ca5c Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Fri, 8 Aug 2025 09:40:31 -0500
Subject: [PATCH 36/53] change(web): moves isSubstitutionAlignable unit tests

---
 .../cases/edit-distance/alignment-helpers.js  | 76 +++++++++++++++++++
 .../edit-distance/context-tokenization.js     | 73 ------------------
 2 files changed, 76 insertions(+), 73 deletions(-)
 create mode 100644 web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/alignment-helpers.js

diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/alignment-helpers.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/alignment-helpers.js
new file mode 100644
index 0000000000..d36c0a8a79
--- /dev/null
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/alignment-helpers.js
@@ -0,0 +1,76 @@
+import { assert } from 'chai';
+import { getEditPathLastMatch, isSubstitutionAlignable } from '#./correction/alignment-helpers.js';
+
+
+describe('isSubstitutionAlignable', () => {
+  it(`returns true:  'ca' => 'can'`, () => {
+    assert.isTrue(isSubstitutionAlignable('can', 'ca'));
+  });
+
+  // Leading word in context window starts sliding out of said window.
+  it(`returns true:  'can' => 'an'`, () => {
+    assert.isTrue(isSubstitutionAlignable('an', 'can'));
+  });
+
+  // Same edits on both sides:  not valid.
+  it(`returns false: 'apple' => 'grapples'`, () => {
+    assert.isFalse(isSubstitutionAlignable('grapples', 'apple'));
+  });
+
+  // Edits on one side:  valid.
+  it(`returns true: 'apple' => 'grapple'`, () => {
+    assert.isTrue(isSubstitutionAlignable('grapple', 'apple'));
+  });
+
+  // Edits on one side:  valid.
+  it(`returns true: 'apple' => 'grapple'`, () => {
+    assert.isTrue(isSubstitutionAlignable('apples', 'apple'));
+  });
+
+  // Same edits on both sides:  not valid.
+  it(`returns false: 'grapples' => 'apple'`, () => {
+    assert.isFalse(isSubstitutionAlignable('apple', 'grapples'));
+  });
+
+  // Substitution:  not valid when not permitted via parameter.
+  it(`returns false:  'apple' => 'banana'`, () => {
+    // edit path:  'insert' ('b' of banana), 'match' (on leading a), rest are 'substitute'.
+    assert.isFalse(isSubstitutionAlignable('banana', 'apple'));
+  });
+
+  // Substitution:  not valid if too much is substituted, even if allowed via parameter.
+  it(`returns false:  'apple' => 'banana' (subs allowed)`, () => {
+    // edit path:  'insert' ('b' of banana), 'match' (on leading a), rest are 'substitute'.
+    // 1 match vs 4 substitute = no bueno.  It'd require too niche of a keyboard rule.
+    assert.isFalse(isSubstitutionAlignable('banana', 'apple', true));
+  });
+
+  it(`returns true: 'a' => 'à' (subs allowed)`, () => {
+    assert.isTrue(isSubstitutionAlignable('à', 'a', true));
+  });
+
+  // Leading substitution:  valid if enough of the remaining word matches.
+  // Could totally happen from a legit Keyman keyboard rule.
+  it(`returns true: 'can' => 'van' (subs allowed)`, () => {
+    assert.isTrue(isSubstitutionAlignable('van', 'can', true));
+  });
+
+  // Trailing substitution:  invalid if not allowed.
+  it(`returns false: 'can' => 'cap' (subs not allowed)`, () => {
+    assert.isFalse(isSubstitutionAlignable('cap', 'can'));
+  });
+
+  // Trailing substitution:  valid.
+  it(`returns false: 'can' => 'cap' (subs allowed)`, () => {
+    assert.isTrue(isSubstitutionAlignable('cap', 'can', true));
+  });
+
+  it(`returns true:  'clasts' => 'clasps' (subs allowed)`, () => {
+    assert.isTrue(isSubstitutionAlignable('clasps', 'clasts', true));
+  });
+
+  // random deletion at the start + later substitution = still permitted
+  it(`returns false:  'clasts' => 'lasps' (subs allowed)`, () => {
+    assert.isTrue(isSubstitutionAlignable('lasps', 'clasts', true));
+  });
+});
\ No newline at end of file
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js
index a263c233c4..e1ee95654f 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js
@@ -20,79 +20,6 @@ function toToken(text) {
   return token;
 }
 
-describe('isSubstitutionAlignable', () => {
-  it(`returns true:  'ca' => 'can'`, () => {
-    assert.isTrue(isSubstitutionAlignable('can', 'ca'));
-  });
-
-  // Leading word in context window starts sliding out of said window.
-  it(`returns true:  'can' => 'an'`, () => {
-    assert.isTrue(isSubstitutionAlignable('an', 'can'));
-  });
-
-  // Same edits on both sides:  not valid.
-  it(`returns false: 'apple' => 'grapples'`, () => {
-    assert.isFalse(isSubstitutionAlignable('grapples', 'apple'));
-  });
-
-  // Edits on one side:  valid.
-  it(`returns true: 'apple' => 'grapple'`, () => {
-    assert.isTrue(isSubstitutionAlignable('grapple', 'apple'));
-  });
-
-  // Edits on one side:  valid.
-  it(`returns true: 'apple' => 'grapple'`, () => {
-    assert.isTrue(isSubstitutionAlignable('apples', 'apple'));
-  });
-
-  // Same edits on both sides:  not valid.
-  it(`returns false: 'grapples' => 'apple'`, () => {
-    assert.isFalse(isSubstitutionAlignable('apple', 'grapples'));
-  });
-
-  // Substitution:  not valid when not permitted via parameter.
-  it(`returns false:  'apple' => 'banana'`, () => {
-    // edit path:  'insert' ('b' of banana), 'match' (on leading a), rest are 'substitute'.
-    assert.isFalse(isSubstitutionAlignable('banana', 'apple'));
-  });
-
-  // Substitution:  not valid if too much is substituted, even if allowed via parameter.
-  it(`returns false:  'apple' => 'banana' (subs allowed)`, () => {
-    // edit path:  'insert' ('b' of banana), 'match' (on leading a), rest are 'substitute'.
-    // 1 match vs 4 substitute = no bueno.  It'd require too niche of a keyboard rule.
-    assert.isFalse(isSubstitutionAlignable('banana', 'apple', true));
-  });
-
-  it(`returns true: 'a' => 'à' (subs allowed)`, () => {
-    assert.isTrue(isSubstitutionAlignable('à', 'a', true));
-  });
-
-  // Leading substitution:  valid if enough of the remaining word matches.
-  // Could totally happen from a legit Keyman keyboard rule.
-  it(`returns true: 'can' => 'van' (subs allowed)`, () => {
-    assert.isTrue(isSubstitutionAlignable('van', 'can', true));
-  });
-
-  // Trailing substitution:  invalid if not allowed.
-  it(`returns false: 'can' => 'cap' (subs not allowed)`, () => {
-    assert.isFalse(isSubstitutionAlignable('cap', 'can'));
-  });
-
-  // Trailing substitution:  valid.
-  it(`returns false: 'can' => 'cap' (subs allowed)`, () => {
-    assert.isTrue(isSubstitutionAlignable('cap', 'can', true));
-  });
-
-  it(`returns true:  'clasts' => 'clasps' (subs allowed)`, () => {
-    assert.isTrue(isSubstitutionAlignable('clasps', 'clasts', true));
-  });
-
-  // random deletion at the start + later substitution = still permitted
-  it(`returns false:  'clasts' => 'lasps' (subs allowed)`, () => {
-    assert.isTrue(isSubstitutionAlignable('lasps', 'clasts', true));
-  });
-});
-
 describe('ContextTokenization', function() {
   describe("<constructor>", () => {
     it("constructs from just a token array", () => {

From 2e30e2d1ad0cd67349fc16e943b057bdce7fb6a0 Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Fri, 8 Aug 2025 09:40:50 -0500
Subject: [PATCH 37/53] feat(web): adds getEditPathLastMatch unit tests

---
 .../cases/edit-distance/alignment-helpers.js  | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/alignment-helpers.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/alignment-helpers.js
index d36c0a8a79..8c3f313556 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/alignment-helpers.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/alignment-helpers.js
@@ -1,6 +1,25 @@
 import { assert } from 'chai';
 import { getEditPathLastMatch, isSubstitutionAlignable } from '#./correction/alignment-helpers.js';
 
+describe('getEditPathLastMatch', () => {
+  it('returns the last match when no substitutions exist', () => {
+    const path = ['delete', 'delete', 'match', 'match', 'match', 'match', 'insert'];
+    assert.equal(getEditPathLastMatch(path), path.lastIndexOf('match'));
+  });
+
+  it('returns the last match when no substitutions exist left of a "match"', () => {
+    const path = ['delete', 'delete', 'match', 'match', 'match', 'match', 'substitute', 'insert'];
+    assert.equal(getEditPathLastMatch(path), path.lastIndexOf('match'));
+  });
+
+  // is intended to handle application of suggestions.
+  it('returns the second-to-last match when a substitution exists before final "match"', () => {
+    // limitation:  if there is _anything_ after that last match, the first assertion will fail.
+    const path = ['delete', 'delete', 'match', 'match', 'match', 'substitute', 'match'];
+    assert.notEqual(getEditPathLastMatch(path), path.lastIndexOf('match'));
+    assert.equal(getEditPathLastMatch(path), path.lastIndexOf('match', path.lastIndexOf('match')-1));
+  });
+});
 
 describe('isSubstitutionAlignable', () => {
   it(`returns true:  'ca' => 'can'`, () => {

From ea8108aa1830aa114e732e92abc1fdd32ca29bd1 Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Fri, 8 Aug 2025 09:41:51 -0500
Subject: [PATCH 38/53] change(web): undoes minor tweaks to
 essentially-unchanged test set

---
 .../tests/mocha/cases/edit-distance/context-tokenization.js    | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js
index e1ee95654f..8ba132ed56 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js
@@ -1,5 +1,4 @@
 import { assert } from 'chai';
-import { isSubstitutionAlignable } from "#./correction/alignment-helpers.js";
 
 import { ContextToken } from '#./correction/context-token.js';
 import { ContextTokenization } from '#./correction/context-tokenization.js';
@@ -82,4 +81,4 @@ describe('ContextTokenization', function() {
 
     assert.deepEqual(tokenization.exampleInput, rawTextTokens);
   });
-});
+});
\ No newline at end of file

From 19ec81778378ccfbdae0cfae420f4d1c85a470b7 Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Fri, 8 Aug 2025 10:09:54 -0500
Subject: [PATCH 39/53] change(web): fixes unit test deep-equals issues

---
 .../src/main/correction/context-token.ts             |  5 ++++-
 .../cases/edit-distance/context-tokenization.js      | 12 +++++++-----
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts
index 198503d363..22fd3efed7 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts
@@ -100,7 +100,10 @@ export class ContextToken {
       this.searchSpace = new SearchSpace(priorToken.searchSpace);
       this.suggestions = priorToken.suggestions.slice();
 
-      this.appliedSuggestionId = priorToken.appliedSuggestionId;
+      // because of unit tests.
+      if(priorToken.appliedSuggestionId !== undefined) {
+        this.appliedSuggestionId = priorToken.appliedSuggestionId;
+      }
     } else {
       const model = param;
 
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js
index 6a93378d4d..9924a506b9 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js
@@ -7,8 +7,6 @@ import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'
 import { TrieModel } from '#./models/index.js';
 import { default as defaultBreaker } from '@keymanapp/models-wordbreakers';
 
-// TODO:  consider mocking out the need for SearchSpace stuff?
-
 var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'),
   {wordBreaker: defaultBreaker});
 
@@ -29,7 +27,6 @@ describe('ContextTokenization', function() {
     it("constructs from just a token array", () => {
       const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
       let tokenization = new ContextTokenization(rawTextTokens.map((text => toToken(text))));
-
       assert.deepEqual(tokenization.tokens.map((entry) => entry.exampleInput), rawTextTokens);
       assert.deepEqual(tokenization.tokens.map((entry) => entry.isWhitespace), rawTextTokens.map((entry) => entry == ' '));
       assert.isNotOk(tokenization.alignment);
@@ -73,10 +70,15 @@ describe('ContextTokenization', function() {
       let cloned = new ContextTokenization(baseTokenization);
 
       assert.notDeepEqual(cloned, baseTokenization);
-      assert.notDeepEqual(cloned.tokens, baseTokenization.tokens);
       assert.deepEqual(cloned.tokens.map((token) => token.searchSpace.inputSequence),
         baseTokenization.tokens.map((token) => token.searchSpace.inputSequence));
-      assert.deepEqual(cloned.alignment, baseTokenization.alignment);
+
+      // The `.searchSpace` instances will not be deep-equal; there are class properties
+      // that hold functions with closures, configured at runtime.
+      baseTokenization.tokens.forEach((token) => delete token.searchSpace);
+      cloned.tokens.forEach((token) => delete token.searchSpace);
+
+      assert.deepEqual(cloned, baseTokenization);
     });
   });
 

From 4cd4b7728231ab5ea75549f68232b62b5ff3ccd6 Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Fri, 8 Aug 2025 10:15:43 -0500
Subject: [PATCH 40/53] change(web): adds docs, prevents bad .suggestions logic
 in clone-constructor path

---
 .../src/main/correction/context-state.ts            | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts
index 244f4f362b..2663f711b4 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts
@@ -67,7 +67,16 @@ export class ContextState {
    */
   isManuallyApplied?: boolean;
 
+  /**
+   * Deep-copies a previously-constructed instance.
+   * @param stateToClone
+   */
   constructor(stateToClone: ContextState);
+  /**
+   * Constructs a new instance based on the current context.
+   * @param context The context available within the current sliding context-window
+   * @param model The active lexical model.
+   */
   constructor(context: Context, model: LexicalModel);
   constructor(param1: Context | ContextState, model?: LexicalModel) {
     if(!(param1 instanceof ContextState)) {
@@ -83,7 +92,9 @@ export class ContextState {
 
       // A shallow copy of the array is fine, but we'd be best off
       // not aliasing the array itself.
-      this.suggestions = [].concat(stateToClone.suggestions);
+      if(stateToClone.suggestions?.length ?? 0 > 0) {
+        this.suggestions = [].concat(stateToClone.suggestions);
+      }
     }
   }
 }
\ No newline at end of file

From a8ca07243a3584844a38d89e046e12573d922559 Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Fri, 8 Aug 2025 10:26:53 -0500
Subject: [PATCH 41/53] change(web): implements changes from PR review

changed unit test names added by prior PR

polished up the relocated code now found at initFromReset
---
 .../src/main/correction/context-state.ts      | 24 +++++--------------
 .../cases/edit-distance/context-state.js      |  6 ++---
 2 files changed, 9 insertions(+), 21 deletions(-)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts
index b8cb498525..9713bfaee1 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts
@@ -125,13 +125,9 @@ export class ContextState {
    * initialization or a context reset.
    */
   private initFromReset() {
-    const context = this.context;
-    const lexicalModel = this.model;
-
-    const tokenizedContext = determineModelTokenizer(lexicalModel)(context).left;
-
-    let baseTokens = tokenizedContext.map(function(entry) {
-      let token = new ContextToken(lexicalModel, entry.text);
+    const tokenizedContext = determineModelTokenizer(this.model)(this.context).left;
+    const baseTokens = tokenizedContext.map((entry) => {
+      const token = new ContextToken(this.model, entry.text);
 
       if(entry.isWhitespace) {
         token.isWhitespace = true;
@@ -141,17 +137,9 @@ export class ContextState {
     });
 
     // And now build the final context state object, which includes whitespace 'tokens'.);
-    const tokenization: ContextToken[] = [];
-
-    while(baseTokens.length > 0) {
-      tokenization.push(baseTokens.splice(0, 1)[0]);
+    if(baseTokens.length == 0) {
+      baseTokens.push(new ContextToken(this.model));
     }
-
-    if(tokenization.length == 0) {
-      let token = new ContextToken(lexicalModel);
-      tokenization.push(token);
-    }
-
-    this.tokenization = new ContextTokenization(tokenization);
+    this.tokenization = new ContextTokenization(baseTokens);
   }
 }
\ No newline at end of file
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-state.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-state.js
index 8ffdea5ef6..e6387cba57 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-state.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-state.js
@@ -26,7 +26,7 @@ describe('ContextState', () => {
   });
 
   describe('initializing without prior tokenization', () => {
-    it('<empty context>', () => {
+    it('creates one empty token for an empty context', () => {
       let context = { left: '', right: '' };
       let state = new ContextState(context, plainModel);
       assert.isOk(state.tokenization);
@@ -34,7 +34,7 @@ describe('ContextState', () => {
       assert.equal(state.tokenization.tail.exampleInput, '');
     });
 
-    it('with initial text (without ending whitespace', () => {
+    it('creates tokens for initial text (without ending whitespace)', () => {
       let context = { left: 'the quick brown fox', right: '' };
       let state = new ContextState(context, plainModel);
       assert.isOk(state.tokenization);
@@ -42,7 +42,7 @@ describe('ContextState', () => {
       assert.deepEqual(state.tokenization.exampleInput, ['the', ' ', 'quick', ' ', 'brown', ' ', 'fox']);
     });
 
-    it('with initial text (with ending whitespace', () => {
+    it('creates tokens for initial text (with extra empty token for ending whitespace)', () => {
       let context = { left: 'the quick brown fox ', right: '' };
       let state = new ContextState(context, plainModel);
       assert.isOk(state.tokenization);

From 3f9b4e498da5ad46cdbd8665be76bfb2c435e5aa Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Fri, 8 Aug 2025 10:29:24 -0500
Subject: [PATCH 42/53] change(web): drops unnecessary context getter

---
 .../src/main/correction/context-state.ts            | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts
index 9713bfaee1..80e82712ae 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts
@@ -18,7 +18,7 @@ export class ContextState {
    * The context window in view for the represented Context state,
    * as passed between the predictive-text worker and its host.
    */
-  private _context: Context;
+  readonly context: Context;
 
   /**
    * The active lexical model operating upon the Context.
@@ -88,8 +88,7 @@ export class ContextState {
   constructor(context: Context, model: LexicalModel, tokenization?: ContextTokenization);
   constructor(param1: Context | ContextState, model?: LexicalModel, tokenization?: ContextTokenization) {
     if(!(param1 instanceof ContextState)) {
-      const context = param1;
-      this._context = context;
+      this.context = param1;
       this.model = model;
       if(tokenization) {
         this.tokenization = tokenization;
@@ -111,14 +110,6 @@ export class ContextState {
     }
   }
 
-  /**
-   * The context window in view for the represented Context state,
-   * as passed between the predictive-text worker and its host.
-   */
-  get context(): Context {
-    return this._context;
-  }
-
   /**
    * Initializes the ContextState instance for use when no valid prior
    * information is available - typically, immediately after engine

From bd89b58f93c610da16c0cacd904ba421b0126aca Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Fri, 8 Aug 2025 12:23:36 -0500
Subject: [PATCH 43/53] change(web): fix typo

---
 .../worker-thread/src/main/correction/context-tracker.ts        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
index f012c5ac7b..7efdee1067 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
@@ -168,7 +168,7 @@ export class ContextTracker extends CircularArray<ContextState> {
     // If we have a perfect match with a pre-existing context, no mutations have
     // happened; just re-use the old context state.
     if(tailEditLength == 0 && leadTokenShift == 0 && tailTokenShift == 0) {
-      return { state: matchState, baseState: matchState, headTokensRemoved: 0, tailTokensAdded: 0 };;
+      return { state: matchState, baseState: matchState, headTokensRemoved: 0, tailTokensAdded: 0 };
     } else {
       // If we didn't get any input, we really should perfectly match
       // a previous context state.  If such a state is out of our cache,

From 1bc5d19e990b2e997d0dd8a32c08121a36ab258e Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Fri, 8 Aug 2025 12:57:33 -0500
Subject: [PATCH 44/53] refactor(web): removes double-buffered state style,
 renames ContextTransition.replaceFinal -> finalize

---
 .../src/main/correction/context-tracker.ts    |  8 +-
 .../src/main/correction/context-transition.ts | 83 +++++++++++--------
 2 files changed, 53 insertions(+), 38 deletions(-)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
index d98d288a68..f6c629e1d4 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
@@ -139,7 +139,7 @@ export class ContextTracker extends CircularArray<ContextState> {
     // If we have a perfect match with a pre-existing context, no mutations have
     // happened; just re-use the old context state.
     if(tailEditLength == 0 && leadTokenShift == 0 && tailTokenShift == 0) {
-      baseTransition.replaceFinal(matchState, transformDistribution);
+      baseTransition.finalize(matchState, transformDistribution);
       return baseTransition;
     } else {
       // If we didn't get any input, we really should perfectly match
@@ -167,7 +167,7 @@ export class ContextTracker extends CircularArray<ContextState> {
     if(tailEditLength == 0 && tailTokenShift == 0) {
       const state = new ContextState(context, lexicalModel);
       state.tokenization = new ContextTokenization(tokenization, alignmentResults);
-      baseTransition.replaceFinal(state, transformDistribution);
+      baseTransition.finalize(state, transformDistribution);
       return baseTransition;
     }
 
@@ -320,7 +320,7 @@ export class ContextTracker extends CircularArray<ContextState> {
 
     const state = new ContextState(context, lexicalModel);
     state.tokenization = new ContextTokenization(tokenization, alignmentResults);
-    baseTransition.replaceFinal(state, transformDistribution, preservationTransform);
+    baseTransition.finalize(state, transformDistribution, preservationTransform);
     return baseTransition;
   }
 
@@ -410,7 +410,7 @@ export class ContextTracker extends CircularArray<ContextState> {
     const transition = new ContextTransition(state, /* TODO:  we need a clear value here in the future! */ null);
     // Hacky, but holds the course for now.  This should only really happen from context resets, which can
     // then use a different path.
-    transition.replaceFinal(state, []);
+    transition.finalize(state, []);
     return transition;
   }
 
diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts
index abc0653db7..aa1235ff1f 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts
@@ -4,11 +4,23 @@ import { LexicalModelTypes } from '@keymanapp/common-types';
 import Distribution = LexicalModelTypes.Distribution;
 import Transform = LexicalModelTypes.Transform;
 
+/**
+ * Represents the transition between two context states as triggered
+ * by input keystrokes or applied suggestions.
+ */
 export class ContextTransition {
-  private states: [ContextState, ContextState];
-  private baseIndex = 0;
+  /**
+   * Represents the state of the context before the transition event occurred.
+   */
+  readonly base: ContextState;
+  private _final: ContextState;
 
+  /**
+   * Indicates the fat-finger distribution for the incoming keystroke related to
+   * the context transition event.
+   */
   inputDistribution?: Distribution<Transform>;
+
   // The transform ID in play.
   private _transitionId?: number;
 
@@ -27,60 +39,63 @@ export class ContextTransition {
    */
   preservationTransform?: Transform;
 
+  /**
+   * Constructs a partial context transition object for use during the process
+   * of analyzing context transitions or for representing the base state of a
+   * reset context.
+   * @param context  The base state for the represented context transition
+   * @param transitionId  The unique ID corresponding to the transition event
+   * or context state.
+   */
   constructor(context: ContextState, transitionId: number);
+  /**
+   * Deep-copies a ContextTransition instance.
+   * @param baseTransition
+   */
   constructor(baseTransition: ContextTransition);
   constructor(param: ContextState | ContextTransition, transitionId?: number) {
     if(!(param instanceof ContextTransition)) {
       const contextState = param;
       // We're initializing a ContextTransition from a blank or reset context.
-      const baseState = contextState;
-      this.states = [baseState, null];
+      this.base = contextState;
+      this._final = null;
       this._transitionId = transitionId;
     } else {
       const baseTransition = param;
       Object.assign(this, baseTransition);
 
       // These need to be deep-copied.
-      this.states = baseTransition.states.map((entry) => new ContextState(entry)) as [ContextState, ContextState];
+      this.base = new ContextState(baseTransition.base);
+      this._final = new ContextState(baseTransition._final);
     }
   }
 
-  get base(): ContextState {
-    return this.states[this.baseIndex];
-  }
-
+  /**
+   * Gets the context state resulting from the context transition event,
+   * including any generated suggestions and data regarding potential
+   * application thereof.
+   */
   get final(): ContextState {
-    return this.states[this.finalIndex]
-  }
-
-  private get finalIndex(): number {
-    return (this.baseIndex + 1) % 2;
+    return this._final;
   }
 
+  /**
+   * The unique ID corresponding to the transition event or context state.
+   */
   get transitionId(): number {
     return this._transitionId;
   }
 
-  commitTransition(): ContextTransition {
-    // Preserve a deep-copy of the current object before proceeding.
-    const cloned = new ContextTransition(this);
-
-    // Commit 'final' and make it the new 'base'.
-    const finalIndex = this.baseIndex;
-    this.baseIndex = this.finalIndex;
-
-    // The old 'base' does not make a valid new 'final' - drop it.
-    this.states[finalIndex] = null;
-
-    // And drop the old transition data while we're at it.
-    this.inputDistribution = null;
-    this._transitionId = null;
-
-    return cloned;
-  }
-
-  replaceFinal(state: ContextState, inputDistribution: Distribution<Transform>, preservationTransform?: Transform) {
-    this.states[this.finalIndex] = state;
+  /**
+   * Records the context state resulting from the context transition generated
+   * by a keystroke.
+   * @param state  The context state to record as the result of the transition
+   * @param inputDistribution Fat-finger data corresponding to the triggering keystroke
+   * @param preservationTransform Portions of the most likely input that do not contribute to the final token
+   * in the final context's tokenization.
+   */
+  finalize(state: ContextState, inputDistribution: Distribution<Transform>, preservationTransform?: Transform) {
+    this._final = state;
     this.inputDistribution = inputDistribution;
     // Long-term, this should never be null... but we need to allow it at this point
     // in the refactoring process.

From b544aeff371c8dfe812bd40fa0a63fa3d80f27cf Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Tue, 12 Aug 2025 20:32:28 +0700
Subject: [PATCH 45/53] change(web): Apply suggestions from code review

Co-authored-by: Marc Durdin <marc@durdin.net>
---
 .../src/main/correction/context-token.ts      | 33 ++++++++-----------
 1 file changed, 13 insertions(+), 20 deletions(-)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts
index 198503d363..de715522e6 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts
@@ -1,3 +1,11 @@
+/*
+ * Keyman is copyright (C) SIL Global. MIT License.
+ * 
+ * Created by jahorton on 2025-07-30
+ * 
+ * Represents cached data about one token (either a word or a unit of whitespace)
+ * in the context and associated correction-search progress and results.
+ */
 import { buildMergedTransform } from "@keymanapp/models-templates";
 import { SearchSpace } from "./distance-modeler.js";
 import { KMWString } from "@keymanapp/web-utils";
@@ -19,26 +27,11 @@ import Transform = LexicalModelTypes.Transform;
  * @param transformId
  * @returns
  */
-function textToCharTransforms(text: string, transformId?: number) {
-  let perCharTransforms: Transform[] = [];
-
-  for(let i=0; i < KMWString.length(text); i++) {
-    let char = KMWString.charAt(text, i); // is SMP-aware
-
-    let transform: Transform = {
-      insert: char,
-      deleteLeft: 0
-    };
-
-    if(transformId) {
-      transform.id = transformId
-    }
-
-    perCharTransforms.push(transform);
-  }
-
-  return perCharTransforms;
-}
+function textToCharTransforms(text: string, transformId?: number): Transform[] {
+  return transformId ?
+    [...text].map(insert => ({insert, deleteLeft: 0, id: transformId})) :
+    [...text].map(insert => ({insert, deleteLeft: 0}));
+}  
 
 /**
  * Represents cached data about one token (either a word or a unit of whitespace)

From 0598e080bd7a9882c302dd62a2c3f8cd79deaf4e Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Tue, 12 Aug 2025 08:41:15 -0500
Subject: [PATCH 46/53] change(web): import ordering, SearchSpace.inputSequence
 getter

---
 .../src/main/correction/context-token.ts      | 13 ++++++------
 .../src/main/correction/distance-modeler.ts   | 19 +++++++++++------
 ...ontext-token.js => context-token.tests.js} | 21 ++++++++++++-------
 3 files changed, 34 insertions(+), 19 deletions(-)
 rename web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/{context-token.js => context-token.tests.js} (92%)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts
index de715522e6..c637dea3e6 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts
@@ -1,16 +1,17 @@
 /*
  * Keyman is copyright (C) SIL Global. MIT License.
- * 
+ *
  * Created by jahorton on 2025-07-30
- * 
+ *
  * Represents cached data about one token (either a word or a unit of whitespace)
  * in the context and associated correction-search progress and results.
  */
-import { buildMergedTransform } from "@keymanapp/models-templates";
-import { SearchSpace } from "./distance-modeler.js";
-import { KMWString } from "@keymanapp/web-utils";
 
+import { buildMergedTransform } from "@keymanapp/models-templates";
 import { LexicalModelTypes } from '@keymanapp/common-types';
+
+import { SearchSpace } from "./distance-modeler.js";
+
 import Distribution = LexicalModelTypes.Distribution;
 import LexicalModel = LexicalModelTypes.LexicalModel;
 import Suggestion = LexicalModelTypes.Suggestion;
@@ -31,7 +32,7 @@ function textToCharTransforms(text: string, transformId?: number): Transform[] {
   return transformId ?
     [...text].map(insert => ({insert, deleteLeft: 0, id: transformId})) :
     [...text].map(insert => ({insert, deleteLeft: 0}));
-}  
+}
 
 /**
  * Represents cached data about one token (either a word or a unit of whitespace)
diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts
index f61b804b4a..956a369bc3 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts
@@ -366,7 +366,7 @@ export class SearchSpace {
 
   private tierOrdering: SearchSpaceTier[] = [];
   private selectionQueue: PriorityQueue<SearchSpaceTier>;
-  inputSequence: Distribution<Transform>[] = [];
+  private _inputSequence: Distribution<Transform>[] = [];
   private minInputCost: number[] = [];
   private rootNode: SearchNode;
 
@@ -397,7 +397,7 @@ export class SearchSpace {
     this.buildQueueSpaceComparator();
 
     if(arg1 instanceof SearchSpace) {
-      this.inputSequence = [].concat(arg1.inputSequence);
+      this._inputSequence = [].concat(arg1._inputSequence);
       this.minInputCost = [].concat(arg1.minInputCost);
       this.rootNode = arg1.rootNode;
       this.completedPaths = [].concat(arg1.completedPaths);
@@ -475,6 +475,13 @@ export class SearchSpace {
     }
   }
 
+  /**
+   * Retrieves the sequence of inputs
+   */
+  public get inputSequence() {
+    return [...this._inputSequence];
+  }
+
   increaseMaxEditDistance() {
     this.tierOrdering.forEach(function(tier) { tier.increaseMaxEditDistance() });
   }
@@ -482,11 +489,11 @@ export class SearchSpace {
   get correctionsEnabled() {
     // When corrections are disabled, the Web engine will only provide individual Transforms
     // for an input, not a distribution.  No distributions means we shouldn't do corrections.
-    return !!this.inputSequence.find((distribution) => distribution.length > 1);
+    return !!this._inputSequence.find((distribution) => distribution.length > 1);
   }
 
   addInput(inputDistribution: Distribution<Transform>) {
-    this.inputSequence.push(inputDistribution);
+    this._inputSequence.push(inputDistribution);
 
     // Assumes that `inputDistribution` is already sorted.
     this.minInputCost.push(-Math.log(inputDistribution[0].p));
@@ -607,9 +614,9 @@ export class SearchSpace {
 
       let deletionEdges: SearchNode[] = [];
       if(!substitutionsOnly) {
-        deletionEdges       = currentNode.buildDeletionEdges(this.inputSequence[inputIndex-1]);
+        deletionEdges       = currentNode.buildDeletionEdges(this._inputSequence[inputIndex-1]);
       }
-      let substitutionEdges = currentNode.buildSubstitutionEdges(this.inputSequence[inputIndex-1]);
+      let substitutionEdges = currentNode.buildSubstitutionEdges(this._inputSequence[inputIndex-1]);
 
       // Note:  we're live-modifying the tier's cost here!  The priority queue loses its guarantees as a result.
       nextTier.correctionQueue.enqueueAll(deletionEdges.concat(substitutionEdges));
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-token.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-token.tests.js
similarity index 92%
rename from web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-token.js
rename to web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-token.tests.js
index 78df8e9c2b..dc8701dc25 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-token.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-token.tests.js
@@ -1,14 +1,21 @@
+/*
+ * Keyman is copyright (C) SIL Global. MIT License.
+ *
+ * Created by jahorton on 2025-07-30
+ *
+ * This file contains low-level unit tests designed to validate the behavior
+ * of the ContextToken class.
+ */
+
 import { assert } from 'chai';
 
+// Aliased due to JS keyword.
+import { default as defaultBreaker } from '@keymanapp/models-wordbreakers';
+import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs';
+
 import { ContextToken } from '#./correction/context-token.js';
 import { ExecutionTimer } from '#./correction/execution-timer.js';
-import * as models from '#./models/index.js';
-
-import { default as defaultBreaker } from '@keymanapp/models-wordbreakers';
-
-import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs';
-
-var TrieModel = models.TrieModel;
+import { TrieModel } from '#./models/index.js';
 
 var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'),
   {wordBreaker: defaultBreaker});

From efa4ffee27fd7bcf310730e102313bd3ef283ffd Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Tue, 12 Aug 2025 20:49:11 +0700
Subject: [PATCH 47/53] change(web): Apply suggestions from code review

Co-authored-by: Marc Durdin <marc@durdin.net>
---
 .../src/main/correction/context-tokenization.ts      | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
index b6a99b5af5..66e634d8af 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
@@ -35,15 +35,9 @@ export class ContextTokenization {
    * tokens represented by this tokenization instance.
    */
   get exampleInput(): string[] {
-    const sequence: string[] = [];
-
-    for(const token of this.tokens) {
+    return this.tokens
       // Hide any tokens representing wordbreaks.  (Thinking ahead to phrase-level possibilities)
-      if(token.exampleInput !== null) {
-        sequence.push(token.exampleInput);
-      }
-    }
-
-    return sequence;
+      .filter(token => token.exampleInput !== null)
+      .map(token => token.exampleInput);
   }
 }
\ No newline at end of file

From e69c6cfecd55bcc61349abef49c8167131f7a8a3 Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Tue, 12 Aug 2025 08:54:23 -0500
Subject: [PATCH 48/53] change(web): add header comments, reorder imports

---
 .../main/correction/context-tokenization.ts    | 11 ++++++++++-
 ...zation.js => context-tokenization.tests.js} | 18 +++++++++++++-----
 2 files changed, 23 insertions(+), 6 deletions(-)
 rename web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/{context-tokenization.js => context-tokenization.tests.js} (90%)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
index 66e634d8af..9a115b3e58 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
@@ -1,3 +1,12 @@
+/*
+ * Keyman is copyright (C) SIL Global. MIT License.
+ *
+ * Created by jahorton on 2025-07-30
+ *
+ * Represents cached data about one potential tokenization of contents of
+ * the sliding context window for one specific instance of context state.
+ */
+
 import { ContextToken } from './context-token.js';
 import { TrackedContextStateAlignment } from './context-tracker.js';
 
@@ -36,7 +45,7 @@ export class ContextTokenization {
    */
   get exampleInput(): string[] {
     return this.tokens
-      // Hide any tokens representing wordbreaks.  (Thinking ahead to phrase-level possibilities)
+      // Hide any tokens representing invisible wordbreaks.  (Thinking ahead to phrase-level possibilities)
       .filter(token => token.exampleInput !== null)
       .map(token => token.exampleInput);
   }
diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.tests.js
similarity index 90%
rename from web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js
rename to web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.tests.js
index 8ba132ed56..6b0e0fffa2 100644
--- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js
+++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.tests.js
@@ -1,13 +1,21 @@
+/*
+ * Keyman is copyright (C) SIL Global. MIT License.
+ *
+ * Created by jahorton on 2025-07-30
+ *
+ * This file contains low-level tests designed to validate the behavior of the
+ * of the ContextTokenization class and its integration with the lower-level
+ * classes that it utilizes.
+ */
+
 import { assert } from 'chai';
 
-import { ContextToken } from '#./correction/context-token.js';
-import { ContextTokenization } from '#./correction/context-tokenization.js';
-
-import * as models from '#./models/index.js';
 import { default as defaultBreaker } from '@keymanapp/models-wordbreakers';
 import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs';
 
-var TrieModel = models.TrieModel;
+import { ContextToken } from '#./correction/context-token.js';
+import { ContextTokenization } from '#./correction/context-tokenization.js';
+import { TrieModel } from '#./models/index.js';
 
 var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'),
   {wordBreaker: defaultBreaker});

From 15e8c1acfba1cb4b1bc90d228128918e219febbd Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Thu, 14 Aug 2025 13:13:27 -0500
Subject: [PATCH 49/53] docs(web): add header comments to new files

---
 .../src/main/correction/alignment-helpers.ts           | 10 ++++++++++
 .../worker-thread/context/alignment-helpers.tests.ts   | 10 ++++++++++
 2 files changed, 20 insertions(+)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/alignment-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/alignment-helpers.ts
index 54679e200e..b2b2e73675 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/alignment-helpers.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/alignment-helpers.ts
@@ -1,3 +1,13 @@
+/*
+ * Keyman is copyright (C) SIL Global. MIT License.
+ *
+ * Created by jahorton on 2025-07-30
+ *
+ * This file defines methods used as helpers when aligning cached context state
+ * information with incoming contexts and when validating partial substitution
+ * edits for aligned context tokens.
+ */
+
 import { ClassicalDistanceCalculation, EditOperation } from "./classical-calculation.js";
 
 /**
diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/alignment-helpers.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/alignment-helpers.tests.ts
index 4de4bd13ec..2c9d7d1992 100644
--- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/alignment-helpers.tests.ts
+++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/alignment-helpers.tests.ts
@@ -1,3 +1,13 @@
+/*
+ * Keyman is copyright (C) SIL Global. MIT License.
+ *
+ * Created by jahorton on 2025-07-30
+ *
+ * This file contains low-level tests designed to validate helper functions
+ * used when aligning cached context states to incoming contexts and when
+ * validating potential substitution edit operations.
+ */
+
 import { assert } from 'chai';
 import { EditOperation, getEditPathLastMatch, isSubstitutionAlignable } from '@keymanapp/lm-worker/test-index';
 

From d3fabb48482d86f35648a6ade45b1f16dc637480 Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Thu, 14 Aug 2025 13:32:20 -0500
Subject: [PATCH 50/53] docs(web): adds standard header to new file, reorders
 imports

---
 .../src/main/correction/context-state.ts             | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts
index 2663f711b4..9e51a1c2bd 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts
@@ -1,6 +1,16 @@
-import { ContextTokenization } from './context-tokenization.js';
+/*
+ * Keyman is copyright (C) SIL Global. MIT License.
+ *
+ * Created by jahorton on 2025-07-30
+ *
+ * Represents cached data about the state of the sliding context window either
+ * before or after a context transition event and related functionality.
+ */
 
 import { LexicalModelTypes } from '@keymanapp/common-types';
+
+import { ContextTokenization } from './context-tokenization.js';
+
 import Context = LexicalModelTypes.Context;
 import Distribution = LexicalModelTypes.Distribution;
 import LexicalModel = LexicalModelTypes.LexicalModel;

From 87f0bdb32b3f6406a308001d5e3812ad120774c0 Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Thu, 14 Aug 2025 13:45:58 -0500
Subject: [PATCH 51/53] docs(web): add common header to new test file

---
 .../worker-thread/context/context-state.tests.ts         | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts
index 10196bd708..b229118783 100644
--- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts
+++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts
@@ -1,3 +1,12 @@
+/*
+ * Keyman is copyright (C) SIL Global. MIT License.
+ *
+ * Created by jahorton on 2025-08-01
+ *
+ * This file tests designed to validate the behavior of ContextState class and
+ * its integration with the lower-level classes that it utilizes.
+ */
+
 import { assert } from 'chai';
 
 import { default as defaultBreaker } from '@keymanapp/models-wordbreakers';

From 37721c0d6a395aa1f64815ea7b104ec7593e7949 Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Thu, 14 Aug 2025 13:56:00 -0500
Subject: [PATCH 52/53] docs(web): add a doc-header

---
 .../worker-thread/context/context-tracker.tests.ts       | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts
index 37749cc843..0856bc9aeb 100644
--- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts
+++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts
@@ -1,3 +1,12 @@
+/*
+ * Keyman is copyright (C) SIL Global. MIT License.
+ *
+ * Created by jahorton on 2025-07-30
+ *
+ * This file contains tests designed to validate the context-caching
+ * and context-tracking components for the Keyman predictive-text worker.
+ */
+
 import { assert } from 'chai';
 
 import { default as defaultBreaker } from '@keymanapp/models-wordbreakers';

From f224b8bfa52e699ff92a3dac15581693b9145afb Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Thu, 14 Aug 2025 14:09:59 -0500
Subject: [PATCH 53/53] change(web): add standard header, reorder imports

---
 .../src/main/correction/context-transition.ts        | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts
index aa1235ff1f..01f6288e0a 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts
@@ -1,6 +1,16 @@
-import { ContextState } from './context-state.js';
+/*
+ * Keyman is copyright (C) SIL Global. MIT License.
+ *
+ * Created by jahorton on 2025-07-30
+ *
+ * Represents cached data about a single context transition event, as well
+ * as the state of the context both before and after the transition.
+ */
 
 import { LexicalModelTypes } from '@keymanapp/common-types';
+
+import { ContextState } from './context-state.js';
+
 import Distribution = LexicalModelTypes.Distribution;
 import Transform = LexicalModelTypes.Transform;