fix(web): fix unit test issues from last commit

web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts

@@ -13,6 +13,7 @@ import { KMWString } from 'keyman/common/web-utils';
 import { ContextToken } from './context-token.js';
 import { TransformUtils } from '../transformUtils.js';
 import { computeDistance, EditOperation, EditTuple } from './classical-calculation.js';
+import { LegacyQuotientRoot } from './legacy-quotient-root.js';
 import { determineModelTokenizer } from '../model-helpers.js';
 import { ExtendedEditOperation, SegmentableDistanceCalculation } from './segmentable-calculation.js';
 import { PathInputProperties } from './search-quotient-node.js';
@@ -20,7 +21,6 @@ import { TransitionEdge } from './tokenization-subsets.js';
 
 import LexicalModel = LexicalModelTypes.LexicalModel;
 import Transform = LexicalModelTypes.Transform;
-import { LegacyQuotientRoot } from './legacy-quotient-root.js';
 
 // May be able to "get away" with 2 & 5 or so, but having extra will likely help
 // with edit path stability.
@@ -565,7 +565,7 @@ export class ContextTokenization {
         removedTokenCount
       },
       tokenizedTransform: transformMap,
-      isBksp: transform.insert == '' && transform.deleteLeft == 1
+      isBksp: TransformUtils.isBackspace(transform)
     };
   }
 

web/src/engine/predictive-text/worker-thread/src/main/correction/transition-helpers.ts

@@ -13,6 +13,7 @@ import { ContextToken } from './context-token.js';
 import { ContextTokenization } from './context-tokenization.js';
 import { SearchQuotientCluster } from './search-quotient-cluster.js';
 import { legacySubsetKeyer, TokenizationSubset, TokenizationSubsetBuilder } from './tokenization-subsets.js';
+import { TransformUtils } from '#./transformUtils.js';
 
 import Distribution = LexicalModelTypes.Distribution;
 import Transform = LexicalModelTypes.Transform;
@@ -53,7 +54,7 @@ export function precomputeTransitions(
   const trueInput = transformDistribution[0].sample;
   const lexicalModel = startTokenizations[0]?.tail.searchModule.model;
 
-  if(trueInput.insert == '' && trueInput.deleteLeft == 0) {
+  if(TransformUtils.isBackspace(trueInput)) {
     transformDistribution = [transformDistribution[0]];
   }
 
@@ -139,23 +140,26 @@ export function transitionTokenizations(
       // If the last token is empty and has no flag for a revertable transition,
       // attempt to copy the previous token's revertable transition flag.
       const tokens = remadeTokenization.tokens;
-      const lastTokenIndex = tokens.length - 1;
-      if(tokens[lastTokenIndex].isEmptyToken && tokens[lastTokenIndex-1]) {
-        tokens[lastTokenIndex].appliedTransitionId ??= tokens[lastTokenIndex-1].appliedTransitionId
-      }
 
+      // If we have a pure backspace operation, we should forget fat-finger data
+      // and reconstruct the token without corrective data.
       if(precomp[1].isBksp) {
         const appliedEdge = precomp[1];
-        const affectedTokenCount = appliedEdge.inputs[0].sample.size;
+        const affectedTokenCount = appliedEdge.inputs[0].sample.size - appliedEdge.alignment.removedTokenCount;
 
         for(let i = 0; i < affectedTokenCount; i++)  {
           const index = tokens.length - affectedTokenCount + i;
           const token = tokens[index];
 
-          remadeTokenization.tokens[index] = ContextToken.fromRawText(token.searchModule.model, token.exampleInput, token.isPartial, trueInput.id);
+          tokens[index] = ContextToken.fromRawText(token.searchModule.model, token.exampleInput, token.isPartial, trueInput.id);
         }
       }
 
+      const lastTokenIndex = tokens.length - 1;
+      if(tokens[lastTokenIndex].isEmptyToken && tokens[lastTokenIndex-1]) {
+        tokens[lastTokenIndex].appliedTransitionId ??= tokens[lastTokenIndex-1].appliedTransitionId
+      }
+
       return remadeTokenization;
     });
 

web/src/test/auto/headless/engine/predictive-text/worker-thread/context/transition-helpers.tests.ts

@@ -28,6 +28,7 @@ import {
   SearchQuotientCluster,
   TokenizationSubset,
   TokenizationTransitionEdits,
+  TransformUtils,
   TransitionEdge,
   transitionTokenizations
 } from '@keymanapp/lm-worker/test-index';
@@ -52,7 +53,8 @@ const plainModel = new TrieModel(
 function buildOutboundTransitionEdge (
   baseTokenization: ContextTokenization,
   inputs: Distribution<Required<Transform>>,
-  tokenizedInputs: Distribution<Map<number, Transform>>
+  tokenizedInputs: Distribution<Map<number, Transform>>,
+  isBksp: boolean
 ): TransitionEdge {
   const primaryTokenizedInput = tokenizedInputs[0].sample;
   const relativeTailIndex = [...primaryTokenizedInput.keys()][0];
@@ -77,7 +79,8 @@ function buildOutboundTransitionEdge (
       removedTokenCount: 0
     },
     inputs: tokenizedInputs,
-    inputSubsetId: generateSubsetId()
+    inputSubsetId: generateSubsetId(),
+    isBksp
   };
 }
 
@@ -132,14 +135,16 @@ function generateFixtureForTokenizationOutboundTransition (
   inputPropBase: PathInputProperties
 ) {
   return dists.map((dist) => {
+    const isBksp = TransformUtils.isBackspace(dist.raw[0].sample);
     const primaryTokenizedInput = dist.tokenized[0].sample;
-    const tokenizationEdge = buildOutboundTransitionEdge(srcTokenization, dist.raw, dist.tokenized);
+    const tokenizationEdge = buildOutboundTransitionEdge(srcTokenization, dist.raw, dist.tokenized, isBksp);
 
     // Is only built for use in constructing the subset keys.  We only need data
     // from one of the inputs here.
     const keyable: TokenizationTransitionEdits = {
       tokenizedTransform: primaryTokenizedInput,
-      alignment: tokenizationEdge.alignment
+      alignment: tokenizationEdge.alignment,
+      isBksp
     };
 
     const key = precomputationSubsetKeyer(keyable);