fix(web): restore backspace-input token reconstruction

Before epic/autocorrect, input of a pure backspacing transform would perform special operations within the predictive-text worker, reconstructing the remainder of the token and erasing fat-finger data.  This PR fixes the regression.

This is notably useful when attempting to erase part of an applied suggestion, as applied suggestions wholesale-replace the original versions of affected tokens with a single transform input.  Backspacing into the transform will then break that unitary transform into pieces better suited for ongoing predictive-text operations.

Build-bot: skip build:web build:android

web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts

@@ -97,7 +97,7 @@ export class ContextToken {
    * @param model
    * @param rawText
    */
-  static fromRawText(model: LexicalModel, rawText: string, isPartial?: boolean) {
+  static fromRawText(model: LexicalModel, rawText: string, isPartial?: boolean, transitionId?: number) {
     rawText ||= '';
 
     // Supports the old pathway for: updateWithBackspace(tokenText: string, transitionId: number)
@@ -108,7 +108,7 @@ export class ContextToken {
       let inputMetadata: PathInputProperties = {
         segment: {
           start: 0,
-          transitionId: undefined
+          transitionId: transitionId
         },
         bestProbFromSet: BASE_PROBABILITY,
         subsetId: generateSubsetId()

web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts

@@ -154,6 +154,11 @@ export interface TokenizationTransitionEdits {
    * the end of the original context's tail token.
    */
   tokenizedTransform: Map<number, Transform>;
+
+  /**
+   * Indicates that this tokenization-transition handles a backspace transform.
+   */
+  isBksp?: boolean;
 }
 
 /**
@@ -560,6 +565,7 @@ export class ContextTokenization {
         removedTokenCount
       },
       tokenizedTransform: transformMap,
+      isBksp: transform.insert == '' && transform.deleteLeft == 1
     };
   }
 

web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts

@@ -161,23 +161,30 @@ export class ContextTransition {
       transformToApply: Transform,
       inputDistribution: Distribution<Transform>
     ) => {
+      const appliesSuggestion = transformToApply == suggestion.transform;
+
       const appliedDistribution = [{sample: transformToApply, p: 1}];
-      const { subsets: applicationSubsets, keyMatchingUserContext } = precomputeTransitions(
+      const { subsets: transitionSubsets, keyMatchingUserContext } = precomputeTransitions(
         [rootTokenization], appliedDistribution
       );
 
       // Filter out insert and delete edges here!  ONLY the primary substitution
       // edge should be permitted!
-      const directSuggestionSubset: typeof applicationSubsets = new Map();
+      let applicationSubsets: typeof transitionSubsets = new Map();
 
-      // When applying suggestions, only consider the actual tokenization that would result.
-      directSuggestionSubset.set(keyMatchingUserContext, applicationSubsets.get(keyMatchingUserContext));
+      const currentContextSubset = transitionSubsets.get(keyMatchingUserContext);
+      if(appliesSuggestion) {
+        // When applying suggestions, only consider the actual tokenization that would result.
+        applicationSubsets.set(keyMatchingUserContext, currentContextSubset);
 
-      // TODO:  verify that 'insert' and 'delete' edit-spurs are ignored (once
-      // they're supported)
+        // TODO:  verify that 'insert' and 'delete' edit-spurs are ignored (once
+        // they're supported)
+      } else {
+        applicationSubsets = transitionSubsets;
+      }
 
       const resultingTokenization = transitionTokenizations(
-        directSuggestionSubset,
+        applicationSubsets,
         appliedDistribution
       ).get(keyMatchingUserContext);
 

web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-subsets.ts

@@ -34,6 +34,11 @@ export interface TransitionEdge {
    */
   inputs: Distribution<Map<number, Transform>>
 
+  /**
+   * Indicates that the modeled transition handles a raw BKSP.
+   */
+  isBksp?: boolean;
+
   /**
    * A unique identifier associated with this TransitionEdge and its
    * transforms within `SearchSpace`s.  This ID assists with detecting when
@@ -105,6 +110,10 @@ export function editKeyer(precomputation: TokenizationTransitionEdits): string[]
     }).join(','));
   }
 
+  if(precomputation.isBksp) {
+    components.push('ISBKSP');
+  }
+
   return components;
 }
 
@@ -243,7 +252,8 @@ export class TokenizationSubsetBuilder {
     const forTokenization: TransitionEdge = entry.transitionEdges.get(tokenization) ?? {
       alignment: precomputation.alignment,
       inputs: [],
-      inputSubsetId: generateSubsetId()
+      inputSubsetId: generateSubsetId(),
+      isBksp: precomputation.isBksp
     };
 
     // Adds the incoming tokenized transform data for the pairing...

web/src/engine/predictive-text/worker-thread/src/main/correction/transition-helpers.ts

@@ -46,13 +46,17 @@ export function precomputeTransitions(
    * context edited by the user.
    */
   keyMatchingUserContext: string
-  } {
+} {
   keyer ??= legacySubsetKeyer;
 
   let keyMatchingUserContext: string;
   const trueInput = transformDistribution[0].sample;
   const lexicalModel = startTokenizations[0]?.tail.searchModule.model;
 
+  if(trueInput.insert == '' && trueInput.deleteLeft == 0) {
+    transformDistribution = [transformDistribution[0]];
+  }
+
   const subsetBuilder = new TokenizationSubsetBuilder(keyer);
 
   for(let baseTokenization of startTokenizations) {
@@ -140,6 +144,18 @@ export function transitionTokenizations(
         tokens[lastTokenIndex].appliedTransitionId ??= tokens[lastTokenIndex-1].appliedTransitionId
       }
 
+      if(precomp[1].isBksp) {
+        const appliedEdge = precomp[1];
+        const affectedTokenCount = appliedEdge.inputs[0].sample.size;
+
+        for(let i = 0; i < affectedTokenCount; i++)  {
+          const index = tokens.length - affectedTokenCount + i;
+          const token = tokens[index];
+
+          remadeTokenization.tokens[index] = ContextToken.fromRawText(token.searchModule.model, token.exampleInput, token.isPartial, trueInput.id);
+        }
+      }
+
       return remadeTokenization;
     });