Merge pull request #15950 from keymanapp/fix/web/create-default-keep

fix(web): fix bugs in createDefaultKeep, extend unit testing 🚂 🔪

web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts

@@ -971,7 +971,7 @@ export function processSimilarity(
 
 /**
  * Generates metadata for a new 'keep' suggestion based solely upon the existing
- * context.
+ * context and the most likely input.
  *
  * This method is designed for use when no appropriate 'keep' suggestion was
  * generated by the correction-search process.
@@ -987,19 +987,28 @@ export function createDefaultKeep(
 ): CorrectionPredictionTuple {
   const { sample: inputTransform, p: inputTransformProb } = trueInput;
   const wordbreak = determineModelWordbreaker(lexicalModel);
+  const tokenizer = determineModelTokenizer(lexicalModel);
 
   const postContext = models.applyTransform(inputTransform, context);
   const truePrefix = wordbreak(postContext);
 
-  // Generate a full-word 'keep' replacement like other suggestions when one is not otherwise
-  // produced; we want to replace the full token in the same manner used for other suggestions.
-  const basePrefixLength = KMWString.length(truePrefix) - KMWString.length(inputTransform.insert) + inputTransform.deleteLeft;
-  const keepTransform = {
-    insert: truePrefix,
-    deleteLeft: basePrefixLength
-  };
+  const tokenization = tokenizer(context);
+  const postTokenization = tokenizer(postContext);
+
+  const tokenMapper = (t: models.Token) => {
+    return {
+      exampleInput: t.text,
+      codepointLength: KMWString.length(t.text)
+    } as ContextTokenLike;
+  }
+  const transitionEffects = determineSuggestionRange(tokenization.left.map(tokenMapper), postTokenization.left.map(tokenMapper), (a, b) => a.exampleInput == b.exampleInput);
+
+  const keepTransition = {
+    insert:     transitionEffects.tokensToPredict.reduce((accum, curr) => accum + curr.exampleInput, ''),
+    deleteLeft: transitionEffects.tokensToRemove.reduce((accum, curr) => accum + curr.codepointLength, 0)
+  }
+  let keepSuggestion = models.transformToSuggestion(keepTransition);
 
-  let keepSuggestion = models.transformToSuggestion(keepTransform);
   // This is the one case where the transform doesn't insert the full word; we need to override the displayAs param.
   keepSuggestion.displayAs = truePrefix;
 

web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts

@@ -91,8 +91,8 @@ const testModelWithCasing = new DummyModel({
   // No suggestions needed here, so we don't define any.
 });
 
-describe('produceKeep', () => {
-  it(`creates an 'exact'-match suggestion based on primary input and current context`, () => {
+describe('createDefaultKeep', () => {
+  it(`creates an 'exact'-match suggestion based on simple primary input`, () => {
     const context: Context = {
       left: 'iphon',
       right: '',
@@ -132,4 +132,209 @@ describe('produceKeep', () => {
     const tuple = createDefaultKeep(testModelWithCasing, context, trueInput);
     assert.deepEqual(tuple, expectedKeep);
   });
+
+  it(`creates an 'exact'-match suggestion based on full word after a backspace`, () => {
+    const context: Context = {
+      left: 'iphone ',
+      right: '',
+      startOfBuffer: true,
+      endOfBuffer: true
+    };
+
+    const trueInput: ProbabilityMass<Transform> = {
+      sample: {
+        insert: '',
+        deleteLeft: 1
+      },
+      p: 1
+    };
+
+    const expectedKeep: CorrectionPredictionTuple = {
+      correction: {
+        sample: 'iphone',
+        p: 1
+      },
+      prediction: {
+        sample: {
+          transform: {
+            insert: 'iphone',
+            deleteLeft: 7
+          },
+          displayAs: '<iphone>',
+          matchesModel: false,
+          tag: 'keep'
+        },
+        p: 1
+      },
+      totalProb: 1,
+      matchLevel: SuggestionSimilarity.exact
+    };
+
+    const tuple = createDefaultKeep(testModelWithCasing, context, trueInput);
+    assert.deepEqual(tuple, expectedKeep);
+  });
+
+  it(`creates an 'exact'-match suggestion based on complex deletion`, () => {
+    const context: Context = {
+      left: 'iphone a',
+      right: '',
+      startOfBuffer: true,
+      endOfBuffer: true
+    };
+
+    const trueInput: ProbabilityMass<Transform> = {
+      sample: {
+        insert: 'e',
+        deleteLeft: 3
+      },
+      p: 1
+    };
+
+    const expectedKeep: CorrectionPredictionTuple = {
+      correction: {
+        sample: 'iphone',
+        p: 1
+      },
+      prediction: {
+        sample: {
+          transform: {
+            insert: 'iphone',
+            deleteLeft: 8
+          },
+          displayAs: '<iphone>',
+          matchesModel: false,
+          tag: 'keep'
+        },
+        p: 1
+      },
+      totalProb: 1,
+      matchLevel: SuggestionSimilarity.exact
+    };
+
+    const tuple = createDefaultKeep(testModelWithCasing, context, trueInput);
+    assert.deepEqual(tuple, expectedKeep);
+  });
+
+  it(`creates an 'exact'-match suggestion based on complex insertion`, () => {
+    const context: Context = {
+      left: 'iphon',
+      right: '',
+      startOfBuffer: true,
+      endOfBuffer: true
+    };
+
+    const trueInput: ProbabilityMass<Transform> = {
+      sample: {
+        insert: 'es and',
+        deleteLeft: 0
+      },
+      p: 1
+    };
+
+    const expectedKeep: CorrectionPredictionTuple = {
+      correction: {
+        sample: 'and',
+        p: 1
+      },
+      prediction: {
+        sample: {
+          transform: {
+            insert: 'iphones and',
+            deleteLeft: 5
+          },
+          displayAs: '<and>',
+          matchesModel: false,
+          tag: 'keep'
+        },
+        p: 1
+      },
+      totalProb: 1,
+      matchLevel: SuggestionSimilarity.exact
+    };
+
+    const tuple = createDefaultKeep(testModelWithCasing, context, trueInput);
+    assert.deepEqual(tuple, expectedKeep);
+  });
+
+  it(`creates an 'exact'-match suggestion based on complex replacement`, () => {
+    const context: Context = {
+      left: 'iphone ',
+      right: '',
+      startOfBuffer: true,
+      endOfBuffer: true
+    };
+
+    const trueInput: ProbabilityMass<Transform> = {
+      sample: {
+        insert: 's',
+        deleteLeft: 1
+      },
+      p: 1
+    };
+
+    const expectedKeep: CorrectionPredictionTuple = {
+      correction: {
+        sample: 'iphones',
+        p: 1
+      },
+      prediction: {
+        sample: {
+          transform: {
+            insert: 'iphones',
+            deleteLeft: 7
+          },
+          displayAs: '<iphones>',
+          matchesModel: false,
+          tag: 'keep'
+        },
+        p: 1
+      },
+      totalProb: 1,
+      matchLevel: SuggestionSimilarity.exact
+    };
+
+    const tuple = createDefaultKeep(testModelWithCasing, context, trueInput);
+    assert.deepEqual(tuple, expectedKeep);
+  });
+
+  it(`creates an empty 'exact'-match suggestion after adding a wordbreak`, () => {
+    const context: Context = {
+      left: 'iphon',
+      right: '',
+      startOfBuffer: true,
+      endOfBuffer: true
+    };
+
+    const trueInput: ProbabilityMass<Transform> = {
+      sample: {
+        insert: 'e ',
+        deleteLeft: 0
+      },
+      p: 1
+    };
+
+    const expectedKeep: CorrectionPredictionTuple = {
+      correction: {
+        sample: '',
+        p: 1
+      },
+      prediction: {
+        sample: {
+          transform: {
+            insert: 'iphone ',
+            deleteLeft: 5
+          },
+          displayAs: '<>',
+          matchesModel: false,
+          tag: 'keep'
+        },
+        p: 1
+      },
+      totalProb: 1,
+      matchLevel: SuggestionSimilarity.exact
+    };
+
+    const tuple = createDefaultKeep(testModelWithCasing, context, trueInput);
+    assert.deepEqual(tuple, expectedKeep);
+  });
 });