Merge pull request #16349 from keymanapp/fix/web/context-token-reconstruction-transition-ids

fix(web): annotate transition ID on reconstructed context tokens 🚂

web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts

@@ -17,6 +17,7 @@ import { generateSubsetId } from './tokenization-subsets.js';
 
 import Distribution = LexicalModelTypes.Distribution;
 import LexicalModel = LexicalModelTypes.LexicalModel;
+import ProbabilityMass = LexicalModelTypes.ProbabilityMass;
 import Transform = LexicalModelTypes.Transform;
 
 /**
@@ -113,7 +114,11 @@ export class ContextToken {
         bestProbFromSet: BASE_PROBABILITY,
         subsetId: generateSubsetId()
       };
-      searchModule = new LegacyQuotientSpur(searchModule, [{sample: transform, p: BASE_PROBABILITY}], inputMetadata);
+      const edge: ProbabilityMass<Transform> = {sample: transform, p: BASE_PROBABILITY};
+      if(transitionId !== undefined) {
+        edge.sample.id = transitionId;
+      }
+      searchModule = new LegacyQuotientSpur(searchModule, [edge], inputMetadata);
     });
 
     return new ContextToken(searchModule, isPartial);

web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts

@@ -527,6 +527,9 @@ export class ContextTokenization {
     // Mutates stackedInserts, stackedDeletes.
     const baseRemovedTokenCount = Math.max(0, stackedDeletes.length - stackedInserts.length);
     const transformMap = assembleTransforms(stackedInserts, stackedDeletes, tailIndex);
+    if(transform.id !== undefined) {
+      transformMap.forEach((v) => v.id = transform.id);
+    }
 
     // If there's an empty transform in the 0 position and we already know we're
     // dropping tokens - and only deleting - we're dropping an

web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts

@@ -51,21 +51,8 @@ describe('ContextToken', function() {
     KMWString.enableSupplementaryPlane(true);
   });
 
-  describe("<constructor>", () => {
-    it("(model: LexicalModel)", async () => {
-      let token = new ContextToken(new LegacyQuotientRoot(plainModel));
-
-      assert.equal(token.searchModule.inputCount, 0);
-      assert.isEmpty(token.exampleInput);
-      assert.isFalse(token.isWhitespace);
-
-      // While searchSpace has no inputs, it _can_ match lexicon entries (via insertions).
-      let searchIterator = getBestMatches([token.searchModule], new ExecutionTimer(Number.POSITIVE_INFINITY, Number.POSITIVE_INFINITY));
-      let firstEntry = await searchIterator.next();
-      assert.isFalse(firstEntry.done);
-    });
-
-    it("(model: LexicalModel, text: string)", () => {
+  describe('fromRawText', () => {
+    it("may construct a fresh ContextToken from a model and plain text", () => {
       let token = ContextToken.fromRawText(plainModel, "and");
 
       assert.equal(token.searchModule.bestExample.text, 'and');
@@ -82,7 +69,40 @@ describe('ContextToken', function() {
       assert.isFalse(token.isWhitespace);
     });
 
-    it("(token: ContextToken", () => {
+    it("may assign a transition ID to the constructed ContextToken", () => {
+      const TRANSITION_ID = 6;
+      let token = ContextToken.fromRawText(plainModel, "and", false, TRANSITION_ID);
+
+      assert.equal(token.searchModule.bestExample.text, 'and');
+      assert.equal(token.exampleInput, 'and');
+
+      assert.equal(token.searchModule.inputCount, 3);
+      assert.isTrue(quotientPathHasInputs(
+        token.searchModule, [
+        [{sample: { insert: 'a', deleteLeft: 0, id: TRANSITION_ID }, p: 1}],
+        [{sample: { insert: 'n', deleteLeft: 0, id: TRANSITION_ID }, p: 1}],
+        [{sample: { insert: 'd', deleteLeft: 0, id: TRANSITION_ID }, p: 1}]
+      ]));
+
+      assert.isFalse(token.isWhitespace);
+    });
+  });
+
+  describe("<constructor>", () => {
+    it("constructs from SearchQuotient root nodes", async () => {
+      let token = new ContextToken(new LegacyQuotientRoot(plainModel));
+
+      assert.equal(token.searchModule.inputCount, 0);
+      assert.isEmpty(token.exampleInput);
+      assert.isFalse(token.isWhitespace);
+
+      // While searchSpace has no inputs, it _can_ match lexicon entries (via insertions).
+      let searchIterator = getBestMatches([token.searchModule], new ExecutionTimer(Number.POSITIVE_INFINITY, Number.POSITIVE_INFINITY));
+      let firstEntry = await searchIterator.next();
+      assert.isFalse(firstEntry.done);
+    });
+
+    it("may clone a previously-existing ContextToken", () => {
       // Same as in a test above, since we verified that it works correctly.
       let baseToken = ContextToken.fromRawText(plainModel, "and");
       let clonedToken = new ContextToken(baseToken);

web/src/test/auto/headless/engine/predictive-text/worker-thread/context/transition-helpers.tests.ts

@@ -13,8 +13,11 @@ import { default as defaultBreaker } from '@keymanapp/models-wordbreakers';
 import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs';
 import { LexicalModelTypes } from '@keymanapp/common-types';
 
+import { KMWString } from 'keyman/common/web-utils';
+
 import {
   buildEdgeWindow,
+  ContextState,
   ContextToken,
   ContextTokenization,
   generateSubsetId,
@@ -99,6 +102,9 @@ function buildTokenizationForSimpleInputs (
     // Only the 'insert' and 'deleteLeft' fields are set during transform
     // tokenization at this time.
     map.set(0, { insert: sample.insert, deleteLeft: sample.deleteLeft });
+    if(sample.id !== undefined) {
+      map.get(0).id = sample.id;
+    }
     return { sample: map, p };
   })
 };
@@ -501,4 +507,32 @@ describe('transitionTokenizations', () => {
       assertMatchingTokenization(actual, expected, msg);
     }
   });
+
+  it('handles backspace transitions correctly for a recently reset context', () => {
+    const baseState = new ContextState({
+      left: 'an apple a',
+      startOfBuffer: true,
+      endOfBuffer: true
+    }, plainModel);
+
+    const dist: Distribution<Transform> = [{
+      sample: {
+        insert: '',
+        deleteLeft: 3
+      },
+      p: 1
+    }]
+    const precomputedTransition = precomputeTransitions([baseState.tokenization], dist);
+
+    const result = transitionTokenizations(precomputedTransition.subsets, dist);
+
+    const resultTokenization = result.get(precomputedTransition.keyMatchingUserContext);
+
+    assert.isOk(resultTokenization);
+    const resultTail = resultTokenization.tail;
+
+    assert.equal(resultTail.exampleInput, 'appl');
+    assert.equal(resultTail.inputCount, resultTail.searchModule.codepointLength);
+    assert.equal(resultTail.searchModule.codepointLength, KMWString.length(resultTail.exampleInput));
+  });
 });