Merge branch 'refactor/web/independent-compute-alignment' into change/web/map-transform-tokenization

web/src/engine/predictive-text/worker-thread/src/main/correction/alignment-helpers.ts

@@ -8,6 +8,7 @@
  * edits for aligned context tokens.
  */
 
+import { SENTINEL_CODE_UNIT } from '@keymanapp/models-templates';
 import { ClassicalDistanceCalculation, EditOperation } from "./classical-calculation.js";
 
 /**
@@ -18,7 +19,14 @@ export type ContextStateAlignment = {
   /**
    * Denotes whether or not alignment is possible between two contexts.
    */
-  canAlign: false
+  canAlign: false,
+
+  /**
+   * Indicates the edit path that could not be handled.  (Useful for error reporting)
+   *
+   * The edit path does not include actual user text and is sanitized.
+   */
+  editPath: EditOperation[];
 } | {
   /**
    * Denotes whether or not alignment is possible between two contexts.
@@ -201,10 +209,27 @@ export function computeAlignment(
   isSliding: boolean,
   noSubVerify?: boolean
 ): ContextStateAlignment {
+  const src = tokenizationToMatch.map(value => ({key: value}));
+  const dst = incomingTokenization.map(value => ({key: value}));
+
+  // let changedEmptyTail = false;
+  if(dst[dst.length - 1].key == '') {
+    // Only allow matching if the tokenizations are identical, thus the empty
+    // token was unaffected.
+    if(src.length != dst.length || src[dst.length - 1].key != '') {
+      // Do not allow empty-token matches to match each other; this complicates
+      // things when applying zero-root suggestions.
+      //
+      // The SENTINEL char should never appear in raw text, thus should never
+      // match anything in the "tokenization to match".
+      dst[dst.length - 1].key = SENTINEL_CODE_UNIT;
+    }
+  }
+
   // Inverted order, since 'match' existed before our new context.
   let mapping = ClassicalDistanceCalculation.computeDistance(
-    tokenizationToMatch.map(value => ({key: value})),
-    incomingTokenization.map(value => ({key: value})),
+    src,
+    dst,
     // Diagonal width to consider must be at least 2, as adding a single
     // whitespace after a token tends to add two tokens: one for whitespace,
     // one for the empty token to follow it.
@@ -212,6 +237,12 @@ export function computeAlignment(
   );
 
   let editPath = mapping.editPath();
+
+  const failure: ContextStateAlignment = {
+    canAlign: false,
+    editPath
+  };
+
   // Special case:  new context bootstrapping - first token often substitutes.
   // The text length is small enough that no words should be able to rotate out the start of the context.
   // Special handling needed in case of no 'match'; the rest of the method assumes at least one 'match'.
@@ -222,9 +253,7 @@ export function computeAlignment(
       if(editPath[i] == 'substitute') {
         subCount++;
         if(!noSubVerify && !isSubstitutionAlignable(incomingTokenization[i], tokenizationToMatch[i], true)) {
-          return {
-            canAlign: false
-          };
+          return failure;
         }
       } else if(editPath[i] == 'match') {
         // If a substitution is already recorded, treat the 'match' as a substitution.
@@ -254,16 +283,12 @@ export function computeAlignment(
 
   if(firstMatch == -1) {
     // If there are no matches, there's no alignment.
-    return {
-      canAlign: false
-    };
+    return failure;
   }
 
   // Transpositions are not allowed at the token level during context alignment.
   if(editPath.find((entry) => entry.indexOf('transpose') > -1)) {
-    return {
-      canAlign: false
-    };
+    return failure;
   }
 
   const lastMatch = getEditPathLastMatch(editPath);
@@ -272,9 +297,7 @@ export function computeAlignment(
   // continuous block of 'match' entries.  If there's anything else in
   // the middle, we have a context mismatch.
   if(lastMatch == -1) {
-    return {
-      canAlign: false
-    };
+    return failure;
   }
 
   let matchLength = lastMatch - firstMatch + 1;
@@ -290,9 +313,7 @@ export function computeAlignment(
   if(tailInsertLength > 0 && tailDeleteLength > 0) {
     // Something's gone weird if this happens; that should appear as a substitution instead.
     // Otherwise, we have a VERY niche edit scenario.
-    return {
-      canAlign: false
-    };
+    return failure;
   }
   const tailSubstituteLength = (editPath.length - 1 - lastMatch) - tailInsertLength - tailDeleteLength;
 
@@ -330,9 +351,7 @@ export function computeAlignment(
         // All deletions should appear at the sliding window edge; if a deletion appears
         // after the edge, but before the first match, something's wrong.
         if(priorEdit && priorEdit != 'delete') {
-          return {
-            canAlign: false
-          };
+          return failure;
         }
         leadTokensRemoved++;
         break;
@@ -342,9 +361,7 @@ export function computeAlignment(
         // Any extras in the front would be pure inserts, not substitutions, due to
         // the sliding context window and its implications.
         if(leadSubstitutions++ > 0) {
-          return {
-            canAlign: false
-          };
+          return failure;
         }
 
         // Find the word before and after substitution.
@@ -360,9 +377,7 @@ export function computeAlignment(
         // Exception: if the word is at the start of the context window and the
         // context window is likely sliding, don't check it.
         if(!noSubVerify && !atSlidePoint && !isSubstitutionAlignable(incomingSub, matchingSub)) {
-          return {
-            canAlign: false
-          };
+          return failure;
         }
 
         // There's no major need to drop parts of a token being 'slid' out of the context window.
@@ -372,9 +387,7 @@ export function computeAlignment(
       case 'insert':
         // Only allow an insert at the leading edge, as with 'delete's.
         if(priorEdit && priorEdit != 'insert') {
-          return {
-            canAlign: false
-          };
+          return failure;
         }
         // In case of backspaces, it's also possible to 'insert' a 'new'
         // token - an old one that's slid back into view.
@@ -383,9 +396,7 @@ export function computeAlignment(
       default:
         // No 'match' can exist before the first found index for a 'match'.
         // No 'transpose-' edits should exist within this section, either.
-        return {
-          canAlign: false
-        };
+        return failure;
     }
     priorEdit = editPath[i];
   }

web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts

@@ -197,7 +197,10 @@ export class ContextState {
     // computeDistance 'radius' in the called function.
     const alignmentResults = this.tokenization.computeAlignment(tokenizedContext.map((token) => token.text), isSliding, isApplyingSuggestion);
 
-    if(!alignmentResults.canAlign) {
+    if(alignmentResults.canAlign == false) { // Needs to be explicit for TS type inference.
+      if(console && console.error) {
+        console.error(`Could not align contexts with edit path ${JSON.stringify(alignmentResults.editPath)}`);
+      }
       return null;
     }
 

web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts

@@ -107,7 +107,9 @@ export class ContextTokenization {
     // If we have a perfect match with a pre-existing tokenization, no mutations have
     // happened; just re-use the old context tokenization.
     if(tailEditLength == 0 && leadTokenShift == 0 && tailTokenShift == 0) {
-      return this;
+      // We must build a new instance in case the original did not have
+      // alignment data (like when it's the initial context!)
+      return new ContextTokenization(this.tokens, alignment);
     } else {
       // If we didn't get any input, we really should perfectly match
       // a previous context state.  If such a state is out of our cache,

web/src/test/auto/headless/engine/predictive-text/worker-thread/context/alignment-helpers.tests.ts

@@ -182,7 +182,7 @@ describe('computeAlignment', () => {
     ];
 
     const computedAlignment = computeAlignment(baseContext, newContext, false);
-    assert.deepEqual(computedAlignment, {canAlign: false});
+    assert.deepEqual(computedAlignment, {canAlign: false, editPath: ['substitute', 'substitute', 'substitute', 'substitute', 'substitute']});
   });
 
   it("detects unalignable contexts - too many mismatching tokens", () => {
@@ -194,7 +194,7 @@ describe('computeAlignment', () => {
     ];
 
     const computedAlignment = computeAlignment(baseContext, newContext, false);
-    assert.deepEqual(computedAlignment, {canAlign: false});
+    assert.deepEqual(computedAlignment, {canAlign: false, editPath: ['substitute', 'substitute', 'match', 'match', 'match']});
   });
 
   it("fails alignment for leading-edge word substitutions", () => {
@@ -206,7 +206,7 @@ describe('computeAlignment', () => {
     ];
 
     const computedAlignment = computeAlignment(baseContext, newContext, false);
-    assert.deepEqual(computedAlignment, {canAlign: false});
+    assert.deepEqual(computedAlignment, {canAlign: false, editPath: ['substitute', 'match', 'match', 'match', 'match']});
   });
 
   it("fails alignment for small leading-edge word substitutions", () => {
@@ -218,7 +218,7 @@ describe('computeAlignment', () => {
     ];
 
     const computedAlignment = computeAlignment(baseContext, newContext, false);
-    assert.deepEqual(computedAlignment, {canAlign: false});
+    assert.deepEqual(computedAlignment, {canAlign: false, editPath: ['substitute', 'match', 'match', 'match', 'match']});
   });
 
   it("properly matches and aligns when lead token is modified", () => {
@@ -428,7 +428,7 @@ describe('computeAlignment', () => {
     ];
 
     const computedAlignment = computeAlignment(baseContext, newContext, false);
-    assert.deepEqual(computedAlignment, {canAlign: false});
+    assert.deepEqual(computedAlignment, {canAlign: false, editPath: ["match", "delete", "match", "match", "match"]});
   });
 
   it("fails alignment for mid-head insertion", () => {
@@ -440,7 +440,7 @@ describe('computeAlignment', () => {
     ];
 
     const computedAlignment = computeAlignment(baseContext, newContext, false);
-    assert.deepEqual(computedAlignment, {canAlign: false});
+    assert.deepEqual(computedAlignment, {canAlign: false, editPath: ["match", "insert", "match", "match", "match"]});
   });
 
   it("fails alignment for mid-tail deletion", () => {
@@ -452,7 +452,7 @@ describe('computeAlignment', () => {
     ];
 
     const computedAlignment = computeAlignment(baseContext, newContext, false);
-    assert.deepEqual(computedAlignment, {canAlign: false});
+    assert.deepEqual(computedAlignment, {canAlign: false, editPath: ["match", "match", "match", "delete", "match"]});
   });
 
   it("fails alignment for mid-tail insertion", () => {
@@ -464,7 +464,7 @@ describe('computeAlignment', () => {
     ];
 
     const computedAlignment = computeAlignment(baseContext, newContext, false);
-    assert.deepEqual(computedAlignment, {canAlign: false});
+    assert.deepEqual(computedAlignment, {canAlign: false, editPath: ["match", "match", "match", "match", "insert", "match"]});
   });
 
   it("handles late-context suggestion application after backspace", () => {
@@ -485,6 +485,25 @@ describe('computeAlignment', () => {
     });
   });
 
+  it("handles late-context application of default suggestion", () => {
+    const baseContext = [
+      'quick', ' ', 'brown', ' ', 'fox', ' ', 'jumped', ' ', 'over', ' ', ''
+    ];
+    const newContext = [
+      'quick', ' ', 'brown', ' ', 'fox', ' ', 'jumped', ' ', 'over', ' ', 'the', ' ', ''
+    ];
+
+    const computedAlignment = computeAlignment(baseContext, newContext, false);
+
+    assert.deepEqual(computedAlignment, {
+      canAlign: true,
+      leadTokenShift: 0,
+      matchLength: 10,
+      tailEditLength: 1,
+      tailTokenShift: 2
+    });
+  });
+
   it("handles sliding context-window scenarios", () => {
     // // Explicitly-defined window, though it's not needed directly by the method.
     // const config = {

web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts

@@ -80,7 +80,61 @@ describe('ContextState', () => {
 
 
   describe('analyzeTransition', function() {
-    it("properly matches and aligns when lead token is removed", function() {
+    it("properly matches and aligns when no context changes occur (end of word)", function() {
+      let existingContext = {
+        left: "an apple a day keeps the doctor", startOfBuffer: true, endOfBuffer: true
+      };
+      let transform = {
+        insert: '',
+        deleteLeft: 0
+      };
+      let newContext = {
+        left: "an apple a day keeps the doctor", startOfBuffer: true, endOfBuffer: true
+      };
+      let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
+
+      let baseState = new ContextState(existingContext, plainModel);
+      let newContextMatch = baseState.analyzeTransition(newContext, toWrapperDistribution(transform));
+      assert.isNotNull(newContextMatch?.final);
+      assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
+
+      // Phrased this way to facilitate TS type-inference; assert.isTrue() does
+      // NOT do this for us!
+      if(!newContextMatch.final.tokenization.alignment.canAlign) {
+        assert.fail("context alignment failed");
+      }
+      assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0);
+      assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0);
+    });
+
+    it("properly matches and aligns when no context changes occur (after whitespace)", function() {
+      let existingContext = {
+        left: "an apple a day keeps the doctor ", startOfBuffer: true, endOfBuffer: true
+      };
+      let transform = {
+        insert: '',
+        deleteLeft: 0
+      };
+      let newContext = {
+        left: "an apple a day keeps the doctor ", startOfBuffer: true, endOfBuffer: true
+      };
+      let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
+
+      let baseState = new ContextState(existingContext, plainModel);
+      let newContextMatch = baseState.analyzeTransition(newContext, toWrapperDistribution(transform));
+      assert.isNotNull(newContextMatch?.final);
+      assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
+
+      // Phrased this way to facilitate TS type-inference; assert.isTrue() does
+      // NOT do this for us!
+      if(!newContextMatch.final.tokenization.alignment.canAlign) {
+        assert.fail("context alignment failed");
+      }
+      assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0);
+      assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0);
+    });
+
+    it("properly matches and aligns when lead token is removed (end of word)", function() {
       let existingContext = {
         left: "an apple a day keeps the doctor", startOfBuffer: true, endOfBuffer: true
       };
@@ -105,6 +159,31 @@ describe('ContextState', () => {
       assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0);
     });
 
+    it("properly matches and aligns when lead token is removed (after whitespace)", function() {
+      let existingContext = {
+        left: "an apple a day keeps the doctor ", startOfBuffer: true, endOfBuffer: true
+      };
+      let transform = {
+        insert: '',
+        deleteLeft: 0
+      };
+      let newContext = {
+        left: " apple a day keeps the doctor ", startOfBuffer: true, endOfBuffer: true
+      };
+      let rawTokens = [" ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
+
+      let baseState = new ContextState(existingContext, plainModel);
+      let newContextMatch = baseState.analyzeTransition(newContext, toWrapperDistribution(transform));
+      assert.isNotNull(newContextMatch?.final);
+      assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
+
+      if(!newContextMatch.final.tokenization.alignment.canAlign) {
+        assert.fail("context alignment failed");
+      }
+      assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, -1);
+      assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0);
+    });
+
     it("properly matches and aligns when lead token + following whitespace are removed", function() {
       let existingContext = {
         left: "an apple a day keeps the doctor", startOfBuffer: true, endOfBuffer: true

web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts

@@ -14,7 +14,7 @@ import { default as defaultBreaker } from '@keymanapp/models-wordbreakers';
 import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs';
 import { LexicalModelTypes } from '@keymanapp/common-types';
 
-import { ContextToken, ContextTokenization, models } from '@keymanapp/lm-worker/test-index';
+import { ContextStateAlignment, ContextToken, ContextTokenization, models } from '@keymanapp/lm-worker/test-index';
 
 import Transform = LexicalModelTypes.Transform;
 import TrieModel = models.TrieModel;
@@ -43,7 +43,7 @@ describe('ContextTokenization', function() {
 
     it("constructs from a token array + alignment data", () => {
       const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
-      let alignment = {
+      let alignment: ContextStateAlignment = {
         canAlign: true,
         leadTokenShift: 0,
         matchLength: 6,

web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts

@@ -14,9 +14,6 @@ import { ModelCompositor, models } from '@keymanapp/lm-worker/test-index';
 
 import TrieModel = models.TrieModel;
 
-
-var emptyInput = (id: number) => [{sample: {insert: '', deleteLeft: 0, id: id}, p: 1}];
-
 describe('ContextTracker', function() {
   describe('suggestion acceptance tracking', function() {
     let englishPunctuation = {
@@ -62,7 +59,7 @@ describe('ContextTracker', function() {
       compositor.initContextTracker(baseContext, 0);
       const contextTracker = compositor.contextTracker;
 
-      let preAppliedTransition = contextTracker.analyzeState(model, baseContext, emptyInput(0));
+      let preAppliedTransition = contextTracker.latest;
       // We'll ignore and overwrite the results.  We do need the prediction round to occur, though.
       await compositor.predict([{sample: postTransform, p: 1}], baseContext);
       contextTracker.latest.final.suggestions = [baseSuggestion];
@@ -74,8 +71,7 @@ describe('ContextTracker', function() {
       contextTracker.unitTestEndPoints.cache().keys().forEach((key) => assert.isDefined(key));
 
       // Next step - on the followup context, is the replacement still active?
-      let postContext = models.applyTransform(baseSuggestion.appendedTransform, models.applyTransform(baseSuggestion.transform, baseContext));
-      let postContextMatch = compositor.contextTracker.analyzeState(model, postContext, emptyInput(2));
+      let postContextMatch = contextTracker.unitTestEndPoints.cache().get(2);
       assert.equal(postContextMatch.final.appliedSuggestionId, baseSuggestion.id);
 
       // Penultimate token corresponds to whitespace, which does not have a 'raw' representation.

web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts

@@ -19,8 +19,6 @@ import Suggestion = LexicalModelTypes.Suggestion;
 import Transform = LexicalModelTypes.Transform;
 import TrieModel = models.TrieModel;
 
-var emptyInput = (id: number) => [{sample: {insert: '', deleteLeft: 0, id: id}, p: 1}];
-
 describe('ModelCompositor', function() {
   describe('Prediction with 14.0+ models', function() {
     describe('Basic suggestion generation', function() {
@@ -115,6 +113,7 @@ describe('ModelCompositor', function() {
         let context = {
           left: '', startOfBuffer: true, endOfBuffer: true,
         };
+        compositor.initContextTracker(context, 0); // Initialize context tracking first!
 
         // The 'weights' involved imply that we have an edge-case fat finger on the bottom of
         // the 'q' key, slightly in its favor.
@@ -123,7 +122,6 @@ describe('ModelCompositor', function() {
           {sample: {insert: 'a', deleteLeft: 0, id: 1}, p: 0.4}   // but at lower weight than 'and' (998).
         ];
 
-        await compositor.predict({insert: '', deleteLeft: 0, id: 0}, context); // Initialize context tracking first!
         let suggestions = await compositor.predict(inputDistribution, context);
 
         // remove the keep suggestion; we're not testing that here.
@@ -898,12 +896,11 @@ describe('ModelCompositor', function() {
       assert.equal(reversion.transformId, -baseSuggestion.transformId);
       assert.equal(reversion.id, -baseSuggestion.id);
 
-      let postContext = models.applyTransform(baseSuggestion.appendedTransform, models.applyTransform(baseSuggestion.transform, baseContext));
-      const appliedContextState = compositor.contextTracker.analyzeState(model, postContext, emptyInput(15));
+      const appliedContextState = compositor.contextTracker.unitTestEndPoints.cache().get(13);
 
       // Accepting the suggestion rewrites the latest context transition.
-      assert.equal(compositor.contextTracker.unitTestEndPoints.cache().size, 2);
-      assert.sameMembers(compositor.contextTracker.unitTestEndPoints.cache().keys(), [15, baseSuggestion.transformId]);
+      assert.equal(compositor.contextTracker.unitTestEndPoints.cache().size, 1);
+      assert.sameMembers(compositor.contextTracker.unitTestEndPoints.cache().keys(), [baseSuggestion.transformId]);
 
       // The replacement should be marked on the context-tracking token for the applied version of the results.
       assert.equal(suggestionContextState.final.appliedSuggestionId, undefined);