Merge branch 'refactor/web/distribution-tokenization' into refactor/web/pred-context-transition

web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts

@@ -114,11 +114,13 @@ export class ContextTracker extends CircularArray<ContextState> {
     // the distribution should be tokenized already.
     transformDistribution?: Distribution<Transform> // transform distribution is needed here.
   ): ContextTransition {
-    const tokenizedContext = determineModelTokenizer(lexicalModel)(context).left;
     const baseTransition = new ContextTransition(matchState, matchState.appliedInput?.id);
-
     const transformSequenceDistribution = tokenizeAndFilterDistribution(context, lexicalModel, transformDistribution);
 
+    if(transformDistribution?.[0]) {
+      context = applyTransform(transformDistribution[0].sample, context);
+    }
+    const tokenizedContext = determineModelTokenizer(lexicalModel)(context).left;
     const alignmentResults = matchState.tokenization.computeAlignment(tokenizedContext.map((token) => token.text));
 
     if(!alignmentResults.canAlign) {
@@ -350,16 +352,12 @@ export class ContextTracker extends CircularArray<ContextState> {
     }
 
     const inputTransform = transformDistribution?.[0];
-    if(inputTransform) {
-      // These two methods apply transforms internally; do not mutate context here.
-      // This particularly matters for the 'distribution' variant.
-      context = applyTransform(inputTransform.sample, context);
-    }
+    const postContext = inputTransform ? applyTransform(inputTransform.sample, context) : context;
 
     const tokenize = determineModelTokenizer(model);
-    const tokenizedContext = tokenize(context);
+    const tokenizedPostContext = tokenize(postContext)
 
-    if(tokenizedContext.left.length > 0) {
+    if(tokenizedPostContext.left.length > 0) {
       for(let i = this.count - 1; i >= 0; i--) {
         const priorMatchState = this.item(i);
 
@@ -376,10 +374,10 @@ export class ContextTracker extends CircularArray<ContextState> {
           //
           // `priorTaggedContext` must not be `null`!
           const doublecheckContext = applyTransform(transformDistribution[0].sample, priorTaggedContext);
-          if(doublecheckContext.left != context.left) {
+          if(doublecheckContext.left != postContext.left) {
             continue;
           }
-        } else if(priorTaggedContext?.left != context.left) {
+        } else if(priorTaggedContext?.left != postContext.left) {
           continue;
         }
 

web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js

@@ -1,20 +1,15 @@
 import { assert } from 'chai';
 
 import { ContextTracker } from '#./correction/context-tracker.js';
-import { tokenizeTransformDistribution } from '#./correction/transform-tokenization.js';
 import ModelCompositor from '#./model-compositor.js';
 import * as models from '#./models/index.js';
 import * as wordBreakers from '@keymanapp/models-wordbreakers';
-import { determineModelTokenizer } from '#./model-helpers.js';
 
 import { default as defaultBreaker } from '@keymanapp/models-wordbreakers';
-import { deepCopy } from '@keymanapp/web-utils';
 
 import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs';
 import { ContextState } from '#./correction/context-state.js';
 
-const tokenizer = determineModelTokenizer(new models.DummyModel({wordbreaker: defaultBreaker}));
-
 var TrieModel = models.TrieModel;
 
 var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'),
@@ -81,14 +76,11 @@ describe('ContextTracker', function() {
         insert: 'r',
         deleteLeft: 0
       }
-      let newContext = {
-        left: "an apple a day keeps the doctor"
-      };
       let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
 
       let baseState = new ContextState(existingContext, plainModel);
       baseState.initFromReset();
-      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
+      let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.final);
       assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0);
@@ -104,14 +96,11 @@ describe('ContextTracker', function() {
         insert: ' ',
         deleteLeft: 0
       }
-      let newContext = {
-        left: "an apple a day keeps the doctor "
-      };
       let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
 
       let baseState = new ContextState(existingContext, plainModel);
       baseState.initFromReset();
-      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
+      let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.final);
       assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       // We want to preserve the added whitespace when predicting a token that follows after it.
@@ -133,14 +122,11 @@ describe('ContextTracker', function() {
         insert: '',
         deleteLeft: 1
       }
-      let newContext = {
-        left: "an apple a day keeps the doctor"
-      };
       let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
 
       let baseState = new ContextState(existingContext, plainModel);
       baseState.initFromReset();
-      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
+      let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform));
       assert.isOk(newContextMatch?.final);
       assert.deepEqual(newContextMatch?.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
 
@@ -157,14 +143,11 @@ describe('ContextTracker', function() {
         insert: 'a',
         deleteLeft: 0
       }
-      let newContext = {
-        left: "'a"
-      };
       let rawTokens = ["'", "a"];
 
       let baseState = new ContextState(existingContext, plainModel);
       baseState.initFromReset();
-      let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
+      let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform));
       assert.isNotNull(newContextMatch?.final);
       assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
       assert.deepEqual(newContextMatch.preservationTransform, { insert: '', deleteLeft: 0 });
@@ -188,7 +171,7 @@ describe('ContextTracker', function() {
         deleteLeft: 0
       }
       let newContext = {
-        left: "apple a day keeps the doctor "
+        left: "apple a day keeps the doctor"
       };
       let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
 
@@ -217,15 +200,12 @@ describe('ContextTracker', function() {
         insert: 'd ',
         deleteLeft: 0
       }
-      let newContext = {
-        left: "and "
-      };
       let rawTokens = ["and", " ", ""];
 
       let baseState = new ContextState(existingContext, plainModel);
       baseState.initFromReset();
       let newContextMatch = ContextTracker.attemptMatchContext(
-        newContext,
+        existingContext,
         plainModel,
         baseState,
         [{sample: transform, p: 1}]
@@ -252,15 +232,12 @@ describe('ContextTracker', function() {
         insert: 'tor ',
         deleteLeft: 0
       }
-      let newContext = {
-        left: "apple a day keeps the doctor "
-      };
       let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
 
       let baseState = new ContextState(existingContext, plainModel);
       baseState.initFromReset();
       let newContextMatch = ContextTracker.attemptMatchContext(
-        newContext,
+        existingContext,
         plainModel,
         baseState,
         [{sample: transform, p: 1}]
@@ -301,7 +278,7 @@ describe('ContextTracker', function() {
         deleteLeft: 0
       }
       let problemContextMatch = ContextTracker.attemptMatchContext(
-        {left: "text'\""},
+        {left: "text'"},
         plainModel,
         baseState,
         [{sample: transform, p: 1}]