diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts index 4824b284dd..26f5c47ebf 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts @@ -114,11 +114,13 @@ export class ContextTracker extends CircularArray { // the distribution should be tokenized already. transformDistribution?: Distribution // transform distribution is needed here. ): ContextTransition { - const tokenizedContext = determineModelTokenizer(lexicalModel)(context).left; const baseTransition = new ContextTransition(matchState, matchState.appliedInput?.id); - const transformSequenceDistribution = tokenizeAndFilterDistribution(context, lexicalModel, transformDistribution); + if(transformDistribution?.[0]) { + context = applyTransform(transformDistribution[0].sample, context); + } + const tokenizedContext = determineModelTokenizer(lexicalModel)(context).left; const alignmentResults = matchState.tokenization.computeAlignment(tokenizedContext.map((token) => token.text)); if(!alignmentResults.canAlign) { @@ -350,16 +352,12 @@ export class ContextTracker extends CircularArray { } const inputTransform = transformDistribution?.[0]; - if(inputTransform) { - // These two methods apply transforms internally; do not mutate context here. - // This particularly matters for the 'distribution' variant. - context = applyTransform(inputTransform.sample, context); - } + const postContext = inputTransform ? applyTransform(inputTransform.sample, context) : context; const tokenize = determineModelTokenizer(model); - const tokenizedContext = tokenize(context); + const tokenizedPostContext = tokenize(postContext) - if(tokenizedContext.left.length > 0) { + if(tokenizedPostContext.left.length > 0) { for(let i = this.count - 1; i >= 0; i--) { const priorMatchState = this.item(i); @@ -376,10 +374,10 @@ export class ContextTracker extends CircularArray { // // `priorTaggedContext` must not be `null`! const doublecheckContext = applyTransform(transformDistribution[0].sample, priorTaggedContext); - if(doublecheckContext.left != context.left) { + if(doublecheckContext.left != postContext.left) { continue; } - } else if(priorTaggedContext?.left != context.left) { + } else if(priorTaggedContext?.left != postContext.left) { continue; } diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js index 019a7c6e45..93847dcb2c 100644 --- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js +++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js @@ -1,20 +1,15 @@ import { assert } from 'chai'; import { ContextTracker } from '#./correction/context-tracker.js'; -import { tokenizeTransformDistribution } from '#./correction/transform-tokenization.js'; import ModelCompositor from '#./model-compositor.js'; import * as models from '#./models/index.js'; import * as wordBreakers from '@keymanapp/models-wordbreakers'; -import { determineModelTokenizer } from '#./model-helpers.js'; import { default as defaultBreaker } from '@keymanapp/models-wordbreakers'; -import { deepCopy } from '@keymanapp/web-utils'; import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; import { ContextState } from '#./correction/context-state.js'; -const tokenizer = determineModelTokenizer(new models.DummyModel({wordbreaker: defaultBreaker})); - var TrieModel = models.TrieModel; var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'), @@ -81,14 +76,11 @@ describe('ContextTracker', function() { insert: 'r', deleteLeft: 0 } - let newContext = { - left: "an apple a day keeps the doctor" - }; let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"]; let baseState = new ContextState(existingContext, plainModel); baseState.initFromReset(); - let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform)); + let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.final); assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0); @@ -104,14 +96,11 @@ describe('ContextTracker', function() { insert: ' ', deleteLeft: 0 } - let newContext = { - left: "an apple a day keeps the doctor " - }; let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""]; let baseState = new ContextState(existingContext, plainModel); baseState.initFromReset(); - let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform)); + let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.final); assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); // We want to preserve the added whitespace when predicting a token that follows after it. @@ -133,14 +122,11 @@ describe('ContextTracker', function() { insert: '', deleteLeft: 1 } - let newContext = { - left: "an apple a day keeps the doctor" - }; let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"]; let baseState = new ContextState(existingContext, plainModel); baseState.initFromReset(); - let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform)); + let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform)); assert.isOk(newContextMatch?.final); assert.deepEqual(newContextMatch?.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); @@ -157,14 +143,11 @@ describe('ContextTracker', function() { insert: 'a', deleteLeft: 0 } - let newContext = { - left: "'a" - }; let rawTokens = ["'", "a"]; let baseState = new ContextState(existingContext, plainModel); baseState.initFromReset(); - let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform)); + let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.final); assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); assert.deepEqual(newContextMatch.preservationTransform, { insert: '', deleteLeft: 0 }); @@ -188,7 +171,7 @@ describe('ContextTracker', function() { deleteLeft: 0 } let newContext = { - left: "apple a day keeps the doctor " + left: "apple a day keeps the doctor" }; let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""]; @@ -217,15 +200,12 @@ describe('ContextTracker', function() { insert: 'd ', deleteLeft: 0 } - let newContext = { - left: "and " - }; let rawTokens = ["and", " ", ""]; let baseState = new ContextState(existingContext, plainModel); baseState.initFromReset(); let newContextMatch = ContextTracker.attemptMatchContext( - newContext, + existingContext, plainModel, baseState, [{sample: transform, p: 1}] @@ -252,15 +232,12 @@ describe('ContextTracker', function() { insert: 'tor ', deleteLeft: 0 } - let newContext = { - left: "apple a day keeps the doctor " - }; let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""]; let baseState = new ContextState(existingContext, plainModel); baseState.initFromReset(); let newContextMatch = ContextTracker.attemptMatchContext( - newContext, + existingContext, plainModel, baseState, [{sample: transform, p: 1}] @@ -301,7 +278,7 @@ describe('ContextTracker', function() { deleteLeft: 0 } let problemContextMatch = ContextTracker.attemptMatchContext( - {left: "text'\""}, + {left: "text'"}, plainModel, baseState, [{sample: transform, p: 1}]