From 9baeb9247fbbe7844da07d1f22412339cd1c7285 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 5 Aug 2025 08:42:51 -0500 Subject: [PATCH] fix(web): prevent double-application of input transforms during tokenization Fixes: #14456 --- .../src/main/correction/context-tracker.ts | 19 +++++---- .../cases/edit-distance/context-tracker.js | 39 ++++--------------- 2 files changed, 17 insertions(+), 41 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts index 035bcd99c0..2fc66d06e6 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts @@ -144,9 +144,12 @@ export class ContextTracker extends CircularArray { // the distribution should be tokenized already. transformDistribution?: Distribution // transform distribution is needed here. ): ContextMatchResult { - const tokenizedContext = determineModelTokenizer(lexicalModel)(context).left; const transformSequenceDistribution = tokenizeAndFilterDistribution(context, lexicalModel, transformDistribution); + if(transformDistribution?.[0]) { + context = applyTransform(transformDistribution[0].sample, context); + } + const tokenizedContext = determineModelTokenizer(lexicalModel)(context).left; const alignmentResults = matchState.tokenization.computeAlignment(tokenizedContext.map((token) => token.text)); if(!alignmentResults.canAlign) { @@ -386,16 +389,12 @@ export class ContextTracker extends CircularArray { transformDistribution = null; } const inputTransform = transformDistribution?.[0]; - if(inputTransform) { - // These two methods apply transforms internally; do not mutate context here. - // This particularly matters for the 'distribution' variant. - context = applyTransform(inputTransform.sample, context); - } + const postContext = inputTransform ? applyTransform(inputTransform.sample, context) : context; const tokenize = determineModelTokenizer(model); - const tokenizedContext = tokenize(context); + const tokenizedPostContext = tokenize(postContext) - if(tokenizedContext.left.length > 0) { + if(tokenizedPostContext.left.length > 0) { for(let i = this.count - 1; i >= 0; i--) { const priorMatchState = this.item(i); @@ -412,10 +411,10 @@ export class ContextTracker extends CircularArray { // // `priorTaggedContext` must not be `null`! const doublecheckContext = applyTransform(transformDistribution[0].sample, priorTaggedContext); - if(doublecheckContext.left != context.left) { + if(doublecheckContext.left != postContext.left) { continue; } - } else if(priorTaggedContext?.left != context.left) { + } else if(priorTaggedContext?.left != postContext.left) { continue; } diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js index 4e2a8c142e..11695eaf0f 100644 --- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js +++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js @@ -1,20 +1,15 @@ import { assert } from 'chai'; import { ContextTracker } from '#./correction/context-tracker.js'; -import { tokenizeTransformDistribution } from '#./correction/transform-tokenization.js'; import ModelCompositor from '#./model-compositor.js'; import * as models from '#./models/index.js'; import * as wordBreakers from '@keymanapp/models-wordbreakers'; -import { determineModelTokenizer } from '#./model-helpers.js'; import { default as defaultBreaker } from '@keymanapp/models-wordbreakers'; -import { deepCopy } from '@keymanapp/web-utils'; import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; import { ContextState } from '#./correction/context-state.js'; -const tokenizer = determineModelTokenizer(new models.DummyModel({wordbreaker: defaultBreaker})); - var TrieModel = models.TrieModel; var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'), @@ -81,14 +76,11 @@ describe('ContextTracker', function() { insert: 'r', deleteLeft: 0 } - let newContext = { - left: "an apple a day keeps the doctor" - }; let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"]; let baseState = new ContextState(existingContext, plainModel); baseState.initFromReset(); - let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform)); + let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.state); assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); assert.equal(newContextMatch.headTokensRemoved, 0); @@ -104,14 +96,11 @@ describe('ContextTracker', function() { insert: ' ', deleteLeft: 0 } - let newContext = { - left: "an apple a day keeps the doctor " - }; let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""]; let baseState = new ContextState(existingContext, plainModel); baseState.initFromReset(); - let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform)); + let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.state); assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); // We want to preserve the added whitespace when predicting a token that follows after it. @@ -133,14 +122,11 @@ describe('ContextTracker', function() { insert: '', deleteLeft: 1 } - let newContext = { - left: "an apple a day keeps the doctor" - }; let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"]; let baseState = new ContextState(existingContext, plainModel); baseState.initFromReset(); - let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform)); + let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform)); assert.isOk(newContextMatch?.state); assert.deepEqual(newContextMatch?.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); @@ -157,14 +143,11 @@ describe('ContextTracker', function() { insert: 'a', deleteLeft: 0 } - let newContext = { - left: "'a" - }; let rawTokens = ["'", "a"]; let baseState = new ContextState(existingContext, plainModel); baseState.initFromReset(); - let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform)); + let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.state); assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); assert.deepEqual(newContextMatch.preservationTransform, { insert: '', deleteLeft: 0 }); @@ -188,7 +171,7 @@ describe('ContextTracker', function() { deleteLeft: 0 } let newContext = { - left: "apple a day keeps the doctor " + left: "apple a day keeps the doctor" }; let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""]; @@ -217,15 +200,12 @@ describe('ContextTracker', function() { insert: 'd ', deleteLeft: 0 } - let newContext = { - left: "and " - }; let rawTokens = ["and", " ", ""]; let baseState = new ContextState(existingContext, plainModel); baseState.initFromReset(); let newContextMatch = ContextTracker.attemptMatchContext( - newContext, + existingContext, plainModel, baseState, [{sample: transform, p: 1}] @@ -252,15 +232,12 @@ describe('ContextTracker', function() { insert: 'tor ', deleteLeft: 0 } - let newContext = { - left: "apple a day keeps the doctor " - }; let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""]; let baseState = new ContextState(existingContext, plainModel); baseState.initFromReset(); let newContextMatch = ContextTracker.attemptMatchContext( - newContext, + existingContext, plainModel, baseState, [{sample: transform, p: 1}] @@ -301,7 +278,7 @@ describe('ContextTracker', function() { deleteLeft: 0 } let problemContextMatch = ContextTracker.attemptMatchContext( - {left: "text'\""}, + {left: "text'"}, plainModel, baseState, [{sample: transform, p: 1}]