From daf2ad5481aa5977f490cf66667b1ca34a0865b5 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 31 Jul 2025 13:16:16 -0500 Subject: [PATCH 1/4] refactor(web): refactor tracked-context state-transition return object --- .../src/main/correction/context-tracker.ts | 74 +++++---------- .../src/main/correction/context-transition.ts | 90 +++++++++++++++++++ .../src/main/model-compositor.ts | 2 +- .../worker-thread/src/main/predict-helpers.ts | 6 +- .../cases/edit-distance/context-tracker.js | 90 +++++++++---------- 5 files changed, 160 insertions(+), 102 deletions(-) create mode 100644 web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts index 035bcd99c0..7a07618fed 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts @@ -11,6 +11,7 @@ import Transform = LexicalModelTypes.Transform; import { ContextToken } from './context-token.js'; import { ContextTokenization } from './context-tokenization.js'; import { ContextState } from './context-state.js'; +import { ContextTransition } from './context-transition.js'; class CircularArray { static readonly DEFAULT_ARRAY_SIZE = 5; @@ -103,37 +104,6 @@ class CircularArray { } } -interface ContextMatchResult { - /** - * Represents the current state of the context after applying incoming keystroke data. - */ - state: ContextState; - - /** - * Represents the previously-cached context state that best matches `state` if available. - * May be `null` if no such state could be found within the context-state cache. - */ - baseState: ContextState; - - /** - * Indicates the portion of the incoming keystroke data, if any, that applies to - * tokens before the last pre-caret token and thus should not be replaced by predictions - * based upon `state`. If the provided context state + the incoming transform do not - * adequately match the current context, the match attempt will fail with a `null` result. - * - * Should generally be non-null if the token before the caret did not previously exist. - * - * The result may be null if it does not match the prior context state or if bookkeeping - * based upon it is problematic - say, if wordbreaking effects shift due to new input, - * causing a mismatch with the prior state's tokenization. - * (Refer to #12494 for an example case.) - */ - preservationTransform?: Transform; - - headTokensRemoved: number; - tailTokensAdded: number; -} - export class ContextTracker extends CircularArray { // Aim: relocate to ContextTransition in some form? // Or can we split it up in some manner across the different types? @@ -143,8 +113,10 @@ export class ContextTracker extends CircularArray { matchState: ContextState, // the distribution should be tokenized already. transformDistribution?: Distribution // transform distribution is needed here. - ): ContextMatchResult { + ): ContextTransition { const tokenizedContext = determineModelTokenizer(lexicalModel)(context).left; + const baseTransition = new ContextTransition(matchState, matchState.appliedInput?.id); + const transformSequenceDistribution = tokenizeAndFilterDistribution(context, lexicalModel, transformDistribution); const alignmentResults = matchState.tokenization.computeAlignment(tokenizedContext.map((token) => token.text)); @@ -165,7 +137,8 @@ export class ContextTracker extends CircularArray { // If we have a perfect match with a pre-existing context, no mutations have // happened; just re-use the old context state. if(tailEditLength == 0 && leadTokenShift == 0 && tailTokenShift == 0) { - return { state: matchState, baseState: matchState, headTokensRemoved: 0, tailTokensAdded: 0 };; + baseTransition.replaceFinal(new ContextState(matchState), transformDistribution); + return baseTransition; } else { // If we didn't get any input, we really should perfectly match // a previous context state. If such a state is out of our cache, @@ -192,12 +165,8 @@ export class ContextTracker extends CircularArray { if(tailEditLength == 0 && tailTokenShift == 0) { const state = new ContextState(context, lexicalModel); state.tokenization = new ContextTokenization(tokenization, alignmentResults); - return { - state: state, - baseState: matchState, - headTokensRemoved: -leadTokenShift, - tailTokensAdded: tailTokenShift - } + baseTransition.replaceFinal(state, transformDistribution); + return baseTransition; } // *** @@ -349,14 +318,8 @@ export class ContextTracker extends CircularArray { const state = new ContextState(context, lexicalModel); state.tokenization = new ContextTokenization(tokenization, alignmentResults); - - return { - state, - baseState: matchState, - preservationTransform, - headTokensRemoved: alignmentResults.leadTokenShift < 0 ? -alignmentResults.leadTokenShift : 0, - tailTokensAdded: alignmentResults.tailTokenShift - }; + baseTransition.replaceFinal(state, transformDistribution, preservationTransform); + return baseTransition; } // Aim: relocate to ContextState in some form... or ContextTransition? @@ -375,7 +338,7 @@ export class ContextTracker extends CircularArray { context: Context, transformDistribution?: Distribution, preserveMatchState?: boolean - ): ContextMatchResult { + ): ContextTransition { if(!model.traverseFromRoot) { // Assumption: LexicalModel provides a valid traverseFromRoot function. (Is technically optional) // Without it, no 'corrections' may be made; the model can only be used to predict, not correct. @@ -385,6 +348,7 @@ export class ContextTracker extends CircularArray { if(transformDistribution?.length == 0) { transformDistribution = null; } + const inputTransform = transformDistribution?.[0]; if(inputTransform) { // These two methods apply transforms internally; do not mutate context here. @@ -421,17 +385,17 @@ export class ContextTracker extends CircularArray { let result = ContextTracker.attemptMatchContext(context, model, this.item(i), transformDistribution); - if(result?.state) { + if(result?.final) { // Keep it reasonably current! And it's probably fine to have it more than once // in the history. However, if it's the most current already, there's no need // to refresh it. - if(this.newest != result.state && this.newest != priorMatchState) { + if(this.newest != result.final && this.newest != priorMatchState) { // Already has a taggedContext. this.enqueue(priorMatchState); } - if(result.state != this.item(i)) { - this.enqueue(result.state); + if(result.final != this.item(i)) { + this.enqueue(result.final); } return result; } @@ -446,7 +410,11 @@ export class ContextTracker extends CircularArray { let state = new ContextState(context, model); state.initFromReset(); this.enqueue(state); - return { state, baseState: null, headTokensRemoved: 0, tailTokensAdded: 0 }; + const transition = new ContextTransition(state, /* TODO: we need a clear value here in the future! */ null); + // Hacky, but holds the course for now. This should only really happen from context resets, which can + // then use a different path. + transition.replaceFinal(state, []); + return transition; } clearCache() { diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts new file mode 100644 index 0000000000..abc0653db7 --- /dev/null +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts @@ -0,0 +1,90 @@ +import { ContextState } from './context-state.js'; + +import { LexicalModelTypes } from '@keymanapp/common-types'; +import Distribution = LexicalModelTypes.Distribution; +import Transform = LexicalModelTypes.Transform; + +export class ContextTransition { + private states: [ContextState, ContextState]; + private baseIndex = 0; + + inputDistribution?: Distribution; + // The transform ID in play. + private _transitionId?: number; + + /** + * Indicates the portion of the incoming keystroke data, if any, that applies to + * tokens before the last pre-caret token and thus should not be replaced by predictions + * based upon `state`. If the provided context state + the incoming transform do not + * adequately match the current context, the match attempt will fail with a `null` result. + * + * Should generally be non-null if the token before the caret did not previously exist. + * + * The result may be null if it does not match the prior context state or if bookkeeping + * based upon it is problematic - say, if wordbreaking effects shift due to new input, + * causing a mismatch with the prior state's tokenization. + * (Refer to #12494 for an example case.) + */ + preservationTransform?: Transform; + + constructor(context: ContextState, transitionId: number); + constructor(baseTransition: ContextTransition); + constructor(param: ContextState | ContextTransition, transitionId?: number) { + if(!(param instanceof ContextTransition)) { + const contextState = param; + // We're initializing a ContextTransition from a blank or reset context. + const baseState = contextState; + this.states = [baseState, null]; + this._transitionId = transitionId; + } else { + const baseTransition = param; + Object.assign(this, baseTransition); + + // These need to be deep-copied. + this.states = baseTransition.states.map((entry) => new ContextState(entry)) as [ContextState, ContextState]; + } + } + + get base(): ContextState { + return this.states[this.baseIndex]; + } + + get final(): ContextState { + return this.states[this.finalIndex] + } + + private get finalIndex(): number { + return (this.baseIndex + 1) % 2; + } + + get transitionId(): number { + return this._transitionId; + } + + commitTransition(): ContextTransition { + // Preserve a deep-copy of the current object before proceeding. + const cloned = new ContextTransition(this); + + // Commit 'final' and make it the new 'base'. + const finalIndex = this.baseIndex; + this.baseIndex = this.finalIndex; + + // The old 'base' does not make a valid new 'final' - drop it. + this.states[finalIndex] = null; + + // And drop the old transition data while we're at it. + this.inputDistribution = null; + this._transitionId = null; + + return cloned; + } + + replaceFinal(state: ContextState, inputDistribution: Distribution, preservationTransform?: Transform) { + this.states[this.finalIndex] = state; + this.inputDistribution = inputDistribution; + // Long-term, this should never be null... but we need to allow it at this point + // in the refactoring process. + this._transitionId = inputDistribution?.find((entry) => entry.sample.id !== undefined)?.sample.id; + this.preservationTransform = preservationTransform; + } +} \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts index cf30c16a46..bbee5a170c 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts @@ -260,7 +260,7 @@ export class ModelCompositor { if(this.contextTracker) { let contextState = this.contextTracker.newest; if(!contextState) { - contextState = this.contextTracker.analyzeState(this.lexicalModel, context).state; + contextState = this.contextTracker.analyzeState(this.lexicalModel, context).final; } contextState.tokenization.tail.appliedSuggestionId = suggestion.id; diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 12c2547f4f..de9fc44840 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -180,11 +180,11 @@ export async function correctAndEnumerate( // facilitates a more thorough correction-search pattern. // Token replacement benefits greatly from knowledge of the prior context state. - let { state: contextState } = contextTracker.analyzeState( + let contextState = contextTracker.analyzeState( lexicalModel, context, null - ); + ).final; // Corrections and predictions are based upon the post-context state, though. const contextChangeAnalysis = contextTracker.analyzeState( @@ -194,7 +194,7 @@ export async function correctAndEnumerate( ? transformDistribution : null ); - const postContextState = contextChangeAnalysis.state; + const postContextState = contextChangeAnalysis.final; // TODO: Should we filter backspaces & whitespaces out of the transform distribution? // Ideally, the answer (in the future) will be no, but leaving it in right now may pose an issue. diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js index 4e2a8c142e..019a7c6e45 100644 --- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js +++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js @@ -45,10 +45,10 @@ describe('ContextTracker', function() { let baseState = new ContextState(existingContext, plainModel); baseState.initFromReset(); let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform)); - assert.isNotNull(newContextMatch?.state); - assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); - assert.equal(newContextMatch.headTokensRemoved, 1); - assert.equal(newContextMatch.tailTokensAdded, 0); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, -1); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0); }); it("properly matches and aligns when lead token + following whitespace are removed", function() { @@ -67,10 +67,10 @@ describe('ContextTracker', function() { let baseState = new ContextState(existingContext, plainModel); baseState.initFromReset(); let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform)); - assert.isNotNull(newContextMatch?.state); - assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); - assert.equal(newContextMatch.headTokensRemoved, 2); - assert.equal(newContextMatch.tailTokensAdded, 0); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, -2); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0); }); it("properly matches and aligns when final token is edited", function() { @@ -89,10 +89,10 @@ describe('ContextTracker', function() { let baseState = new ContextState(existingContext, plainModel); baseState.initFromReset(); let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform)); - assert.isNotNull(newContextMatch?.state); - assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); - assert.equal(newContextMatch.headTokensRemoved, 0); - assert.equal(newContextMatch.tailTokensAdded, 0); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0); }); // Needs improved context-state management (due to 2x tokens) @@ -112,17 +112,17 @@ describe('ContextTracker', function() { let baseState = new ContextState(existingContext, plainModel); baseState.initFromReset(); let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform)); - assert.isNotNull(newContextMatch?.state); - assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); // We want to preserve the added whitespace when predicting a token that follows after it. assert.deepEqual(newContextMatch.preservationTransform, { insert: ' ', deleteLeft: 0 }); // The 'wordbreak' transform - let state = newContextMatch?.state; + let state = newContextMatch?.final; assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence); assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence); - assert.equal(newContextMatch.headTokensRemoved, 0); - assert.equal(newContextMatch.tailTokensAdded, 2); + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 2); }); it("properly matches and aligns when a 'wordbreak' is removed via backspace", function() { @@ -141,12 +141,12 @@ describe('ContextTracker', function() { let baseState = new ContextState(existingContext, plainModel); baseState.initFromReset(); let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform)); - assert.isOk(newContextMatch?.state); - assert.deepEqual(newContextMatch?.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.isOk(newContextMatch?.final); + assert.deepEqual(newContextMatch?.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); // The 'wordbreak' transform - assert.equal(newContextMatch.headTokensRemoved, 0); - assert.equal(newContextMatch.tailTokensAdded, -2); + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, -2); }); it("properly matches and aligns when an implied 'wordbreak' occurs (as when following \"'\")", function() { @@ -165,17 +165,17 @@ describe('ContextTracker', function() { let baseState = new ContextState(existingContext, plainModel); baseState.initFromReset(); let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform)); - assert.isNotNull(newContextMatch?.state); - assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); assert.deepEqual(newContextMatch.preservationTransform, { insert: '', deleteLeft: 0 }); // The 'wordbreak' transform - let state = newContextMatch.state; + let state = newContextMatch.final; assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence); assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence); - assert.equal(newContextMatch.headTokensRemoved, 0); - assert.equal(newContextMatch.tailTokensAdded, 1); + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 1); }) // Needs improved context-state management (due to 2x tokens) @@ -195,18 +195,18 @@ describe('ContextTracker', function() { let baseState = new ContextState(existingContext, plainModel); baseState.initFromReset(); let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform)); - assert.isNotNull(newContextMatch?.state); - assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); // We want to preserve the added whitespace when predicting a token that follows after it. assert.deepEqual(newContextMatch.preservationTransform, { insert: ' ', deleteLeft: 0 }); // The 'wordbreak' transform - let state = newContextMatch.state; + let state = newContextMatch.final; assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence); assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence); - assert.equal(newContextMatch.headTokensRemoved, 2); - assert.equal(newContextMatch.tailTokensAdded, 2); + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, -2); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 2); }); it("properly matches and aligns when initial token is modified AND a 'wordbreak' is added'", function() { @@ -230,18 +230,18 @@ describe('ContextTracker', function() { baseState, [{sample: transform, p: 1}] ); - assert.isNotNull(newContextMatch?.state); - assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); // We want to preserve all text preceding the new token when applying a suggestion. assert.deepEqual(newContextMatch.preservationTransform, { insert: 'd ', deleteLeft: 0}); // The 'wordbreak' transform - let state = newContextMatch.state; + let state = newContextMatch.final; assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence); assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence); - assert.equal(newContextMatch.headTokensRemoved, 0); - assert.equal(newContextMatch.tailTokensAdded, 2); + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 2); }); it("properly matches and aligns when tail token is modified AND a 'wordbreak' is added'", function() { @@ -265,18 +265,18 @@ describe('ContextTracker', function() { baseState, [{sample: transform, p: 1}] ); - assert.isNotNull(newContextMatch?.state); - assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); // We want to preserve all text preceding the new token when applying a suggestion. assert.deepEqual(newContextMatch.preservationTransform, { insert: 'tor ', deleteLeft: 0 }); // The 'wordbreak' transform - let state = newContextMatch.state; + let state = newContextMatch.final; assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence); assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence); - assert.equal(newContextMatch.headTokensRemoved, 0); - assert.equal(newContextMatch.tailTokensAdded, 2); + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 2); }); it('rejects hard-to-handle case: tail token is split into three rather than two', function() { @@ -368,7 +368,7 @@ describe('ContextTracker', function() { let compositor = new ModelCompositor(model); let baseContextMatch = compositor.contextTracker.analyzeState(model, baseContext); - baseContextMatch.state.tokenization.tail.replacements = [{ + baseContextMatch.final.tokenization.tail.replacements = [{ suggestion: baseSuggestion, tokenWidth: 1 }]; @@ -376,7 +376,7 @@ describe('ContextTracker', function() { let reversion = compositor.acceptSuggestion(baseSuggestion, baseContext, postTransform); // Actual test assertion - was the replacement tracked? - assert.equal(baseContextMatch.state.tokenization.tail.appliedSuggestionId, baseSuggestion.id); + assert.equal(baseContextMatch.final.tokenization.tail.appliedSuggestionId, baseSuggestion.id); assert.equal(reversion.id, -baseSuggestion.id); // Next step - on the followup context, is the replacement still active? @@ -384,10 +384,10 @@ describe('ContextTracker', function() { let postContextMatch = compositor.contextTracker.analyzeState(model, postContext); // Penultimate token corresponds to whitespace, which does not have a 'raw' representation. - assert.equal(postContextMatch.state.tokenization.tokens[postContextMatch.state.tokenization.tokens.length - 2].exampleInput, ' '); + assert.equal(postContextMatch.final.tokenization.tokens[postContextMatch.final.tokenization.tokens.length - 2].exampleInput, ' '); // Final token is empty (follows a wordbreak) - assert.equal(postContextMatch.state.tokenization.tail.exampleInput, ''); + assert.equal(postContextMatch.final.tokenization.tail.exampleInput, ''); }); }); }); \ No newline at end of file From 1dbe62e42eafe0f50771e85e1773d5a3d4493eaf Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 4 Aug 2025 14:18:50 -0500 Subject: [PATCH 2/4] fix(web): stop unnecessarily cloning reused ContextState --- .../worker-thread/src/main/correction/context-tracker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts index 7a07618fed..4824b284dd 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts @@ -137,7 +137,7 @@ export class ContextTracker extends CircularArray { // If we have a perfect match with a pre-existing context, no mutations have // happened; just re-use the old context state. if(tailEditLength == 0 && leadTokenShift == 0 && tailTokenShift == 0) { - baseTransition.replaceFinal(new ContextState(matchState), transformDistribution); + baseTransition.replaceFinal(matchState, transformDistribution); return baseTransition; } else { // If we didn't get any input, we really should perfectly match From 1bc5d19e990b2e997d0dd8a32c08121a36ab258e Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Fri, 8 Aug 2025 12:57:33 -0500 Subject: [PATCH 3/4] refactor(web): removes double-buffered state style, renames ContextTransition.replaceFinal -> finalize --- .../src/main/correction/context-tracker.ts | 8 +- .../src/main/correction/context-transition.ts | 83 +++++++++++-------- 2 files changed, 53 insertions(+), 38 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts index d98d288a68..f6c629e1d4 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts @@ -139,7 +139,7 @@ export class ContextTracker extends CircularArray { // If we have a perfect match with a pre-existing context, no mutations have // happened; just re-use the old context state. if(tailEditLength == 0 && leadTokenShift == 0 && tailTokenShift == 0) { - baseTransition.replaceFinal(matchState, transformDistribution); + baseTransition.finalize(matchState, transformDistribution); return baseTransition; } else { // If we didn't get any input, we really should perfectly match @@ -167,7 +167,7 @@ export class ContextTracker extends CircularArray { if(tailEditLength == 0 && tailTokenShift == 0) { const state = new ContextState(context, lexicalModel); state.tokenization = new ContextTokenization(tokenization, alignmentResults); - baseTransition.replaceFinal(state, transformDistribution); + baseTransition.finalize(state, transformDistribution); return baseTransition; } @@ -320,7 +320,7 @@ export class ContextTracker extends CircularArray { const state = new ContextState(context, lexicalModel); state.tokenization = new ContextTokenization(tokenization, alignmentResults); - baseTransition.replaceFinal(state, transformDistribution, preservationTransform); + baseTransition.finalize(state, transformDistribution, preservationTransform); return baseTransition; } @@ -410,7 +410,7 @@ export class ContextTracker extends CircularArray { const transition = new ContextTransition(state, /* TODO: we need a clear value here in the future! */ null); // Hacky, but holds the course for now. This should only really happen from context resets, which can // then use a different path. - transition.replaceFinal(state, []); + transition.finalize(state, []); return transition; } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts index abc0653db7..aa1235ff1f 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts @@ -4,11 +4,23 @@ import { LexicalModelTypes } from '@keymanapp/common-types'; import Distribution = LexicalModelTypes.Distribution; import Transform = LexicalModelTypes.Transform; +/** + * Represents the transition between two context states as triggered + * by input keystrokes or applied suggestions. + */ export class ContextTransition { - private states: [ContextState, ContextState]; - private baseIndex = 0; + /** + * Represents the state of the context before the transition event occurred. + */ + readonly base: ContextState; + private _final: ContextState; + /** + * Indicates the fat-finger distribution for the incoming keystroke related to + * the context transition event. + */ inputDistribution?: Distribution; + // The transform ID in play. private _transitionId?: number; @@ -27,60 +39,63 @@ export class ContextTransition { */ preservationTransform?: Transform; + /** + * Constructs a partial context transition object for use during the process + * of analyzing context transitions or for representing the base state of a + * reset context. + * @param context The base state for the represented context transition + * @param transitionId The unique ID corresponding to the transition event + * or context state. + */ constructor(context: ContextState, transitionId: number); + /** + * Deep-copies a ContextTransition instance. + * @param baseTransition + */ constructor(baseTransition: ContextTransition); constructor(param: ContextState | ContextTransition, transitionId?: number) { if(!(param instanceof ContextTransition)) { const contextState = param; // We're initializing a ContextTransition from a blank or reset context. - const baseState = contextState; - this.states = [baseState, null]; + this.base = contextState; + this._final = null; this._transitionId = transitionId; } else { const baseTransition = param; Object.assign(this, baseTransition); // These need to be deep-copied. - this.states = baseTransition.states.map((entry) => new ContextState(entry)) as [ContextState, ContextState]; + this.base = new ContextState(baseTransition.base); + this._final = new ContextState(baseTransition._final); } } - get base(): ContextState { - return this.states[this.baseIndex]; - } - + /** + * Gets the context state resulting from the context transition event, + * including any generated suggestions and data regarding potential + * application thereof. + */ get final(): ContextState { - return this.states[this.finalIndex] - } - - private get finalIndex(): number { - return (this.baseIndex + 1) % 2; + return this._final; } + /** + * The unique ID corresponding to the transition event or context state. + */ get transitionId(): number { return this._transitionId; } - commitTransition(): ContextTransition { - // Preserve a deep-copy of the current object before proceeding. - const cloned = new ContextTransition(this); - - // Commit 'final' and make it the new 'base'. - const finalIndex = this.baseIndex; - this.baseIndex = this.finalIndex; - - // The old 'base' does not make a valid new 'final' - drop it. - this.states[finalIndex] = null; - - // And drop the old transition data while we're at it. - this.inputDistribution = null; - this._transitionId = null; - - return cloned; - } - - replaceFinal(state: ContextState, inputDistribution: Distribution, preservationTransform?: Transform) { - this.states[this.finalIndex] = state; + /** + * Records the context state resulting from the context transition generated + * by a keystroke. + * @param state The context state to record as the result of the transition + * @param inputDistribution Fat-finger data corresponding to the triggering keystroke + * @param preservationTransform Portions of the most likely input that do not contribute to the final token + * in the final context's tokenization. + */ + finalize(state: ContextState, inputDistribution: Distribution, preservationTransform?: Transform) { + this._final = state; this.inputDistribution = inputDistribution; // Long-term, this should never be null... but we need to allow it at this point // in the refactoring process. From f224b8bfa52e699ff92a3dac15581693b9145afb Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 14 Aug 2025 14:09:59 -0500 Subject: [PATCH 4/4] change(web): add standard header, reorder imports --- .../src/main/correction/context-transition.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts index aa1235ff1f..01f6288e0a 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts @@ -1,6 +1,16 @@ -import { ContextState } from './context-state.js'; +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2025-07-30 + * + * Represents cached data about a single context transition event, as well + * as the state of the context both before and after the transition. + */ import { LexicalModelTypes } from '@keymanapp/common-types'; + +import { ContextState } from './context-state.js'; + import Distribution = LexicalModelTypes.Distribution; import Transform = LexicalModelTypes.Transform;