diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts index 7efdee1067..f6c629e1d4 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts @@ -11,6 +11,7 @@ import Transform = LexicalModelTypes.Transform; import { ContextToken } from './context-token.js'; import { ContextTokenization } from './context-tokenization.js'; import { ContextState } from './context-state.js'; +import { ContextTransition } from './context-transition.js'; class CircularArray { static readonly DEFAULT_ARRAY_SIZE = 5; @@ -103,37 +104,6 @@ class CircularArray { } } -interface ContextMatchResult { - /** - * Represents the current state of the context after applying incoming keystroke data. - */ - state: ContextState; - - /** - * Represents the previously-cached context state that best matches `state` if available. - * May be `null` if no such state could be found within the context-state cache. - */ - baseState: ContextState; - - /** - * Indicates the portion of the incoming keystroke data, if any, that applies to - * tokens before the last pre-caret token and thus should not be replaced by predictions - * based upon `state`. If the provided context state + the incoming transform do not - * adequately match the current context, the match attempt will fail with a `null` result. - * - * Should generally be non-null if the token before the caret did not previously exist. - * - * The result may be null if it does not match the prior context state or if bookkeeping - * based upon it is problematic - say, if wordbreaking effects shift due to new input, - * causing a mismatch with the prior state's tokenization. - * (Refer to #12494 for an example case.) - */ - preservationTransform?: Transform; - - headTokensRemoved: number; - tailTokensAdded: number; -} - export class ContextTracker extends CircularArray { // Aim: relocate to ContextTransition in some form? // Or can we split it up in some manner across the different types? @@ -143,7 +113,8 @@ export class ContextTracker extends CircularArray { matchState: ContextState, // the distribution should be tokenized already. transformDistribution?: Distribution // transform distribution is needed here. - ): ContextMatchResult { + ): ContextTransition { + const baseTransition = new ContextTransition(matchState, matchState.appliedInput?.id); const transformSequenceDistribution = tokenizeAndFilterDistribution(context, lexicalModel, transformDistribution); if(transformDistribution?.[0]) { @@ -168,7 +139,8 @@ export class ContextTracker extends CircularArray { // If we have a perfect match with a pre-existing context, no mutations have // happened; just re-use the old context state. if(tailEditLength == 0 && leadTokenShift == 0 && tailTokenShift == 0) { - return { state: matchState, baseState: matchState, headTokensRemoved: 0, tailTokensAdded: 0 }; + baseTransition.finalize(matchState, transformDistribution); + return baseTransition; } else { // If we didn't get any input, we really should perfectly match // a previous context state. If such a state is out of our cache, @@ -195,12 +167,8 @@ export class ContextTracker extends CircularArray { if(tailEditLength == 0 && tailTokenShift == 0) { const state = new ContextState(context, lexicalModel); state.tokenization = new ContextTokenization(tokenization, alignmentResults); - return { - state: state, - baseState: matchState, - headTokensRemoved: -leadTokenShift, - tailTokensAdded: tailTokenShift - } + baseTransition.finalize(state, transformDistribution); + return baseTransition; } // *** @@ -352,14 +320,8 @@ export class ContextTracker extends CircularArray { const state = new ContextState(context, lexicalModel); state.tokenization = new ContextTokenization(tokenization, alignmentResults); - - return { - state, - baseState: matchState, - preservationTransform, - headTokensRemoved: alignmentResults.leadTokenShift < 0 ? -alignmentResults.leadTokenShift : 0, - tailTokensAdded: alignmentResults.tailTokenShift - }; + baseTransition.finalize(state, transformDistribution, preservationTransform); + return baseTransition; } // Aim: relocate to ContextState in some form... or ContextTransition? @@ -378,7 +340,7 @@ export class ContextTracker extends CircularArray { context: Context, transformDistribution?: Distribution, preserveMatchState?: boolean - ): ContextMatchResult { + ): ContextTransition { if(!model.traverseFromRoot) { // Assumption: LexicalModel provides a valid traverseFromRoot function. (Is technically optional) // Without it, no 'corrections' may be made; the model can only be used to predict, not correct. @@ -388,6 +350,7 @@ export class ContextTracker extends CircularArray { if(transformDistribution?.length == 0) { transformDistribution = null; } + const inputTransform = transformDistribution?.[0]; const postContext = inputTransform ? applyTransform(inputTransform.sample, context) : context; @@ -420,17 +383,17 @@ export class ContextTracker extends CircularArray { let result = ContextTracker.attemptMatchContext(context, model, this.item(i), transformDistribution); - if(result?.state) { + if(result?.final) { // Keep it reasonably current! And it's probably fine to have it more than once // in the history. However, if it's the most current already, there's no need // to refresh it. - if(this.newest != result.state && this.newest != priorMatchState) { + if(this.newest != result.final && this.newest != priorMatchState) { // Already has a taggedContext. this.enqueue(priorMatchState); } - if(result.state != this.item(i)) { - this.enqueue(result.state); + if(result.final != this.item(i)) { + this.enqueue(result.final); } return result; } @@ -444,7 +407,11 @@ export class ContextTracker extends CircularArray { // this state is only reached on caret moves; thus, transformDistribution is actually just a single null transform. let state = new ContextState(context, model); this.enqueue(state); - return { state, baseState: null, headTokensRemoved: 0, tailTokensAdded: 0 }; + const transition = new ContextTransition(state, /* TODO: we need a clear value here in the future! */ null); + // Hacky, but holds the course for now. This should only really happen from context resets, which can + // then use a different path. + transition.finalize(state, []); + return transition; } clearCache() { diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts new file mode 100644 index 0000000000..01f6288e0a --- /dev/null +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts @@ -0,0 +1,115 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2025-07-30 + * + * Represents cached data about a single context transition event, as well + * as the state of the context both before and after the transition. + */ + +import { LexicalModelTypes } from '@keymanapp/common-types'; + +import { ContextState } from './context-state.js'; + +import Distribution = LexicalModelTypes.Distribution; +import Transform = LexicalModelTypes.Transform; + +/** + * Represents the transition between two context states as triggered + * by input keystrokes or applied suggestions. + */ +export class ContextTransition { + /** + * Represents the state of the context before the transition event occurred. + */ + readonly base: ContextState; + private _final: ContextState; + + /** + * Indicates the fat-finger distribution for the incoming keystroke related to + * the context transition event. + */ + inputDistribution?: Distribution; + + // The transform ID in play. + private _transitionId?: number; + + /** + * Indicates the portion of the incoming keystroke data, if any, that applies to + * tokens before the last pre-caret token and thus should not be replaced by predictions + * based upon `state`. If the provided context state + the incoming transform do not + * adequately match the current context, the match attempt will fail with a `null` result. + * + * Should generally be non-null if the token before the caret did not previously exist. + * + * The result may be null if it does not match the prior context state or if bookkeeping + * based upon it is problematic - say, if wordbreaking effects shift due to new input, + * causing a mismatch with the prior state's tokenization. + * (Refer to #12494 for an example case.) + */ + preservationTransform?: Transform; + + /** + * Constructs a partial context transition object for use during the process + * of analyzing context transitions or for representing the base state of a + * reset context. + * @param context The base state for the represented context transition + * @param transitionId The unique ID corresponding to the transition event + * or context state. + */ + constructor(context: ContextState, transitionId: number); + /** + * Deep-copies a ContextTransition instance. + * @param baseTransition + */ + constructor(baseTransition: ContextTransition); + constructor(param: ContextState | ContextTransition, transitionId?: number) { + if(!(param instanceof ContextTransition)) { + const contextState = param; + // We're initializing a ContextTransition from a blank or reset context. + this.base = contextState; + this._final = null; + this._transitionId = transitionId; + } else { + const baseTransition = param; + Object.assign(this, baseTransition); + + // These need to be deep-copied. + this.base = new ContextState(baseTransition.base); + this._final = new ContextState(baseTransition._final); + } + } + + /** + * Gets the context state resulting from the context transition event, + * including any generated suggestions and data regarding potential + * application thereof. + */ + get final(): ContextState { + return this._final; + } + + /** + * The unique ID corresponding to the transition event or context state. + */ + get transitionId(): number { + return this._transitionId; + } + + /** + * Records the context state resulting from the context transition generated + * by a keystroke. + * @param state The context state to record as the result of the transition + * @param inputDistribution Fat-finger data corresponding to the triggering keystroke + * @param preservationTransform Portions of the most likely input that do not contribute to the final token + * in the final context's tokenization. + */ + finalize(state: ContextState, inputDistribution: Distribution, preservationTransform?: Transform) { + this._final = state; + this.inputDistribution = inputDistribution; + // Long-term, this should never be null... but we need to allow it at this point + // in the refactoring process. + this._transitionId = inputDistribution?.find((entry) => entry.sample.id !== undefined)?.sample.id; + this.preservationTransform = preservationTransform; + } +} \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts index 5a4919a298..8c6fcc9a3a 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts @@ -255,7 +255,7 @@ export class ModelCompositor { if(this.contextTracker) { let contextState = this.contextTracker.newest; if(!contextState) { - contextState = this.contextTracker.analyzeState(this.lexicalModel, context).state; + contextState = this.contextTracker.analyzeState(this.lexicalModel, context).final; } contextState.tokenization.tail.appliedSuggestionId = suggestion.id; diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 642a43838d..55576c24dd 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -241,11 +241,11 @@ export async function correctAndEnumerate( // facilitates a more thorough correction-search pattern. // Token replacement benefits greatly from knowledge of the prior context state. - let { state: contextState } = contextTracker.analyzeState( + let contextState = contextTracker.analyzeState( lexicalModel, context, null - ); + ).final; // Corrections and predictions are based upon the post-context state, though. const contextChangeAnalysis = contextTracker.analyzeState( @@ -255,7 +255,7 @@ export async function correctAndEnumerate( ? transformDistribution : null ); - const postContextState = contextChangeAnalysis.state; + const postContextState = contextChangeAnalysis.final; // TODO: Should we filter backspaces & whitespaces out of the transform distribution? // Ideally, the answer (in the future) will be no, but leaving it in right now may pose an issue. diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts index 0856bc9aeb..f9d649153f 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts @@ -51,10 +51,15 @@ describe('ContextTracker', function() { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform)); - assert.isNotNull(newContextMatch?.state); - assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); - assert.equal(newContextMatch.headTokensRemoved, 1); - assert.equal(newContextMatch.tailTokensAdded, 0); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + + if(!newContextMatch.final.tokenization.alignment.canAlign) { + // Done this way b/c TS can infer types correctly afterward. + assert.fail("context alignment failed"); + } + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, -1); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0); }); it("properly matches and aligns when lead token + following whitespace are removed", function() { @@ -76,10 +81,15 @@ describe('ContextTracker', function() { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform)); - assert.isNotNull(newContextMatch?.state); - assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); - assert.equal(newContextMatch.headTokensRemoved, 2); - assert.equal(newContextMatch.tailTokensAdded, 0); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + + if(!newContextMatch.final.tokenization.alignment.canAlign) { + // Done this way b/c TS can infer types correctly afterward. + assert.fail("context alignment failed"); + } + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, -2); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0); }); it("properly matches and aligns when final token is edited", function() { @@ -96,10 +106,15 @@ describe('ContextTracker', function() { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform)); - assert.isNotNull(newContextMatch?.state); - assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); - assert.equal(newContextMatch.headTokensRemoved, 0); - assert.equal(newContextMatch.tailTokensAdded, 0); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + + if(!newContextMatch.final.tokenization.alignment.canAlign) { + // Done this way b/c TS can infer types correctly afterward. + assert.fail("context alignment failed"); + } + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0); }); // Needs improved context-state management (due to 2x tokens) @@ -117,17 +132,22 @@ describe('ContextTracker', function() { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform)); - assert.isNotNull(newContextMatch?.state); - assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); // We want to preserve the added whitespace when predicting a token that follows after it. assert.deepEqual(newContextMatch.preservationTransform, { insert: ' ', deleteLeft: 0 }); // The 'wordbreak' transform - let state = newContextMatch?.state; + let state = newContextMatch?.final; assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence); assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence); - assert.equal(newContextMatch.headTokensRemoved, 0); - assert.equal(newContextMatch.tailTokensAdded, 2); + + if(!newContextMatch.final.tokenization.alignment.canAlign) { + // Done this way b/c TS can infer types correctly afterward. + assert.fail("context alignment failed"); + } + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 2); }); it("properly matches and aligns when a 'wordbreak' is removed via backspace", function() { @@ -144,12 +164,15 @@ describe('ContextTracker', function() { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform)); - assert.isOk(newContextMatch?.state); - assert.deepEqual(newContextMatch?.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.isOk(newContextMatch?.final); + assert.deepEqual(newContextMatch?.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); - // The 'wordbreak' transform - assert.equal(newContextMatch.headTokensRemoved, 0); - assert.equal(newContextMatch.tailTokensAdded, -2); + if(!newContextMatch.final.tokenization.alignment.canAlign) { + // Done this way b/c TS can infer types correctly afterward. + assert.fail("context alignment failed"); + } + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, -2); }); it("properly matches and aligns when an implied 'wordbreak' occurs (as when following \"'\")", function() { @@ -166,17 +189,21 @@ describe('ContextTracker', function() { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform)); - assert.isNotNull(newContextMatch?.state); - assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); assert.deepEqual(newContextMatch.preservationTransform, { insert: '', deleteLeft: 0 }); // The 'wordbreak' transform - let state = newContextMatch.state; + let state = newContextMatch.final; assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence); assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence); - assert.equal(newContextMatch.headTokensRemoved, 0); - assert.equal(newContextMatch.tailTokensAdded, 1); + if(!newContextMatch.final.tokenization.alignment.canAlign) { + // Done this way b/c TS can infer types correctly afterward. + assert.fail("context alignment failed"); + } + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 1); }) // Needs improved context-state management (due to 2x tokens) @@ -199,18 +226,22 @@ describe('ContextTracker', function() { let baseState = new ContextState(existingContext, plainModel); let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform)); - assert.isNotNull(newContextMatch?.state); - assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); // We want to preserve the added whitespace when predicting a token that follows after it. assert.deepEqual(newContextMatch.preservationTransform, { insert: ' ', deleteLeft: 0 }); // The 'wordbreak' transform - let state = newContextMatch.state; + let state = newContextMatch.final; assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence); assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence); - assert.equal(newContextMatch.headTokensRemoved, 2); - assert.equal(newContextMatch.tailTokensAdded, 2); + if(!newContextMatch.final.tokenization.alignment.canAlign) { + // Done this way b/c TS can infer types correctly afterward. + assert.fail("context alignment failed"); + } + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, -2); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 2); }); it("properly matches and aligns when initial token is modified AND a 'wordbreak' is added'", function() { @@ -232,18 +263,22 @@ describe('ContextTracker', function() { baseState, [{sample: transform, p: 1}] ); - assert.isNotNull(newContextMatch?.state); - assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); // We want to preserve all text preceding the new token when applying a suggestion. assert.deepEqual(newContextMatch.preservationTransform, { insert: 'd ', deleteLeft: 0}); // The 'wordbreak' transform - let state = newContextMatch.state; + let state = newContextMatch.final; assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence); assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence); - assert.equal(newContextMatch.headTokensRemoved, 0); - assert.equal(newContextMatch.tailTokensAdded, 2); + if(!newContextMatch.final.tokenization.alignment.canAlign) { + // Done this way b/c TS can infer types correctly afterward. + assert.fail("context alignment failed"); + } + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 2); }); it("properly matches and aligns when tail token is modified AND a 'wordbreak' is added'", function() { @@ -265,18 +300,22 @@ describe('ContextTracker', function() { baseState, [{sample: transform, p: 1}] ); - assert.isNotNull(newContextMatch?.state); - assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); // We want to preserve all text preceding the new token when applying a suggestion. assert.deepEqual(newContextMatch.preservationTransform, { insert: 'tor ', deleteLeft: 0 }); // The 'wordbreak' transform - let state = newContextMatch.state; + let state = newContextMatch.final; assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence); assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence); - assert.equal(newContextMatch.headTokensRemoved, 0); - assert.equal(newContextMatch.tailTokensAdded, 2); + if(!newContextMatch.final.tokenization.alignment.canAlign) { + // Done this way b/c TS can infer types correctly afterward. + assert.fail("context alignment failed"); + } + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 2); }); it('rejects hard-to-handle case: tail token is split into three rather than two', function() { @@ -377,12 +416,12 @@ describe('ContextTracker', function() { let compositor = new ModelCompositor(model); let baseContextMatch = compositor.contextTracker.analyzeState(model, baseContext); - baseContextMatch.state.tokenization.tail.suggestions = [ baseSuggestion ]; + baseContextMatch.final.tokenization.tail.suggestions = [ baseSuggestion ]; let reversion = compositor.acceptSuggestion(baseSuggestion, baseContext, postTransform); // Actual test assertion - was the replacement tracked? - assert.equal(baseContextMatch.state.tokenization.tail.appliedSuggestionId, baseSuggestion.id); + assert.equal(baseContextMatch.final.tokenization.tail.appliedSuggestionId, baseSuggestion.id); assert.equal(reversion.id, -baseSuggestion.id); // Next step - on the followup context, is the replacement still active? @@ -390,10 +429,10 @@ describe('ContextTracker', function() { let postContextMatch = compositor.contextTracker.analyzeState(model, postContext); // Penultimate token corresponds to whitespace, which does not have a 'raw' representation. - assert.equal(postContextMatch.state.tokenization.tokens[postContextMatch.state.tokenization.tokens.length - 2].exampleInput, ' '); + assert.equal(postContextMatch.final.tokenization.tokens[postContextMatch.final.tokenization.tokens.length - 2].exampleInput, ' '); // Final token is empty (follows a wordbreak) - assert.equal(postContextMatch.state.tokenization.tail.exampleInput, ''); + assert.equal(postContextMatch.final.tokenization.tail.exampleInput, ''); }); }); }); \ No newline at end of file