From 5bd5ccfca43fa72c1e117d3997622b5835715a4a Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Wed, 30 Jul 2025 15:36:15 -0500 Subject: [PATCH] refactor(web): refactor tracked-context state and usage pattern Note that this will temporarily cause predictive-text to throw out older context states (that may be valid rewind targets) in favor of newer ones (that are known to be invalid rewind targets) due to the current context-state caching logic. That'll be resolved later, in an upcoming PR - and preferably, this one shouldn't merge without that one. --- .../src/main/correction/context-state.ts | 89 +++++++++++ .../src/main/correction/context-tracker.ts | 84 ++++------- .../worker-thread/src/main/predict-helpers.ts | 5 +- .../cases/edit-distance/context-tracker.js | 139 +++++++++--------- 4 files changed, 185 insertions(+), 132 deletions(-) create mode 100644 web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts new file mode 100644 index 0000000000..244f4f362b --- /dev/null +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts @@ -0,0 +1,89 @@ +import { ContextTokenization } from './context-tokenization.js'; + +import { LexicalModelTypes } from '@keymanapp/common-types'; +import Context = LexicalModelTypes.Context; +import Distribution = LexicalModelTypes.Distribution; +import LexicalModel = LexicalModelTypes.LexicalModel; +import Suggestion = LexicalModelTypes.Suggestion; +import Transform = LexicalModelTypes.Transform; + +/** + * Represents a state of the active context at some point in time along with the + * results of all related, reusable predictive-text operations. + */ +export class ContextState { + /** + * The context window in view for the represented Context state, + * as passed between the predictive-text worker and its host. + */ + context: Context; + + /** + * The active lexical model operating upon the Context. + */ + model: LexicalModel; + + /** + * Denotes the most likely tokenization for the represented Context. + */ + tokenization: ContextTokenization; + + /** + * Denotes the keystroke-sourced Transform that was last applied to a + * prior ContextState. + * + * Note: if this specific ContextState resulted from applying a + * Suggestion, this may not match text seen in the current Context! + */ + appliedInput?: Transform; + + /** + * Denotes all keystroke data contributing to ContextTokens seen in + * .tokenization. For each contributing context transition, its ID + * may be used to retrieve the original fat-finger distribution for + * potential keystroke effects. + */ + inputTransforms: Map>; + + /** + * The full set of Suggestions produced for the transition to this context state. + */ + suggestions: Suggestion[]; + + /** + * If set, denotes the suggestion ID for the suggestion (from .suggestions) that + * was applied for the final transition to this context state. + */ + appliedSuggestionId?: number; + + /** + * Indicates whether or not the applied suggestion (if it exists) was applied + * directly by the user. + * + * - `true` if directly applied via banner interaction or other explicitly-intended + * behaviors + * - `false` if indirectly applied (say, by triggering whitespace/punctuation input) + * - `undefined` if no suggestion has been applied. + */ + isManuallyApplied?: boolean; + + constructor(stateToClone: ContextState); + constructor(context: Context, model: LexicalModel); + constructor(param1: Context | ContextState, model?: LexicalModel) { + if(!(param1 instanceof ContextState)) { + const context = param1; + this.context = context; + this.model = model; + } else { + const stateToClone = param1; + + Object.assign(this, stateToClone); + this.inputTransforms = new Map(stateToClone.inputTransforms); + this.tokenization = new ContextTokenization(stateToClone.tokenization); + + // A shallow copy of the array is fine, but we'd be best off + // not aliasing the array itself. + this.suggestions = [].concat(stateToClone.suggestions); + } + } +} \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts index e3307e8e8d..f08cc8fac4 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts @@ -1,4 +1,4 @@ -import { applyTransform, buildMergedTransform, Token } from '@keymanapp/models-templates'; +import { applyTransform, buildMergedTransform } from '@keymanapp/models-templates'; import TransformUtils from '../transformUtils.js'; import { determineModelTokenizer } from '../model-helpers.js'; @@ -7,49 +7,10 @@ import { LexicalModelTypes } from '@keymanapp/common-types'; import Context = LexicalModelTypes.Context; import Distribution = LexicalModelTypes.Distribution; import LexicalModel = LexicalModelTypes.LexicalModel; -import Suggestion = LexicalModelTypes.Suggestion; import Transform = LexicalModelTypes.Transform; import { ContextToken } from './context-token.js'; import { ContextTokenization } from './context-tokenization.js'; - -export class TrackedContextSuggestion { - suggestion: Suggestion; - tokenWidth: number; -} - -export class TrackedContextState { - // Stores the post-transform Context. Useful as a debugging reference, but also used to - // pre-validate context state matches in case of discarded changes from multitaps. - taggedContext: Context; - model: LexicalModel; - - tokenization: ContextTokenization; - - /** - * How many tokens were removed from the start of the best-matching ancestor. - * Useful for restoring older states, e.g., when the user moves the caret backwards, we can recover the context at that position. - */ - indexOffset: number; - - constructor(source: TrackedContextState); - constructor(model: LexicalModel); - constructor(obj: TrackedContextState | LexicalModel) { - if(obj instanceof TrackedContextState) { - let source = obj; - // Be sure to deep-copy the tokens! Pointer-aliasing is bad here. - this.tokenization = new ContextTokenization(source.tokenization.tokens.map((token) => new ContextToken(token))); - - this.indexOffset = 0; - this.model = obj.model; - this.taggedContext = obj.taggedContext; - } else { - let lexicalModel = obj; - this.tokenization = null; - this.indexOffset = Number.MIN_SAFE_INTEGER; - this.model = lexicalModel; - } - } -} +import { ContextState } from './context-state.js'; class CircularArray { static readonly DEFAULT_ARRAY_SIZE = 5; @@ -146,13 +107,13 @@ interface ContextMatchResult { /** * Represents the current state of the context after applying incoming keystroke data. */ - state: TrackedContextState; + state: ContextState; /** * Represents the previously-cached context state that best matches `state` if available. * May be `null` if no such state could be found within the context-state cache. */ - baseState: TrackedContextState; + baseState: ContextState; /** * Indicates the portion of the incoming keystroke data, if any, that applies to @@ -173,13 +134,16 @@ interface ContextMatchResult { tailTokensAdded: number; } -export class ContextTracker extends CircularArray { +export class ContextTracker extends CircularArray { static attemptMatchContext( - tokenizedContext: Token[], - matchState: TrackedContextState, + context: Context, + lexicalModel: LexicalModel, + matchState: ContextState, // the distribution should be tokenized already. transformSequenceDistribution?: Distribution ): ContextMatchResult { + const tokenizedContext = determineModelTokenizer(lexicalModel)(context).left; + const alignmentResults = matchState.tokenization.computeAlignment(tokenizedContext.map((token) => token.text)); if(!alignmentResults.canAlign) { @@ -223,7 +187,7 @@ export class ContextTracker extends CircularArray { // If no TAIL mutations have happened, we're safe to return now. if(tailEditLength == 0 && tailTokenShift == 0) { - const state = new TrackedContextState(matchState); + const state = new ContextState(context, lexicalModel); state.tokenization = new ContextTokenization(tokenization, alignmentResults); return { state: state, @@ -292,7 +256,7 @@ export class ContextTracker extends CircularArray { let token: ContextToken; if(isBackspace) { - token = new ContextToken(matchState.model, incomingToken.text); + token = new ContextToken(lexicalModel, incomingToken.text); token.searchSpace.inputSequence.forEach((entry) => entry[0].sample.id = primaryInput.id); } else { // Assumption: there have been no intervening keystrokes since the last well-aligned context. @@ -338,7 +302,7 @@ export class ContextTracker extends CircularArray { preservationTransform = preservationTransform && primaryInput ? buildMergedTransform(preservationTransform, primaryInput) : (primaryInput ?? preservationTransform); } - let pushedToken = new ContextToken(matchState.model); + let pushedToken = new ContextToken(lexicalModel); // TODO: assumes that there was no shift in wordbreaking from the // prior context to the current one. This may actually be a major @@ -380,7 +344,7 @@ export class ContextTracker extends CircularArray { } } - const state = new TrackedContextState(matchState); + const state = new ContextState(context, lexicalModel); state.tokenization = new ContextTokenization(tokenization, alignmentResults); return { @@ -392,10 +356,13 @@ export class ContextTracker extends CircularArray { }; } + // Aim: relocate to ContextState in some form. private static modelContextState( - tokenizedContext: Token[], + context: Context, lexicalModel: LexicalModel - ): TrackedContextState { + ): ContextState { + const tokenizedContext = determineModelTokenizer(lexicalModel)(context).left; + let baseTokens = tokenizedContext.map(function(entry) { let token = new ContextToken(lexicalModel, entry.text); @@ -407,7 +374,7 @@ export class ContextTracker extends CircularArray { }); // And now build the final context state object, which includes whitespace 'tokens'. - let state = new TrackedContextState(lexicalModel); + let state = new ContextState(context, lexicalModel); const tokenization: ContextToken[] = []; while(baseTokens.length > 0) { @@ -423,6 +390,7 @@ export class ContextTracker extends CircularArray { return state; } + // Aim: relocate to ContextState in some form... or ContextTransition? /** * Compares the current, post-input context against the most recently-seen contexts from previous prediction calls, returning * the most information-rich `TrackedContextState` possible. If a match is found, the state will be annotated with the @@ -485,7 +453,7 @@ export class ContextTracker extends CircularArray { // Skip intermediate multitap-produced contexts. // When multitapping, we skip all contexts from prior taps within the same interaction, // but not any contexts from before the multitap started. - const priorTaggedContext = priorMatchState.taggedContext; + const priorTaggedContext = priorMatchState.context; if(priorTaggedContext && transformDistribution && transformDistribution.length > 0) { // Using the potential `matchState` + the incoming transform, do the results line up for // our observed context? If not, skip it. @@ -502,7 +470,7 @@ export class ContextTracker extends CircularArray { continue; } - let result = ContextTracker.attemptMatchContext(tokenizedContext.left, this.item(i), tokenizedDistribution); + let result = ContextTracker.attemptMatchContext(context, model, this.item(i), tokenizedDistribution); if(result?.state) { // Keep it reasonably current! And it's probably fine to have it more than once @@ -513,7 +481,7 @@ export class ContextTracker extends CircularArray { this.enqueue(priorMatchState); } - result.state.taggedContext = context; + result.state.context = context; if(result.state != this.item(i)) { this.enqueue(result.state); } @@ -527,8 +495,8 @@ export class ContextTracker extends CircularArray { // // Assumption: as a caret needs to move to context before any actual transform distributions occur, // this state is only reached on caret moves; thus, transformDistribution is actually just a single null transform. - let state = ContextTracker.modelContextState(tokenizedContext.left, model); - state.taggedContext = context; + let state = ContextTracker.modelContextState(context, model); + state.context = context; this.enqueue(state); return { state, baseState: null, headTokensRemoved: 0, tailTokensAdded: 0 }; } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 30ab7eb9c9..12c2547f4f 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -3,7 +3,8 @@ import { KMWString } from '@keymanapp/web-utils'; import TransformUtils from './transformUtils.js'; import { determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js'; -import { ContextTracker, TrackedContextState } from './correction/context-tracker.js'; +import { ContextTracker } from './correction/context-tracker.js'; +import { ContextState } from './correction/context-state.js'; import { ExecutionTimer } from './correction/execution-timer.js'; import ModelCompositor from './model-compositor.js'; import { LexicalModelTypes } from '@keymanapp/common-types'; @@ -108,7 +109,7 @@ export async function correctAndEnumerate( * * Otherwise, is `null`. */ - postContextState?: TrackedContextState; + postContextState?: ContextState; /** * The suggestions generated based on the user's input state. diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js index 624f6283a2..de21766c0e 100644 --- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js +++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js @@ -30,19 +30,20 @@ describe('ContextTracker', function() { describe('attemptMatchContext', function() { it("properly matches and aligns when lead token is removed", function() { - let existingContext = models.tokenize(defaultBreaker, { + let existingContext = { left: "an apple a day keeps the doctor" - }); + }; let transform = { insert: '', deleteLeft: 0 - } - let newContext = deepCopy(existingContext); - newContext.left.splice(0, 1); + }; + let newContext = { + left: " apple a day keeps the doctor" + }; let rawTokens = [" ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"]; - let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel); - let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform)); + let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel); + let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseContextMatch, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.state); assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); assert.equal(newContextMatch.headTokensRemoved, 1); @@ -50,19 +51,20 @@ describe('ContextTracker', function() { }); it("properly matches and aligns when lead token + following whitespace are removed", function() { - let existingContext = models.tokenize(defaultBreaker, { + let existingContext = { left: "an apple a day keeps the doctor" - }); + }; let transform = { insert: '', deleteLeft: 0 - } - let newContext = deepCopy(existingContext); - newContext.left.splice(0, 2); + }; + let newContext = { + left: "apple a day keeps the doctor" + }; let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"]; - let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel); - let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform)); + let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel); + let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseContextMatch, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.state); assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); assert.equal(newContextMatch.headTokensRemoved, 2); @@ -70,20 +72,20 @@ describe('ContextTracker', function() { }); it("properly matches and aligns when final token is edited", function() { - let existingContext = models.tokenize(defaultBreaker, { + let existingContext = { left: "an apple a day keeps the docto" - }); + }; let transform = { insert: 'r', deleteLeft: 0 } - let newContext = models.tokenize(defaultBreaker, { + let newContext = { left: "an apple a day keeps the doctor" - }); + }; let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"]; - let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel); - let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform)); + let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel); + let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseContextMatch, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.state); assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); assert.equal(newContextMatch.headTokensRemoved, 0); @@ -92,20 +94,20 @@ describe('ContextTracker', function() { // Needs improved context-state management (due to 2x tokens) it("properly matches and aligns when a 'wordbreak' is added", function() { - let existingContext = models.tokenize(defaultBreaker, { + let existingContext = { left: "an apple a day keeps the doctor" - }); + }; let transform = { insert: ' ', deleteLeft: 0 } - let newContext = models.tokenize(defaultBreaker, { + let newContext = { left: "an apple a day keeps the doctor " - }); + }; let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""]; - let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel); - let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform)); + let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel); + let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseContextMatch, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.state); assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); // We want to preserve the added whitespace when predicting a token that follows after it. @@ -120,20 +122,20 @@ describe('ContextTracker', function() { }); it("properly matches and aligns when a 'wordbreak' is removed via backspace", function() { - let existingContext = models.tokenize(defaultBreaker, { + let existingContext = { left: "an apple a day keeps the doctor " - }); + }; let transform = { insert: '', deleteLeft: 1 } - let newContext = models.tokenize(defaultBreaker, { + let newContext = { left: "an apple a day keeps the doctor" - }); + }; let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"]; - let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel); - let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform)); + let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel); + let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseContextMatch, toWrapperDistribution(transform)); assert.isOk(newContextMatch?.state); assert.deepEqual(newContextMatch?.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); @@ -143,20 +145,20 @@ describe('ContextTracker', function() { }); it("properly matches and aligns when an implied 'wordbreak' occurs (as when following \"'\")", function() { - let existingContext = models.tokenize(defaultBreaker, { + let existingContext = { left: "'" - }); + }; let transform = { insert: 'a', deleteLeft: 0 } - let newContext = models.tokenize(defaultBreaker, { + let newContext = { left: "'a" - }); + }; let rawTokens = ["'", "a"]; - let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel); - let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform)); + let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel); + let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseContextMatch, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.state); assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); assert.deepEqual(newContextMatch.preservationTransform, { insert: '', deleteLeft: 0 }); @@ -172,20 +174,20 @@ describe('ContextTracker', function() { // Needs improved context-state management (due to 2x tokens) it("properly matches and aligns when lead token is removed AND a 'wordbreak' is added'", function() { - let existingContext = models.tokenize(defaultBreaker, { + let existingContext = { left: "an apple a day keeps the doctor" - }); + }; let transform = { insert: ' ', deleteLeft: 0 } - let newContext = models.tokenize(defaultBreaker, { + let newContext = { left: "apple a day keeps the doctor " - }); + }; let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""]; - let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel); - let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform)); + let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel); + let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseContextMatch, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.state); assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); // We want to preserve the added whitespace when predicting a token that follows after it. @@ -201,21 +203,22 @@ describe('ContextTracker', function() { }); it("properly matches and aligns when initial token is modified AND a 'wordbreak' is added'", function() { - let existingContext = models.tokenize(defaultBreaker, { + let existingContext = { left: "an" - }); + }; let transform = { insert: 'd ', deleteLeft: 0 } - let newContext = models.tokenize(defaultBreaker, { + let newContext = { left: "and " - }); + }; let rawTokens = ["and", " ", ""]; - let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel); + let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel); let newContextMatch = ContextTracker.attemptMatchContext( - newContext.left, + newContext, + plainModel, baseContextMatch, tokenizeTransformDistribution(tokenizer, {left: "an"}, [{sample: transform, p: 1}]) ); @@ -234,21 +237,22 @@ describe('ContextTracker', function() { }); it("properly matches and aligns when tail token is modified AND a 'wordbreak' is added'", function() { - let existingContext = models.tokenize(defaultBreaker, { + let existingContext = { left: "apple a day keeps the doc" - }); + }; let transform = { insert: 'tor ', deleteLeft: 0 } - let newContext = models.tokenize(defaultBreaker, { + let newContext = { left: "apple a day keeps the doctor " - }); + }; let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""]; - let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel); + let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel); let newContextMatch = ContextTracker.attemptMatchContext( - newContext.left, + newContext, + plainModel, baseContextMatch, tokenizeTransformDistribution(tokenizer, {left: "apple a day keeps the doc"}, [{sample: transform, p: 1}]) ); @@ -271,7 +275,7 @@ describe('ContextTracker', function() { left: "text'" }); assert.equal(baseContext.left.length, 1); - let baseContextMatch = ContextTracker.modelContextState(baseContext.left, plainModel); + let baseContextMatch = ContextTracker.modelContextState({left: "text'"}, plainModel); // Now the actual check. let newContext = models.tokenize(defaultBreaker, { @@ -286,7 +290,8 @@ describe('ContextTracker', function() { deleteLeft: 0 } let problemContextMatch = ContextTracker.attemptMatchContext( - newContext.left, + {left: "text'\""}, + plainModel, baseContextMatch, tokenizeTransformDistribution(tokenizer, {left: "text'"}, [{sample: transform, p: 1}]) ); @@ -296,28 +301,18 @@ describe('ContextTracker', function() { describe('modelContextState', function() { it('models without final wordbreak', function() { - let tokenized = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"].map((entry) => { - return { - text: entry, - isWhitespace: entry == " " - }; - }); + let context = { left: "an apple a day keeps the doctor" }; let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"]; - let state = ContextTracker.modelContextState(tokenized, plainModel); + let state = ContextTracker.modelContextState(context, plainModel); assert.deepEqual(state.tokenization.tokens.map(token => token.exampleInput), rawTokens); }); it('models with final wordbreak', function() { - let tokenized = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""].map((entry) => { - return { - text: entry, - isWhitespace: entry == " " - }; - }); + let context = { left: "an apple a day keeps the doctor " }; let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""]; - let state = ContextTracker.modelContextState(tokenized, plainModel); + let state = ContextTracker.modelContextState(context, plainModel); assert.deepEqual(state.tokenization.tokens.map(token => token.exampleInput), rawTokens); }); });