diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts new file mode 100644 index 0000000000..244f4f362b --- /dev/null +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts @@ -0,0 +1,89 @@ +import { ContextTokenization } from './context-tokenization.js'; + +import { LexicalModelTypes } from '@keymanapp/common-types'; +import Context = LexicalModelTypes.Context; +import Distribution = LexicalModelTypes.Distribution; +import LexicalModel = LexicalModelTypes.LexicalModel; +import Suggestion = LexicalModelTypes.Suggestion; +import Transform = LexicalModelTypes.Transform; + +/** + * Represents a state of the active context at some point in time along with the + * results of all related, reusable predictive-text operations. + */ +export class ContextState { + /** + * The context window in view for the represented Context state, + * as passed between the predictive-text worker and its host. + */ + context: Context; + + /** + * The active lexical model operating upon the Context. + */ + model: LexicalModel; + + /** + * Denotes the most likely tokenization for the represented Context. + */ + tokenization: ContextTokenization; + + /** + * Denotes the keystroke-sourced Transform that was last applied to a + * prior ContextState. + * + * Note: if this specific ContextState resulted from applying a + * Suggestion, this may not match text seen in the current Context! + */ + appliedInput?: Transform; + + /** + * Denotes all keystroke data contributing to ContextTokens seen in + * .tokenization. For each contributing context transition, its ID + * may be used to retrieve the original fat-finger distribution for + * potential keystroke effects. + */ + inputTransforms: Map>; + + /** + * The full set of Suggestions produced for the transition to this context state. + */ + suggestions: Suggestion[]; + + /** + * If set, denotes the suggestion ID for the suggestion (from .suggestions) that + * was applied for the final transition to this context state. + */ + appliedSuggestionId?: number; + + /** + * Indicates whether or not the applied suggestion (if it exists) was applied + * directly by the user. + * + * - `true` if directly applied via banner interaction or other explicitly-intended + * behaviors + * - `false` if indirectly applied (say, by triggering whitespace/punctuation input) + * - `undefined` if no suggestion has been applied. + */ + isManuallyApplied?: boolean; + + constructor(stateToClone: ContextState); + constructor(context: Context, model: LexicalModel); + constructor(param1: Context | ContextState, model?: LexicalModel) { + if(!(param1 instanceof ContextState)) { + const context = param1; + this.context = context; + this.model = model; + } else { + const stateToClone = param1; + + Object.assign(this, stateToClone); + this.inputTransforms = new Map(stateToClone.inputTransforms); + this.tokenization = new ContextTokenization(stateToClone.tokenization); + + // A shallow copy of the array is fine, but we'd be best off + // not aliasing the array itself. + this.suggestions = [].concat(stateToClone.suggestions); + } + } +} \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts index e3307e8e8d..f08cc8fac4 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts @@ -1,4 +1,4 @@ -import { applyTransform, buildMergedTransform, Token } from '@keymanapp/models-templates'; +import { applyTransform, buildMergedTransform } from '@keymanapp/models-templates'; import TransformUtils from '../transformUtils.js'; import { determineModelTokenizer } from '../model-helpers.js'; @@ -7,49 +7,10 @@ import { LexicalModelTypes } from '@keymanapp/common-types'; import Context = LexicalModelTypes.Context; import Distribution = LexicalModelTypes.Distribution; import LexicalModel = LexicalModelTypes.LexicalModel; -import Suggestion = LexicalModelTypes.Suggestion; import Transform = LexicalModelTypes.Transform; import { ContextToken } from './context-token.js'; import { ContextTokenization } from './context-tokenization.js'; - -export class TrackedContextSuggestion { - suggestion: Suggestion; - tokenWidth: number; -} - -export class TrackedContextState { - // Stores the post-transform Context. Useful as a debugging reference, but also used to - // pre-validate context state matches in case of discarded changes from multitaps. - taggedContext: Context; - model: LexicalModel; - - tokenization: ContextTokenization; - - /** - * How many tokens were removed from the start of the best-matching ancestor. - * Useful for restoring older states, e.g., when the user moves the caret backwards, we can recover the context at that position. - */ - indexOffset: number; - - constructor(source: TrackedContextState); - constructor(model: LexicalModel); - constructor(obj: TrackedContextState | LexicalModel) { - if(obj instanceof TrackedContextState) { - let source = obj; - // Be sure to deep-copy the tokens! Pointer-aliasing is bad here. - this.tokenization = new ContextTokenization(source.tokenization.tokens.map((token) => new ContextToken(token))); - - this.indexOffset = 0; - this.model = obj.model; - this.taggedContext = obj.taggedContext; - } else { - let lexicalModel = obj; - this.tokenization = null; - this.indexOffset = Number.MIN_SAFE_INTEGER; - this.model = lexicalModel; - } - } -} +import { ContextState } from './context-state.js'; class CircularArray { static readonly DEFAULT_ARRAY_SIZE = 5; @@ -146,13 +107,13 @@ interface ContextMatchResult { /** * Represents the current state of the context after applying incoming keystroke data. */ - state: TrackedContextState; + state: ContextState; /** * Represents the previously-cached context state that best matches `state` if available. * May be `null` if no such state could be found within the context-state cache. */ - baseState: TrackedContextState; + baseState: ContextState; /** * Indicates the portion of the incoming keystroke data, if any, that applies to @@ -173,13 +134,16 @@ interface ContextMatchResult { tailTokensAdded: number; } -export class ContextTracker extends CircularArray { +export class ContextTracker extends CircularArray { static attemptMatchContext( - tokenizedContext: Token[], - matchState: TrackedContextState, + context: Context, + lexicalModel: LexicalModel, + matchState: ContextState, // the distribution should be tokenized already. transformSequenceDistribution?: Distribution ): ContextMatchResult { + const tokenizedContext = determineModelTokenizer(lexicalModel)(context).left; + const alignmentResults = matchState.tokenization.computeAlignment(tokenizedContext.map((token) => token.text)); if(!alignmentResults.canAlign) { @@ -223,7 +187,7 @@ export class ContextTracker extends CircularArray { // If no TAIL mutations have happened, we're safe to return now. if(tailEditLength == 0 && tailTokenShift == 0) { - const state = new TrackedContextState(matchState); + const state = new ContextState(context, lexicalModel); state.tokenization = new ContextTokenization(tokenization, alignmentResults); return { state: state, @@ -292,7 +256,7 @@ export class ContextTracker extends CircularArray { let token: ContextToken; if(isBackspace) { - token = new ContextToken(matchState.model, incomingToken.text); + token = new ContextToken(lexicalModel, incomingToken.text); token.searchSpace.inputSequence.forEach((entry) => entry[0].sample.id = primaryInput.id); } else { // Assumption: there have been no intervening keystrokes since the last well-aligned context. @@ -338,7 +302,7 @@ export class ContextTracker extends CircularArray { preservationTransform = preservationTransform && primaryInput ? buildMergedTransform(preservationTransform, primaryInput) : (primaryInput ?? preservationTransform); } - let pushedToken = new ContextToken(matchState.model); + let pushedToken = new ContextToken(lexicalModel); // TODO: assumes that there was no shift in wordbreaking from the // prior context to the current one. This may actually be a major @@ -380,7 +344,7 @@ export class ContextTracker extends CircularArray { } } - const state = new TrackedContextState(matchState); + const state = new ContextState(context, lexicalModel); state.tokenization = new ContextTokenization(tokenization, alignmentResults); return { @@ -392,10 +356,13 @@ export class ContextTracker extends CircularArray { }; } + // Aim: relocate to ContextState in some form. private static modelContextState( - tokenizedContext: Token[], + context: Context, lexicalModel: LexicalModel - ): TrackedContextState { + ): ContextState { + const tokenizedContext = determineModelTokenizer(lexicalModel)(context).left; + let baseTokens = tokenizedContext.map(function(entry) { let token = new ContextToken(lexicalModel, entry.text); @@ -407,7 +374,7 @@ export class ContextTracker extends CircularArray { }); // And now build the final context state object, which includes whitespace 'tokens'. - let state = new TrackedContextState(lexicalModel); + let state = new ContextState(context, lexicalModel); const tokenization: ContextToken[] = []; while(baseTokens.length > 0) { @@ -423,6 +390,7 @@ export class ContextTracker extends CircularArray { return state; } + // Aim: relocate to ContextState in some form... or ContextTransition? /** * Compares the current, post-input context against the most recently-seen contexts from previous prediction calls, returning * the most information-rich `TrackedContextState` possible. If a match is found, the state will be annotated with the @@ -485,7 +453,7 @@ export class ContextTracker extends CircularArray { // Skip intermediate multitap-produced contexts. // When multitapping, we skip all contexts from prior taps within the same interaction, // but not any contexts from before the multitap started. - const priorTaggedContext = priorMatchState.taggedContext; + const priorTaggedContext = priorMatchState.context; if(priorTaggedContext && transformDistribution && transformDistribution.length > 0) { // Using the potential `matchState` + the incoming transform, do the results line up for // our observed context? If not, skip it. @@ -502,7 +470,7 @@ export class ContextTracker extends CircularArray { continue; } - let result = ContextTracker.attemptMatchContext(tokenizedContext.left, this.item(i), tokenizedDistribution); + let result = ContextTracker.attemptMatchContext(context, model, this.item(i), tokenizedDistribution); if(result?.state) { // Keep it reasonably current! And it's probably fine to have it more than once @@ -513,7 +481,7 @@ export class ContextTracker extends CircularArray { this.enqueue(priorMatchState); } - result.state.taggedContext = context; + result.state.context = context; if(result.state != this.item(i)) { this.enqueue(result.state); } @@ -527,8 +495,8 @@ export class ContextTracker extends CircularArray { // // Assumption: as a caret needs to move to context before any actual transform distributions occur, // this state is only reached on caret moves; thus, transformDistribution is actually just a single null transform. - let state = ContextTracker.modelContextState(tokenizedContext.left, model); - state.taggedContext = context; + let state = ContextTracker.modelContextState(context, model); + state.context = context; this.enqueue(state); return { state, baseState: null, headTokensRemoved: 0, tailTokensAdded: 0 }; } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 30ab7eb9c9..12c2547f4f 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -3,7 +3,8 @@ import { KMWString } from '@keymanapp/web-utils'; import TransformUtils from './transformUtils.js'; import { determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js'; -import { ContextTracker, TrackedContextState } from './correction/context-tracker.js'; +import { ContextTracker } from './correction/context-tracker.js'; +import { ContextState } from './correction/context-state.js'; import { ExecutionTimer } from './correction/execution-timer.js'; import ModelCompositor from './model-compositor.js'; import { LexicalModelTypes } from '@keymanapp/common-types'; @@ -108,7 +109,7 @@ export async function correctAndEnumerate( * * Otherwise, is `null`. */ - postContextState?: TrackedContextState; + postContextState?: ContextState; /** * The suggestions generated based on the user's input state. diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js index 624f6283a2..de21766c0e 100644 --- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js +++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js @@ -30,19 +30,20 @@ describe('ContextTracker', function() { describe('attemptMatchContext', function() { it("properly matches and aligns when lead token is removed", function() { - let existingContext = models.tokenize(defaultBreaker, { + let existingContext = { left: "an apple a day keeps the doctor" - }); + }; let transform = { insert: '', deleteLeft: 0 - } - let newContext = deepCopy(existingContext); - newContext.left.splice(0, 1); + }; + let newContext = { + left: " apple a day keeps the doctor" + }; let rawTokens = [" ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"]; - let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel); - let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform)); + let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel); + let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseContextMatch, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.state); assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); assert.equal(newContextMatch.headTokensRemoved, 1); @@ -50,19 +51,20 @@ describe('ContextTracker', function() { }); it("properly matches and aligns when lead token + following whitespace are removed", function() { - let existingContext = models.tokenize(defaultBreaker, { + let existingContext = { left: "an apple a day keeps the doctor" - }); + }; let transform = { insert: '', deleteLeft: 0 - } - let newContext = deepCopy(existingContext); - newContext.left.splice(0, 2); + }; + let newContext = { + left: "apple a day keeps the doctor" + }; let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"]; - let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel); - let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform)); + let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel); + let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseContextMatch, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.state); assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); assert.equal(newContextMatch.headTokensRemoved, 2); @@ -70,20 +72,20 @@ describe('ContextTracker', function() { }); it("properly matches and aligns when final token is edited", function() { - let existingContext = models.tokenize(defaultBreaker, { + let existingContext = { left: "an apple a day keeps the docto" - }); + }; let transform = { insert: 'r', deleteLeft: 0 } - let newContext = models.tokenize(defaultBreaker, { + let newContext = { left: "an apple a day keeps the doctor" - }); + }; let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"]; - let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel); - let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform)); + let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel); + let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseContextMatch, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.state); assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); assert.equal(newContextMatch.headTokensRemoved, 0); @@ -92,20 +94,20 @@ describe('ContextTracker', function() { // Needs improved context-state management (due to 2x tokens) it("properly matches and aligns when a 'wordbreak' is added", function() { - let existingContext = models.tokenize(defaultBreaker, { + let existingContext = { left: "an apple a day keeps the doctor" - }); + }; let transform = { insert: ' ', deleteLeft: 0 } - let newContext = models.tokenize(defaultBreaker, { + let newContext = { left: "an apple a day keeps the doctor " - }); + }; let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""]; - let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel); - let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform)); + let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel); + let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseContextMatch, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.state); assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); // We want to preserve the added whitespace when predicting a token that follows after it. @@ -120,20 +122,20 @@ describe('ContextTracker', function() { }); it("properly matches and aligns when a 'wordbreak' is removed via backspace", function() { - let existingContext = models.tokenize(defaultBreaker, { + let existingContext = { left: "an apple a day keeps the doctor " - }); + }; let transform = { insert: '', deleteLeft: 1 } - let newContext = models.tokenize(defaultBreaker, { + let newContext = { left: "an apple a day keeps the doctor" - }); + }; let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"]; - let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel); - let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform)); + let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel); + let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseContextMatch, toWrapperDistribution(transform)); assert.isOk(newContextMatch?.state); assert.deepEqual(newContextMatch?.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); @@ -143,20 +145,20 @@ describe('ContextTracker', function() { }); it("properly matches and aligns when an implied 'wordbreak' occurs (as when following \"'\")", function() { - let existingContext = models.tokenize(defaultBreaker, { + let existingContext = { left: "'" - }); + }; let transform = { insert: 'a', deleteLeft: 0 } - let newContext = models.tokenize(defaultBreaker, { + let newContext = { left: "'a" - }); + }; let rawTokens = ["'", "a"]; - let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel); - let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform)); + let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel); + let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseContextMatch, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.state); assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); assert.deepEqual(newContextMatch.preservationTransform, { insert: '', deleteLeft: 0 }); @@ -172,20 +174,20 @@ describe('ContextTracker', function() { // Needs improved context-state management (due to 2x tokens) it("properly matches and aligns when lead token is removed AND a 'wordbreak' is added'", function() { - let existingContext = models.tokenize(defaultBreaker, { + let existingContext = { left: "an apple a day keeps the doctor" - }); + }; let transform = { insert: ' ', deleteLeft: 0 } - let newContext = models.tokenize(defaultBreaker, { + let newContext = { left: "apple a day keeps the doctor " - }); + }; let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""]; - let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel); - let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform)); + let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel); + let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseContextMatch, toWrapperDistribution(transform)); assert.isNotNull(newContextMatch?.state); assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens); // We want to preserve the added whitespace when predicting a token that follows after it. @@ -201,21 +203,22 @@ describe('ContextTracker', function() { }); it("properly matches and aligns when initial token is modified AND a 'wordbreak' is added'", function() { - let existingContext = models.tokenize(defaultBreaker, { + let existingContext = { left: "an" - }); + }; let transform = { insert: 'd ', deleteLeft: 0 } - let newContext = models.tokenize(defaultBreaker, { + let newContext = { left: "and " - }); + }; let rawTokens = ["and", " ", ""]; - let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel); + let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel); let newContextMatch = ContextTracker.attemptMatchContext( - newContext.left, + newContext, + plainModel, baseContextMatch, tokenizeTransformDistribution(tokenizer, {left: "an"}, [{sample: transform, p: 1}]) ); @@ -234,21 +237,22 @@ describe('ContextTracker', function() { }); it("properly matches and aligns when tail token is modified AND a 'wordbreak' is added'", function() { - let existingContext = models.tokenize(defaultBreaker, { + let existingContext = { left: "apple a day keeps the doc" - }); + }; let transform = { insert: 'tor ', deleteLeft: 0 } - let newContext = models.tokenize(defaultBreaker, { + let newContext = { left: "apple a day keeps the doctor " - }); + }; let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""]; - let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel); + let baseContextMatch = ContextTracker.modelContextState(existingContext, plainModel); let newContextMatch = ContextTracker.attemptMatchContext( - newContext.left, + newContext, + plainModel, baseContextMatch, tokenizeTransformDistribution(tokenizer, {left: "apple a day keeps the doc"}, [{sample: transform, p: 1}]) ); @@ -271,7 +275,7 @@ describe('ContextTracker', function() { left: "text'" }); assert.equal(baseContext.left.length, 1); - let baseContextMatch = ContextTracker.modelContextState(baseContext.left, plainModel); + let baseContextMatch = ContextTracker.modelContextState({left: "text'"}, plainModel); // Now the actual check. let newContext = models.tokenize(defaultBreaker, { @@ -286,7 +290,8 @@ describe('ContextTracker', function() { deleteLeft: 0 } let problemContextMatch = ContextTracker.attemptMatchContext( - newContext.left, + {left: "text'\""}, + plainModel, baseContextMatch, tokenizeTransformDistribution(tokenizer, {left: "text'"}, [{sample: transform, p: 1}]) ); @@ -296,28 +301,18 @@ describe('ContextTracker', function() { describe('modelContextState', function() { it('models without final wordbreak', function() { - let tokenized = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"].map((entry) => { - return { - text: entry, - isWhitespace: entry == " " - }; - }); + let context = { left: "an apple a day keeps the doctor" }; let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"]; - let state = ContextTracker.modelContextState(tokenized, plainModel); + let state = ContextTracker.modelContextState(context, plainModel); assert.deepEqual(state.tokenization.tokens.map(token => token.exampleInput), rawTokens); }); it('models with final wordbreak', function() { - let tokenized = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""].map((entry) => { - return { - text: entry, - isWhitespace: entry == " " - }; - }); + let context = { left: "an apple a day keeps the doctor " }; let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""]; - let state = ContextTracker.modelContextState(tokenized, plainModel); + let state = ContextTracker.modelContextState(context, plainModel); assert.deepEqual(state.tokenization.tokens.map(token => token.exampleInput), rawTokens); }); });