From 012dcaeee911e943094b0209f4ecb04609400d94 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Fri, 12 Sep 2025 15:57:32 -0500 Subject: [PATCH 1/4] refactor(web): differentiate between true and represented inputs for context tokens To be up-front, this is looking ahead significantly to epic/dict-breaker (#12142) concerns. The core idea here: each token should mark which part of the current context it represents - and the best data for the job happens to be the original, true transforms. With epic/dict-breaker and other possible work that would faciltate considering alternative tokenizations, it is quite possible the original source transform(s) for a tokenization may not actually be represented within that tokenization's input set. Consider: if the spacebar appears as a fat-finger alternate for a letter key, we may wish to model an alternate tokenization where we actually _used_ that keystroke as a whitespace instead of a standard letter. The resulting whitespace token shouldn't have the original letter key's transform as part of its represented inputs for such a case. Build-bot: skip build:web Test-bot: skip --- .../src/main/correction/context-state.ts | 2 +- .../src/main/correction/context-token.ts | 55 ++++++++++++++++++- .../main/correction/context-tokenization.ts | 33 ++++++++--- .../worker-thread/src/main/predict-helpers.ts | 2 +- 4 files changed, 80 insertions(+), 12 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts index 305f2319a1..01c3c50536 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts @@ -255,7 +255,7 @@ export class ContextState { const tokens = resultTokenization.tokens; const lastIndex = tokens.length - 1; // Ignore a context-final empty '' token; the interesting one is what comes before. - const nonEmptyTail = tokens[lastIndex].exampleInput != '' ? tokens[lastIndex] : tokens[lastIndex - 1]; + const nonEmptyTail = tokens[lastIndex].sourceText != '' ? tokens[lastIndex] : tokens[lastIndex - 1]; const appliedSuggestionTransitionId = nonEmptyTail?.appliedTransitionId; // Used to construct and represent the part of the incoming transform that diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts index 4734f5bad9..c87c706fd3 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts @@ -58,6 +58,13 @@ export class ContextToken { */ appliedTransitionId?: number; + /** + * Represents the original, 'true' input transforms (tokenized, as necessary) + * applied to the actual context for the set of keystrokes contributing to + * this token. + */ + private _inputRange: Transform[]; + /** * Constructs a new, empty instance for use with the specified LexicalModel. * @param model @@ -84,6 +91,7 @@ export class ContextToken { // In case we are unable to perfectly track context (say, due to multitaps) // we need to ensure that only fully-utilized keystrokes are considered. this.searchSpace = new SearchSpace(priorToken.searchSpace); + this._inputRange = priorToken._inputRange.slice(); // Preserve any annotated applied-suggestion transition ID data; it's useful // for delayed reversion operations. @@ -96,6 +104,7 @@ export class ContextToken { // May be altered outside of the constructor. this.isWhitespace = false; this.searchSpace = new SearchSpace(model); + this._inputRange = []; rawText ||= ''; @@ -103,15 +112,55 @@ export class ContextToken { const rawTransformDistributions: Distribution[] = textToCharTransforms(rawText).map(function(transform) { return [{sample: transform, p: 1.0}]; }); - rawTransformDistributions.forEach((entry) => this.searchSpace.addInput(entry)); + rawTransformDistributions.forEach((entry) => { + this._inputRange.push(entry[0].sample); + this.searchSpace.addInput(entry); + }); } } /** - * Displays text corresponding to the net effects of the most likely inputs received - * that can correspond to the current instance. + * Call this to record the original keystroke Transforms for the context range + * corresponding to this token. + */ + addSourceInput(transform: Transform) { + this._inputRange.push(transform); + } + + /** + * Denotes the original keystroke Transforms comprising the range corresponding + * to this token. + */ + get inputRange(): Readonly { + return this._inputRange; + } + + /** + * Gets a simple, human-readable representation of `inputRange`. + * + * Should not actually be used in code - its use is intended only for + * debugging. + */ + get sourceText(): string { + const composite = this._inputRange.reduce((accum, current) => buildMergedTransform(accum, current), { insert: '', deleteLeft: 0 }); + const prefix = '\u{2421}'.repeat(composite.deleteLeft); + return prefix + composite.insert; + } + + /** + * Generates text corresponding to the net effects of the most likely inputs + * received that can correspond to the current instance. */ get exampleInput(): string { + /* + * TODO: with clear limits (strict cost minimization?) / prior calculation + * attempts, return the best _suggestion_ for this token. This is + * especially relevant for epic/dict-breaker - we want to best model the token + * as it would apply within the word-breaking algorithm. + * + * If not possible, find the best of the deepest search paths and append the + * most likely keystroke data afterward. + */ const transforms = this.searchSpace.inputSequence.map((dist) => dist[0].sample) const composite = transforms.reduce((accum, current) => buildMergedTransform(accum, current), { insert: '', deleteLeft: 0}); return composite.insert; diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index fa4cac89a7..4b19955836 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -7,16 +7,17 @@ * the sliding context window for one specific instance of context state. */ -import { ContextToken } from './context-token.js'; -import { computeAlignment, ContextStateAlignment } from './alignment-helpers.js'; import { Token } from '@keymanapp/models-templates'; - import { LexicalModelTypes } from '@keymanapp/common-types'; + +import { ContextToken } from './context-token.js'; +import TransformUtils from '../transformUtils.js'; +import { computeAlignment, ContextStateAlignment } from './alignment-helpers.js'; + import Distribution = LexicalModelTypes.Distribution; import LexicalModel = LexicalModelTypes.LexicalModel; import ProbabilityMass = LexicalModelTypes.ProbabilityMass; import Transform = LexicalModelTypes.Transform; -import TransformUtils from '../transformUtils.js'; /** * This class represents the sequence of tokens (words and whitespace blocks) @@ -47,6 +48,16 @@ export class ContextTokenization { return this.tokens[this.tokens.length - 1]; } + /** + * Returns plain-text strings representing the most probable representation for all + * tokens represented by this tokenization instance. + */ + get sourceText() { + return this.tokens + .filter(token => token.sourceText !== null) + .map(token => token.sourceText); + } + /** * Returns a plain-text string representing the most probable representation for all * tokens represented by this tokenization instance. @@ -69,7 +80,7 @@ export class ContextTokenization { * the tokenization modeled by this instance. */ computeAlignment(incomingTokenization: string[], isSliding: boolean, noSubVerify?: boolean): ContextStateAlignment { - return computeAlignment(this.exampleInput, incomingTokenization, isSliding, noSubVerify); + return computeAlignment(this.sourceText, incomingTokenization, isSliding, noSubVerify); } /** @@ -143,7 +154,7 @@ export class ContextTokenization { // If a word is being slid out of context-window range, start trimming it - we should // no longer need to worry about reusing its original correction-search results. for(let i = 0; i < leadEditLength; i++) { - if(this.tokens[matchingOffset+i].exampleInput != tokenizedContext[incomingOffset+i].text) { + if(this.tokens[matchingOffset+i].sourceText != tokenizedContext[incomingOffset+i].text) { //this.tokens[matchingOffset]'s clone is at tokenization[incomingOffset] //after the splice call in a previous block. tokenization[incomingOffset+i] = new ContextToken(lexicalModel, tokenizedContext[incomingOffset+i].text); @@ -162,6 +173,10 @@ export class ContextTokenization { // The assumed input from the input distribution is always at index 0. const tokenizedPrimaryInput = hasDistribution ? alignedTransformDistribution[0].sample : null; + + // now that we've identified the 'primary input', sort the distributions. + alignedTransformDistribution.sort((a, b) => b.p - a.p); + // first index: original sample's tokenization // second index: token index within original sample const tokenDistribution = alignedTransformDistribution.map((entry) => { @@ -203,9 +218,12 @@ export class ContextTokenization { // Assumption: there have been no intervening keystrokes since the last well-aligned context. // (May not be valid with epic/dict-breaker or with complex, word-boundary crossing transforms) token = new ContextToken(matchedToken); + // Erase any applied-suggestion transition ID; it is no longer valid. token.appliedTransitionId = undefined; - token.searchSpace.addInput(tokenDistribution.map((seq) => seq.get(tailIndex) ?? { sample: { insert: '', deleteLeft: 0 }, p: 1 })); + const emptySample: ProbabilityMass = { sample: { insert: '', deleteLeft: 0 }, p: 1 }; + token.addSourceInput(primaryInput ?? emptySample.sample); + token.searchSpace.addInput(tokenDistribution.map((seq) => seq.get(tailIndex) ?? emptySample)); } tokenization[incomingIndex] = token; @@ -265,6 +283,7 @@ export class ContextTokenization { // If there are no entries in our would-be distribution, there's no // reason to pass in what amounts to a no-op. if(transformDistribution) { + pushedToken.addSourceInput(primaryInput); // If we ever stop filtering tokenized transform distributions, it may // be worth adding an empty transform here with weight to balance // the distribution back to a cumulative prob sum of 1. diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 44a03bc410..08774c218c 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -394,7 +394,7 @@ export function determineSuggestionAlignment( // Did the wordbreaker (or similar) append a blank token before the caret? If so, // preserve that by preventing corrections from triggering left-deletion. - if(transition.final.tokenization.tail.exampleInput == '') { + if(transition.final.tokenization.tail.sourceText == '') { deleteLeft = 0; } From 2b64a0857bc5a4939895754d07367d5b29fefde3 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Wed, 24 Sep 2025 10:37:12 -0500 Subject: [PATCH 2/4] docs(web): fix sourceText doc comment --- .../worker-thread/src/main/correction/context-token.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts index c87c706fd3..23f438ca17 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts @@ -136,10 +136,7 @@ export class ContextToken { } /** - * Gets a simple, human-readable representation of `inputRange`. - * - * Should not actually be used in code - its use is intended only for - * debugging. + * Gets a simple, compact string-based representation of `inputRange`. */ get sourceText(): string { const composite = this._inputRange.reduce((accum, current) => buildMergedTransform(accum, current), { insert: '', deleteLeft: 0 }); From 5dc28b54921ea1b9653fac884f48d4d155610ba8 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 25 Sep 2025 14:51:57 -0500 Subject: [PATCH 3/4] refactor(web): track original ContextToken source properties + reform use of .exampleText --- .../src/main/correction/context-state.ts | 2 +- .../src/main/correction/context-token.ts | 54 ++++++++++++++++--- .../main/correction/context-tokenization.ts | 19 ++++--- .../worker-thread/src/main/predict-helpers.ts | 2 +- 4 files changed, 60 insertions(+), 17 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts index 01c3c50536..0f733391fe 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts @@ -255,7 +255,7 @@ export class ContextState { const tokens = resultTokenization.tokens; const lastIndex = tokens.length - 1; // Ignore a context-final empty '' token; the interesting one is what comes before. - const nonEmptyTail = tokens[lastIndex].sourceText != '' ? tokens[lastIndex] : tokens[lastIndex - 1]; + const nonEmptyTail = !tokens[lastIndex].isEmptyToken ? tokens[lastIndex] : tokens[lastIndex - 1]; const appliedSuggestionTransitionId = nonEmptyTail?.appliedTransitionId; // Used to construct and represent the part of the incoming transform that diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts index 23f438ca17..30409533ff 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts @@ -16,6 +16,14 @@ import Distribution = LexicalModelTypes.Distribution; import LexicalModel = LexicalModelTypes.LexicalModel; import Transform = LexicalModelTypes.Transform; +/** + * Notes critical properties of the inputs comprising each ContextToken. + */ +export interface TokenInputSource { + trueTransform: Transform; + inputStartIndex: number; +} + /** * Breaks apart a raw text string into individual, single-codepoint * transforms, all set with the specified transform ID. @@ -63,7 +71,7 @@ export class ContextToken { * applied to the actual context for the set of keystrokes contributing to * this token. */ - private _inputRange: Transform[]; + private _inputRange: TokenInputSource[]; /** * Constructs a new, empty instance for use with the specified LexicalModel. @@ -113,7 +121,10 @@ export class ContextToken { return [{sample: transform, p: 1.0}]; }); rawTransformDistributions.forEach((entry) => { - this._inputRange.push(entry[0].sample); + this._inputRange.push({ + trueTransform: entry[0].sample, + inputStartIndex: 0 + }); this.searchSpace.addInput(entry); }); } @@ -123,23 +134,52 @@ export class ContextToken { * Call this to record the original keystroke Transforms for the context range * corresponding to this token. */ - addSourceInput(transform: Transform) { - this._inputRange.push(transform); + addInput(inputSource: TokenInputSource, distribution: Distribution) { + this._inputRange.push(inputSource); + this.searchSpace.addInput(distribution); } /** * Denotes the original keystroke Transforms comprising the range corresponding * to this token. */ - get inputRange(): Readonly { + get inputRange(): Readonly { return this._inputRange; } + /** + * Indicates whether or not this ContextToken likely represents an empty token. + */ + get isEmptyToken(): boolean { + return this.exampleInput == ''; + } + + /** + * Gets a compact string-based representation of `inputRange` that + * maps compatible token source ranges to each other. + */ + get sourceRangeKey(): string { + const components: string[] = []; + + for(const source of this.inputRange) { + const i = source.inputStartIndex; + components.push(`T${source.trueTransform.id}${i != 0 ? '@' + i : ''}`); + } + + return components.join('+'); + } + /** * Gets a simple, compact string-based representation of `inputRange`. + * + * This should only ever be used for debugging purposes. */ get sourceText(): string { - const composite = this._inputRange.reduce((accum, current) => buildMergedTransform(accum, current), { insert: '', deleteLeft: 0 }); + const composite = this._inputRange.reduce((accum, current) => { + const alteredTransform = {...current.trueTransform}; + alteredTransform.insert = alteredTransform.insert.slice(current.inputStartIndex); + return buildMergedTransform(accum, current.trueTransform) + }, { insert: '', deleteLeft: 0 }); const prefix = '\u{2421}'.repeat(composite.deleteLeft); return prefix + composite.insert; } @@ -159,7 +199,7 @@ export class ContextToken { * most likely keystroke data afterward. */ const transforms = this.searchSpace.inputSequence.map((dist) => dist[0].sample) - const composite = transforms.reduce((accum, current) => buildMergedTransform(accum, current), { insert: '', deleteLeft: 0}); + const composite = transforms.reduce((accum, current) => buildMergedTransform(accum, current), {insert: '', deleteLeft: 0}); return composite.insert; } } \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 4b19955836..70f541c0a6 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -9,6 +9,7 @@ import { Token } from '@keymanapp/models-templates'; import { LexicalModelTypes } from '@keymanapp/common-types'; +import { KMWString } from '@keymanapp/web-utils'; import { ContextToken } from './context-token.js'; import TransformUtils from '../transformUtils.js'; @@ -51,11 +52,11 @@ export class ContextTokenization { /** * Returns plain-text strings representing the most probable representation for all * tokens represented by this tokenization instance. + * + * Intended for debugging use only. */ get sourceText() { - return this.tokens - .filter(token => token.sourceText !== null) - .map(token => token.sourceText); + return this.tokens.map(token => token.sourceText); } /** @@ -154,7 +155,7 @@ export class ContextTokenization { // If a word is being slid out of context-window range, start trimming it - we should // no longer need to worry about reusing its original correction-search results. for(let i = 0; i < leadEditLength; i++) { - if(this.tokens[matchingOffset+i].sourceText != tokenizedContext[incomingOffset+i].text) { + if(this.tokens[matchingOffset+i].exampleInput != tokenizedContext[incomingOffset+i].text) { //this.tokens[matchingOffset]'s clone is at tokenization[incomingOffset] //after the splice call in a previous block. tokenization[incomingOffset+i] = new ContextToken(lexicalModel, tokenizedContext[incomingOffset+i].text); @@ -197,6 +198,7 @@ export class ContextTokenization { // edited, those edits occur to the left as well - and further left of whatever // the new tail token is *if* tokens were removed. const firstTailEditIndex = Math.min((1 - tailEditLength), 0) + Math.min(tailTokenShift, 0); + let primaryInputAppliedLen = 0; for(let i = 0; i < tailEditLength; i++) { const tailIndex = firstTailEditIndex + i; @@ -222,11 +224,12 @@ export class ContextTokenization { // Erase any applied-suggestion transition ID; it is no longer valid. token.appliedTransitionId = undefined; const emptySample: ProbabilityMass = { sample: { insert: '', deleteLeft: 0 }, p: 1 }; - token.addSourceInput(primaryInput ?? emptySample.sample); - token.searchSpace.addInput(tokenDistribution.map((seq) => seq.get(tailIndex) ?? emptySample)); + const dist = tokenDistribution.map((seq) => seq.get(tailIndex) ?? emptySample); + token.addInput({trueTransform: primaryInput ?? emptySample.sample, inputStartIndex: primaryInputAppliedLen}, dist); } tokenization[incomingIndex] = token; + primaryInputAppliedLen += KMWString.length(primaryInput?.insert ?? ''); } if(tailTokenShift < 0) { @@ -283,11 +286,10 @@ export class ContextTokenization { // If there are no entries in our would-be distribution, there's no // reason to pass in what amounts to a no-op. if(transformDistribution) { - pushedToken.addSourceInput(primaryInput); // If we ever stop filtering tokenized transform distributions, it may // be worth adding an empty transform here with weight to balance // the distribution back to a cumulative prob sum of 1. - pushedToken.searchSpace.addInput(transformDistribution); + pushedToken.addInput({ trueTransform: primaryInput, inputStartIndex: primaryInputAppliedLen }, transformDistribution); } } else if(incomingToken.text) { // We have no transform data to match against an inserted token with text; abort! @@ -299,6 +301,7 @@ export class ContextTokenization { // Auto-replaces the search space to correspond with the new token. tokenization.push(pushedToken); + primaryInputAppliedLen += KMWString.length(primaryInput.insert); } } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 08774c218c..2252549b16 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -394,7 +394,7 @@ export function determineSuggestionAlignment( // Did the wordbreaker (or similar) append a blank token before the caret? If so, // preserve that by preventing corrections from triggering left-deletion. - if(transition.final.tokenization.tail.sourceText == '') { + if(transition.final.tokenization.tail.isEmptyToken) { deleteLeft = 0; } From 6eedb2e7abcc101a31e1ab811a2c82db9cafaf83 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 25 Sep 2025 14:59:49 -0500 Subject: [PATCH 4/4] change(web): revert no-longer-valid change --- .../src/main/correction/context-tokenization.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 70f541c0a6..164da8ef36 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -64,10 +64,7 @@ export class ContextTokenization { * tokens represented by this tokenization instance. */ get exampleInput(): string[] { - return this.tokens - // Hide any tokens representing invisible wordbreaks. (Thinking ahead to phrase-level possibilities) - .filter(token => token.exampleInput !== null) - .map(token => token.exampleInput); + return this.tokens.map(token => token.exampleInput); } /** @@ -81,7 +78,7 @@ export class ContextTokenization { * the tokenization modeled by this instance. */ computeAlignment(incomingTokenization: string[], isSliding: boolean, noSubVerify?: boolean): ContextStateAlignment { - return computeAlignment(this.sourceText, incomingTokenization, isSliding, noSubVerify); + return computeAlignment(this.exampleInput, incomingTokenization, isSliding, noSubVerify); } /**