From 92051cdc43a090d11aeea9f72d36fef3eaa06dfe Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 23 Oct 2025 16:34:53 -0500 Subject: [PATCH] refactor(web): manage true-input data from SearchQuotientSpur Once we start considering alternate tokenization schemes, we'll want to note how each potential input path aligns with the original input keystrokes. As there can be multiple paths to land within the same token, it's best to store this data on the SearchQuotientSpur objects instead. This becomes especially relevant when considering token splits and merges, which will be the next follow-ups. Build-bot: skip build:web Test-bot: skip --- .../src/main/correction/context-token.ts | 52 +++++++------------ .../main/correction/context-tokenization.ts | 2 +- .../main/correction/legacy-quotient-spur.ts | 6 +-- .../main/correction/search-quotient-node.ts | 15 ++++++ .../main/correction/search-quotient-root.ts | 6 ++- .../main/correction/search-quotient-spur.ts | 30 +++++++++-- .../correction-search/getBestMatches.tests.ts | 12 ++--- .../search-quotient-spur.tests.ts | 18 +++---- 8 files changed, 83 insertions(+), 58 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts index de39a5f8cf..408a7e19ac 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts @@ -11,7 +11,7 @@ import { applyTransform, buildMergedTransform } from "@keymanapp/models-template import { LexicalModelTypes } from '@keymanapp/common-types'; import { deepCopy, KMWString } from "@keymanapp/web-utils"; -import { SearchQuotientNode } from "./search-quotient-node.js"; +import { SearchQuotientNode, TokenInputSource } from "./search-quotient-node.js"; import { TokenSplitMap } from "./context-tokenization.js"; import { LegacyQuotientSpur } from "./legacy-quotient-spur.js"; import { LegacyQuotientRoot } from "./legacy-quotient-root.js"; @@ -20,15 +20,6 @@ import Distribution = LexicalModelTypes.Distribution; import LexicalModel = LexicalModelTypes.LexicalModel; import Transform = LexicalModelTypes.Transform; -/** - * Notes critical properties of the inputs comprising each ContextToken. - */ -export interface TokenInputSource { - trueTransform: Transform; - inputStartIndex: number; - bestProbFromSet: number; -} - /** * Breaks apart a raw text string into individual, single-codepoint * transforms, all set with the specified transform ID. @@ -76,13 +67,6 @@ export class ContextToken { */ appliedTransitionId?: number; - /** - * Represents the original, 'true' input transforms (tokenized, as necessary) - * applied to the actual context for the set of keystrokes contributing to - * this token. - */ - private _inputRange: TokenInputSource[]; - /** * Constructs a new, empty instance for use with the specified LexicalModel. * @param model @@ -109,14 +93,12 @@ export class ContextToken { // In case we are unable to perfectly track context (say, due to multitaps) // we need to ensure that only fully-utilized keystrokes are considered. this._searchModule = priorToken.searchModule; - this._inputRange = priorToken._inputRange.slice(); } else { const model = param; // May be altered outside of the constructor. this.isWhitespace = false; this.isPartial = !!isPartial; - this._inputRange = []; rawText ||= ''; @@ -125,12 +107,12 @@ export class ContextToken { let searchModule: SearchQuotientNode = new LegacyQuotientRoot(model); const BASE_PROBABILITY = 1; textToCharTransforms(rawText).forEach((transform) => { - this._inputRange.push({ + let inputMetadata: TokenInputSource = { trueTransform: transform, inputStartIndex: 0, bestProbFromSet: BASE_PROBABILITY - }); - searchModule = new LegacyQuotientSpur(searchModule, [{sample: transform, p: BASE_PROBABILITY}], 1); + }; + searchModule = new LegacyQuotientSpur(searchModule, [{sample: transform, p: BASE_PROBABILITY}], inputMetadata); }); this._searchModule = searchModule; @@ -142,16 +124,11 @@ export class ContextToken { * corresponding to this token. */ addInput(inputSource: TokenInputSource, distribution: Distribution) { - this._inputRange.push(inputSource); - this._searchModule = new LegacyQuotientSpur(this._searchModule, distribution, inputSource.bestProbFromSet); + this._searchModule = new LegacyQuotientSpur(this._searchModule, distribution, inputSource); } - /** - * Denotes the original keystroke Transforms comprising the range corresponding - * to this token. - */ - get inputRange(): Readonly { - return this._inputRange; + get inputCount() { + return this._searchModule.inputCount; } /** @@ -161,6 +138,14 @@ export class ContextToken { return this.exampleInput == ''; } + /** + * Denotes the original keystroke Transforms comprising the range corresponding + * to this token. + */ + get inputRange() { + return this.searchModule.sourceIdentifiers; + } + /** * Gets the unique identifier that may be used to match this ContextToken with * a correction-search result. @@ -175,8 +160,9 @@ export class ContextToken { */ get sourceRangeKey(): string { const components: string[] = []; + const sources = this.searchModule.sourceIdentifiers; - for(const source of this.inputRange) { + for(const source of sources) { const i = source.inputStartIndex; components.push(`T${source.trueTransform.id}${i != 0 ? '@' + i : ''}`); } @@ -207,10 +193,10 @@ export class ContextToken { let lastSourceInput: TokenInputSource; let lastInputDistrib: Distribution; for(const token of tokensToMerge) { - const inputCount = token.inputRange.length; + const inputCount = token.inputCount; let startIndex = 0; - if(token.inputRange.length == 0) { + if(inputCount == 0) { continue; } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 01c47e0039..72f0f53d24 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -582,7 +582,7 @@ export class ContextTokenization { // If we are completely replacing a token via delete left, erase the deleteLeft; // that part applied to a _previous_ token that no longer exists. // We start at index 0 in the insert string for the "new" token. - if(affectedToken.inputRange.length == 0 && distribution[0].sample.deleteLeft != 0) { + if(affectedToken.inputCount == 0 && distribution[0].sample.deleteLeft != 0) { distribution = distribution.map((mass) => ({sample: { ...mass.sample, deleteLeft: 0 }, p: mass.p })); } affectedToken.addInput({trueTransform: sourceInput, inputStartIndex: appliedLength, bestProbFromSet}, distribution); diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts index 9fd218fa55..efbaeed137 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts @@ -11,7 +11,7 @@ import { LexicalModelTypes } from '@keymanapp/common-types'; import { SearchNode } from './distance-modeler.js'; -import { PathResult, SearchQuotientNode } from './search-quotient-node.js'; +import { PathResult, SearchQuotientNode, TokenInputSource } from './search-quotient-node.js'; import { SearchQuotientSpur } from './search-quotient-spur.js'; import Distribution = LexicalModelTypes.Distribution; @@ -27,8 +27,8 @@ export class LegacyQuotientSpur extends SearchQuotientSpur { * @param inputs * @param bestProbFromSet */ - constructor(space: SearchQuotientNode, inputs: Distribution, bestProbFromSet: number) { - super(space, inputs, space.lowestPossibleSingleCost - Math.log(bestProbFromSet)); + constructor(space: SearchQuotientNode, inputs: Distribution, inputSource: TokenInputSource) { + super(space, inputs, inputSource); this.queueNodes(this.buildEdgesForNodes(space.previousResults.map(r => r.node))); return; } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-node.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-node.ts index 2335d6f31f..72001c05ef 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-node.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-node.ts @@ -39,6 +39,15 @@ type CompleteSearchPath = { export type PathResult = NullPath | IntermediateSearchPath | CompleteSearchPath; +/** + * Notes critical properties of the inputs comprising each ContextToken. + */ +export interface TokenInputSource { + trueTransform: Transform; + inputStartIndex: number; + bestProbFromSet: number; +} + /** * Represents all or a portion of the dynamically-generated graph used to search * for predictive-text corrections. @@ -121,6 +130,12 @@ export interface SearchQuotientNode { * the correction-search graph and its paths. */ readonly bestExample: { text: string, p: number }; + + /** + * Gets components useful for building a string-based representation of the + * keystroke range corrected by this search space. + */ + readonly sourceIdentifiers: TokenInputSource[]; } /** diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-root.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-root.ts index fd23301a8f..7bef9bfae6 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-root.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-root.ts @@ -2,7 +2,7 @@ import { LexicalModelTypes } from '@keymanapp/common-types'; import { SearchNode, SearchResult } from './distance-modeler.js'; -import { generateSpaceSeed, PathResult, SearchQuotientNode } from './search-quotient-node.js'; +import { generateSpaceSeed, PathResult, SearchQuotientNode, TokenInputSource } from './search-quotient-node.js'; import LexicalModel = LexicalModelTypes.LexicalModel; @@ -89,4 +89,8 @@ export class SearchQuotientRoot implements SearchQuotientNode { return [this.rootResult]; } } + + get sourceIdentifiers(): TokenInputSource[] { + return []; + } } \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts index 9cba05fedb..dc3c9c976d 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts @@ -12,7 +12,7 @@ import { QueueComparator as Comparator, KMWString, PriorityQueue } from '@keyman import { LexicalModelTypes } from '@keymanapp/common-types'; import { EDIT_DISTANCE_COST_SCALE, SearchNode, SearchResult } from './distance-modeler.js'; -import { generateSpaceSeed, PathResult, SearchQuotientNode } from './search-quotient-node.js'; +import { generateSpaceSeed, PathResult, SearchQuotientNode, TokenInputSource } from './search-quotient-node.js'; import Distribution = LexicalModelTypes.Distribution; import Transform = LexicalModelTypes.Transform; @@ -25,7 +25,8 @@ export const QUEUE_NODE_COMPARATOR: Comparator = function(arg1, arg2 // Whenever a wordbreak boundary is crossed, a new instance should be made. export abstract class SearchQuotientSpur implements SearchQuotientNode { private selectionQueue: PriorityQueue = new PriorityQueue(QUEUE_NODE_COMPARATOR); - readonly inputs?: Distribution>; + readonly inputs?: Distribution; + readonly inputSource?: TokenInputSource; private parentNode: SearchQuotientNode; readonly spaceId: number; @@ -50,13 +51,14 @@ export abstract class SearchQuotientSpur implements SearchQuotientNode { * * @param parentNode * @param inputs - * @param costHeuristic + * @param inputSource */ - constructor(parentNode: SearchQuotientNode, inputs: Distribution, costHeuristic: number) { + constructor(parentNode: SearchQuotientNode, inputs: Distribution>, inputSource: TokenInputSource) { this.spaceId = generateSpaceSeed(); this.parentNode = parentNode; - this.lowestPossibleSingleCost = (parentNode?.lowestPossibleSingleCost ?? 0) - Math.log(costHeuristic); + this.inputSource = inputSource; + this.lowestPossibleSingleCost = (parentNode?.lowestPossibleSingleCost ?? 0) - Math.log(inputSource?.bestProbFromSet ?? 1); this.inputs = inputs?.length > 0 ? inputs : null; this.inputCount = (parentNode?.inputCount ?? 0) + (this.inputs ? 1 : 0); } @@ -221,4 +223,22 @@ export abstract class SearchQuotientSpur implements SearchQuotientNode { public get previousResults(): SearchResult[] { return Object.values(this.returnedValues ?? {}).map(v => new SearchResult(v)); } + + public get sourceIdentifiers(): TokenInputSource[] { + if(!this.parentNode) { + return []; + } + + const parentSources = this.parentNode.sourceIdentifiers; + if(this.inputSource) { + const inputId = this.inputSource.trueTransform.id; + if(inputId && parentSources.length > 0 && parentSources[parentSources.length - 1].trueTransform.id == inputId) { + return parentSources; + } + + parentSources.push(this.inputSource); + } + + return parentSources; + } } \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/getBestMatches.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/getBestMatches.tests.ts index 5e64aa309c..6346fd5d7f 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/getBestMatches.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/getBestMatches.tests.ts @@ -146,9 +146,9 @@ describe('getBestMatches', () => { {sample: {insert: 'n', deleteLeft: 0}, p: 0.25} ]; - const searchPath1 = new LegacyQuotientSpur(searchPath, synthInput1, 1); - const searchPath2 = new LegacyQuotientSpur(searchPath1, synthInput2, .75); - const searchPath3 = new LegacyQuotientSpur(searchPath2, synthInput3, .75); + const searchPath1 = new LegacyQuotientSpur(searchPath, synthInput1, {trueTransform: synthInput1[0].sample, inputStartIndex: 0, bestProbFromSet: 1}); + const searchPath2 = new LegacyQuotientSpur(searchPath1, synthInput2, {trueTransform: synthInput2[0].sample, inputStartIndex: 0, bestProbFromSet: .75}); + const searchPath3 = new LegacyQuotientSpur(searchPath2, synthInput3, {trueTransform: synthInput3[0].sample, inputStartIndex: 0, bestProbFromSet: .75}); assert.notEqual(searchPath1.spaceId, searchPath.spaceId); assert.notEqual(searchPath2.spaceId, searchPath1.spaceId); @@ -180,9 +180,9 @@ describe('getBestMatches', () => { {sample: {insert: 'n', deleteLeft: 0}, p: 0.25} ]; - const searchPath1 = new LegacyQuotientSpur(searchPath, synthInput1, 1); - const searchPath2 = new LegacyQuotientSpur(searchPath1, synthInput2, .75); - const searchPath3 = new LegacyQuotientSpur(searchPath2, synthInput3, .75); + const searchPath1 = new LegacyQuotientSpur(searchPath, synthInput1, {trueTransform: synthInput1[0].sample, inputStartIndex: 0, bestProbFromSet: 1}); + const searchPath2 = new LegacyQuotientSpur(searchPath1, synthInput2, {trueTransform: synthInput2[0].sample, inputStartIndex: 0, bestProbFromSet: .75}); + const searchPath3 = new LegacyQuotientSpur(searchPath2, synthInput3, {trueTransform: synthInput3[0].sample, inputStartIndex: 0, bestProbFromSet: .75}); assert.notEqual(searchPath1.spaceId, searchPath.spaceId); assert.notEqual(searchPath2.spaceId, searchPath1.spaceId); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-spur.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-spur.tests.ts index 1d9ee2af9a..b122bb4d02 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-spur.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-spur.tests.ts @@ -24,24 +24,24 @@ export function buildSimplePathSplitFixture() { { sample: {insert: 'r', deleteLeft: 0, id: 11}, p: 0.4 }, { sample: {insert: 't', deleteLeft: 0, id: 11}, p: 0.1 } ]; - const path1 = new LegacyQuotientSpur(rootPath, distrib1, distrib1[0].p); + const path1 = new LegacyQuotientSpur(rootPath, distrib1, {trueTransform: distrib1[0].sample, inputStartIndex: 0, bestProbFromSet: distrib1[0].p}); const distrib2 = [ { sample: {insert: 'a', deleteLeft: 0, id: 12}, p: 0.7 }, { sample: {insert: 'e', deleteLeft: 0, id: 12}, p: 0.3 } ]; - const path2 = new LegacyQuotientSpur(path1, distrib2, distrib2[0].p); + const path2 = new LegacyQuotientSpur(path1, distrib2, {trueTransform: distrib2[0].sample, inputStartIndex: 0, bestProbFromSet: distrib2[0].p}); const distrib3 = [ { sample: {insert: 'n', deleteLeft: 0, id: 13}, p: 0.8 }, { sample: {insert: 'r', deleteLeft: 0, id: 13}, p: 0.2 } ]; - const path3 = new LegacyQuotientSpur(path2, distrib3, distrib3[0].p); + const path3 = new LegacyQuotientSpur(path2, distrib3, {trueTransform: distrib3[0].sample, inputStartIndex: 0, bestProbFromSet: distrib3[0].p}); const distrib4 = [ { sample: {insert: 't', deleteLeft: 0, id: 14}, p: 1 } ]; - const path4 = new LegacyQuotientSpur(path3, distrib4, distrib4[0].p); + const path4 = new LegacyQuotientSpur(path3, distrib4, {trueTransform: distrib4[0].sample, inputStartIndex: 0, bestProbFromSet: distrib4[0].p}); return { paths: [rootPath, path1, path2, path3, path4], @@ -68,7 +68,7 @@ describe('SearchQuotientSpur', () => { {sample: {insert: 'o', deleteLeft: 0, id: 13 }, p: 0.2} ]; - const extendedPath = new LegacyQuotientSpur(rootPath, leadEdgeDistribution, leadEdgeDistribution[0].p); + const extendedPath = new LegacyQuotientSpur(rootPath, leadEdgeDistribution, {trueTransform: leadEdgeDistribution[0].sample, inputStartIndex: 0, bestProbFromSet: leadEdgeDistribution[0].p}); assert.equal(extendedPath.inputCount, 1); assert.isNumber(extendedPath.spaceId); @@ -97,7 +97,7 @@ describe('SearchQuotientSpur', () => { const length1Path = new LegacyQuotientSpur( rootPath, leadEdgeDistribution, - leadEdgeDistribution[0].p + {trueTransform: leadEdgeDistribution[0].sample, inputStartIndex: 0, bestProbFromSet: leadEdgeDistribution[0].p} ); const tailEdgeDistribution = [ @@ -109,7 +109,7 @@ describe('SearchQuotientSpur', () => { const length2Path = new LegacyQuotientSpur( length1Path, tailEdgeDistribution, - tailEdgeDistribution[0].p + {trueTransform: tailEdgeDistribution[0].sample, inputStartIndex: 0, bestProbFromSet: tailEdgeDistribution[0].p} ); // Verify that the prior distribution remains fully unaltered. @@ -143,7 +143,7 @@ describe('SearchQuotientSpur', () => { const length1Path = new LegacyQuotientSpur( rootPath, leadEdgeDistribution, - leadEdgeDistribution[0].p + {trueTransform: leadEdgeDistribution[0].sample, inputStartIndex: 0, bestProbFromSet: leadEdgeDistribution[0].p} ); const tailEdgeDistribution = [ @@ -155,7 +155,7 @@ describe('SearchQuotientSpur', () => { const length2Path = new LegacyQuotientSpur( length1Path, tailEdgeDistribution, - tailEdgeDistribution[0].p + {trueTransform: tailEdgeDistribution[0].sample, inputStartIndex: 0, bestProbFromSet: tailEdgeDistribution[0].p} ); // Verify that the prior distribution remains fully unaltered.