diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts index de39a5f8cf..408a7e19ac 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts @@ -11,7 +11,7 @@ import { applyTransform, buildMergedTransform } from "@keymanapp/models-template import { LexicalModelTypes } from '@keymanapp/common-types'; import { deepCopy, KMWString } from "@keymanapp/web-utils"; -import { SearchQuotientNode } from "./search-quotient-node.js"; +import { SearchQuotientNode, TokenInputSource } from "./search-quotient-node.js"; import { TokenSplitMap } from "./context-tokenization.js"; import { LegacyQuotientSpur } from "./legacy-quotient-spur.js"; import { LegacyQuotientRoot } from "./legacy-quotient-root.js"; @@ -20,15 +20,6 @@ import Distribution = LexicalModelTypes.Distribution; import LexicalModel = LexicalModelTypes.LexicalModel; import Transform = LexicalModelTypes.Transform; -/** - * Notes critical properties of the inputs comprising each ContextToken. - */ -export interface TokenInputSource { - trueTransform: Transform; - inputStartIndex: number; - bestProbFromSet: number; -} - /** * Breaks apart a raw text string into individual, single-codepoint * transforms, all set with the specified transform ID. @@ -76,13 +67,6 @@ export class ContextToken { */ appliedTransitionId?: number; - /** - * Represents the original, 'true' input transforms (tokenized, as necessary) - * applied to the actual context for the set of keystrokes contributing to - * this token. - */ - private _inputRange: TokenInputSource[]; - /** * Constructs a new, empty instance for use with the specified LexicalModel. * @param model @@ -109,14 +93,12 @@ export class ContextToken { // In case we are unable to perfectly track context (say, due to multitaps) // we need to ensure that only fully-utilized keystrokes are considered. this._searchModule = priorToken.searchModule; - this._inputRange = priorToken._inputRange.slice(); } else { const model = param; // May be altered outside of the constructor. this.isWhitespace = false; this.isPartial = !!isPartial; - this._inputRange = []; rawText ||= ''; @@ -125,12 +107,12 @@ export class ContextToken { let searchModule: SearchQuotientNode = new LegacyQuotientRoot(model); const BASE_PROBABILITY = 1; textToCharTransforms(rawText).forEach((transform) => { - this._inputRange.push({ + let inputMetadata: TokenInputSource = { trueTransform: transform, inputStartIndex: 0, bestProbFromSet: BASE_PROBABILITY - }); - searchModule = new LegacyQuotientSpur(searchModule, [{sample: transform, p: BASE_PROBABILITY}], 1); + }; + searchModule = new LegacyQuotientSpur(searchModule, [{sample: transform, p: BASE_PROBABILITY}], inputMetadata); }); this._searchModule = searchModule; @@ -142,16 +124,11 @@ export class ContextToken { * corresponding to this token. */ addInput(inputSource: TokenInputSource, distribution: Distribution) { - this._inputRange.push(inputSource); - this._searchModule = new LegacyQuotientSpur(this._searchModule, distribution, inputSource.bestProbFromSet); + this._searchModule = new LegacyQuotientSpur(this._searchModule, distribution, inputSource); } - /** - * Denotes the original keystroke Transforms comprising the range corresponding - * to this token. - */ - get inputRange(): Readonly { - return this._inputRange; + get inputCount() { + return this._searchModule.inputCount; } /** @@ -161,6 +138,14 @@ export class ContextToken { return this.exampleInput == ''; } + /** + * Denotes the original keystroke Transforms comprising the range corresponding + * to this token. + */ + get inputRange() { + return this.searchModule.sourceIdentifiers; + } + /** * Gets the unique identifier that may be used to match this ContextToken with * a correction-search result. @@ -175,8 +160,9 @@ export class ContextToken { */ get sourceRangeKey(): string { const components: string[] = []; + const sources = this.searchModule.sourceIdentifiers; - for(const source of this.inputRange) { + for(const source of sources) { const i = source.inputStartIndex; components.push(`T${source.trueTransform.id}${i != 0 ? '@' + i : ''}`); } @@ -207,10 +193,10 @@ export class ContextToken { let lastSourceInput: TokenInputSource; let lastInputDistrib: Distribution; for(const token of tokensToMerge) { - const inputCount = token.inputRange.length; + const inputCount = token.inputCount; let startIndex = 0; - if(token.inputRange.length == 0) { + if(inputCount == 0) { continue; } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 01c47e0039..72f0f53d24 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -582,7 +582,7 @@ export class ContextTokenization { // If we are completely replacing a token via delete left, erase the deleteLeft; // that part applied to a _previous_ token that no longer exists. // We start at index 0 in the insert string for the "new" token. - if(affectedToken.inputRange.length == 0 && distribution[0].sample.deleteLeft != 0) { + if(affectedToken.inputCount == 0 && distribution[0].sample.deleteLeft != 0) { distribution = distribution.map((mass) => ({sample: { ...mass.sample, deleteLeft: 0 }, p: mass.p })); } affectedToken.addInput({trueTransform: sourceInput, inputStartIndex: appliedLength, bestProbFromSet}, distribution); diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts index 9fd218fa55..efbaeed137 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts @@ -11,7 +11,7 @@ import { LexicalModelTypes } from '@keymanapp/common-types'; import { SearchNode } from './distance-modeler.js'; -import { PathResult, SearchQuotientNode } from './search-quotient-node.js'; +import { PathResult, SearchQuotientNode, TokenInputSource } from './search-quotient-node.js'; import { SearchQuotientSpur } from './search-quotient-spur.js'; import Distribution = LexicalModelTypes.Distribution; @@ -27,8 +27,8 @@ export class LegacyQuotientSpur extends SearchQuotientSpur { * @param inputs * @param bestProbFromSet */ - constructor(space: SearchQuotientNode, inputs: Distribution, bestProbFromSet: number) { - super(space, inputs, space.lowestPossibleSingleCost - Math.log(bestProbFromSet)); + constructor(space: SearchQuotientNode, inputs: Distribution, inputSource: TokenInputSource) { + super(space, inputs, inputSource); this.queueNodes(this.buildEdgesForNodes(space.previousResults.map(r => r.node))); return; } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-node.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-node.ts index 2335d6f31f..72001c05ef 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-node.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-node.ts @@ -39,6 +39,15 @@ type CompleteSearchPath = { export type PathResult = NullPath | IntermediateSearchPath | CompleteSearchPath; +/** + * Notes critical properties of the inputs comprising each ContextToken. + */ +export interface TokenInputSource { + trueTransform: Transform; + inputStartIndex: number; + bestProbFromSet: number; +} + /** * Represents all or a portion of the dynamically-generated graph used to search * for predictive-text corrections. @@ -121,6 +130,12 @@ export interface SearchQuotientNode { * the correction-search graph and its paths. */ readonly bestExample: { text: string, p: number }; + + /** + * Gets components useful for building a string-based representation of the + * keystroke range corrected by this search space. + */ + readonly sourceIdentifiers: TokenInputSource[]; } /** diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-root.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-root.ts index fd23301a8f..7bef9bfae6 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-root.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-root.ts @@ -2,7 +2,7 @@ import { LexicalModelTypes } from '@keymanapp/common-types'; import { SearchNode, SearchResult } from './distance-modeler.js'; -import { generateSpaceSeed, PathResult, SearchQuotientNode } from './search-quotient-node.js'; +import { generateSpaceSeed, PathResult, SearchQuotientNode, TokenInputSource } from './search-quotient-node.js'; import LexicalModel = LexicalModelTypes.LexicalModel; @@ -89,4 +89,8 @@ export class SearchQuotientRoot implements SearchQuotientNode { return [this.rootResult]; } } + + get sourceIdentifiers(): TokenInputSource[] { + return []; + } } \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts index 9cba05fedb..dc3c9c976d 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts @@ -12,7 +12,7 @@ import { QueueComparator as Comparator, KMWString, PriorityQueue } from '@keyman import { LexicalModelTypes } from '@keymanapp/common-types'; import { EDIT_DISTANCE_COST_SCALE, SearchNode, SearchResult } from './distance-modeler.js'; -import { generateSpaceSeed, PathResult, SearchQuotientNode } from './search-quotient-node.js'; +import { generateSpaceSeed, PathResult, SearchQuotientNode, TokenInputSource } from './search-quotient-node.js'; import Distribution = LexicalModelTypes.Distribution; import Transform = LexicalModelTypes.Transform; @@ -25,7 +25,8 @@ export const QUEUE_NODE_COMPARATOR: Comparator = function(arg1, arg2 // Whenever a wordbreak boundary is crossed, a new instance should be made. export abstract class SearchQuotientSpur implements SearchQuotientNode { private selectionQueue: PriorityQueue = new PriorityQueue(QUEUE_NODE_COMPARATOR); - readonly inputs?: Distribution>; + readonly inputs?: Distribution; + readonly inputSource?: TokenInputSource; private parentNode: SearchQuotientNode; readonly spaceId: number; @@ -50,13 +51,14 @@ export abstract class SearchQuotientSpur implements SearchQuotientNode { * * @param parentNode * @param inputs - * @param costHeuristic + * @param inputSource */ - constructor(parentNode: SearchQuotientNode, inputs: Distribution, costHeuristic: number) { + constructor(parentNode: SearchQuotientNode, inputs: Distribution>, inputSource: TokenInputSource) { this.spaceId = generateSpaceSeed(); this.parentNode = parentNode; - this.lowestPossibleSingleCost = (parentNode?.lowestPossibleSingleCost ?? 0) - Math.log(costHeuristic); + this.inputSource = inputSource; + this.lowestPossibleSingleCost = (parentNode?.lowestPossibleSingleCost ?? 0) - Math.log(inputSource?.bestProbFromSet ?? 1); this.inputs = inputs?.length > 0 ? inputs : null; this.inputCount = (parentNode?.inputCount ?? 0) + (this.inputs ? 1 : 0); } @@ -221,4 +223,22 @@ export abstract class SearchQuotientSpur implements SearchQuotientNode { public get previousResults(): SearchResult[] { return Object.values(this.returnedValues ?? {}).map(v => new SearchResult(v)); } + + public get sourceIdentifiers(): TokenInputSource[] { + if(!this.parentNode) { + return []; + } + + const parentSources = this.parentNode.sourceIdentifiers; + if(this.inputSource) { + const inputId = this.inputSource.trueTransform.id; + if(inputId && parentSources.length > 0 && parentSources[parentSources.length - 1].trueTransform.id == inputId) { + return parentSources; + } + + parentSources.push(this.inputSource); + } + + return parentSources; + } } \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/getBestMatches.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/getBestMatches.tests.ts index 5e64aa309c..6346fd5d7f 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/getBestMatches.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/getBestMatches.tests.ts @@ -146,9 +146,9 @@ describe('getBestMatches', () => { {sample: {insert: 'n', deleteLeft: 0}, p: 0.25} ]; - const searchPath1 = new LegacyQuotientSpur(searchPath, synthInput1, 1); - const searchPath2 = new LegacyQuotientSpur(searchPath1, synthInput2, .75); - const searchPath3 = new LegacyQuotientSpur(searchPath2, synthInput3, .75); + const searchPath1 = new LegacyQuotientSpur(searchPath, synthInput1, {trueTransform: synthInput1[0].sample, inputStartIndex: 0, bestProbFromSet: 1}); + const searchPath2 = new LegacyQuotientSpur(searchPath1, synthInput2, {trueTransform: synthInput2[0].sample, inputStartIndex: 0, bestProbFromSet: .75}); + const searchPath3 = new LegacyQuotientSpur(searchPath2, synthInput3, {trueTransform: synthInput3[0].sample, inputStartIndex: 0, bestProbFromSet: .75}); assert.notEqual(searchPath1.spaceId, searchPath.spaceId); assert.notEqual(searchPath2.spaceId, searchPath1.spaceId); @@ -180,9 +180,9 @@ describe('getBestMatches', () => { {sample: {insert: 'n', deleteLeft: 0}, p: 0.25} ]; - const searchPath1 = new LegacyQuotientSpur(searchPath, synthInput1, 1); - const searchPath2 = new LegacyQuotientSpur(searchPath1, synthInput2, .75); - const searchPath3 = new LegacyQuotientSpur(searchPath2, synthInput3, .75); + const searchPath1 = new LegacyQuotientSpur(searchPath, synthInput1, {trueTransform: synthInput1[0].sample, inputStartIndex: 0, bestProbFromSet: 1}); + const searchPath2 = new LegacyQuotientSpur(searchPath1, synthInput2, {trueTransform: synthInput2[0].sample, inputStartIndex: 0, bestProbFromSet: .75}); + const searchPath3 = new LegacyQuotientSpur(searchPath2, synthInput3, {trueTransform: synthInput3[0].sample, inputStartIndex: 0, bestProbFromSet: .75}); assert.notEqual(searchPath1.spaceId, searchPath.spaceId); assert.notEqual(searchPath2.spaceId, searchPath1.spaceId); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-spur.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-spur.tests.ts index 1d9ee2af9a..b122bb4d02 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-spur.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-spur.tests.ts @@ -24,24 +24,24 @@ export function buildSimplePathSplitFixture() { { sample: {insert: 'r', deleteLeft: 0, id: 11}, p: 0.4 }, { sample: {insert: 't', deleteLeft: 0, id: 11}, p: 0.1 } ]; - const path1 = new LegacyQuotientSpur(rootPath, distrib1, distrib1[0].p); + const path1 = new LegacyQuotientSpur(rootPath, distrib1, {trueTransform: distrib1[0].sample, inputStartIndex: 0, bestProbFromSet: distrib1[0].p}); const distrib2 = [ { sample: {insert: 'a', deleteLeft: 0, id: 12}, p: 0.7 }, { sample: {insert: 'e', deleteLeft: 0, id: 12}, p: 0.3 } ]; - const path2 = new LegacyQuotientSpur(path1, distrib2, distrib2[0].p); + const path2 = new LegacyQuotientSpur(path1, distrib2, {trueTransform: distrib2[0].sample, inputStartIndex: 0, bestProbFromSet: distrib2[0].p}); const distrib3 = [ { sample: {insert: 'n', deleteLeft: 0, id: 13}, p: 0.8 }, { sample: {insert: 'r', deleteLeft: 0, id: 13}, p: 0.2 } ]; - const path3 = new LegacyQuotientSpur(path2, distrib3, distrib3[0].p); + const path3 = new LegacyQuotientSpur(path2, distrib3, {trueTransform: distrib3[0].sample, inputStartIndex: 0, bestProbFromSet: distrib3[0].p}); const distrib4 = [ { sample: {insert: 't', deleteLeft: 0, id: 14}, p: 1 } ]; - const path4 = new LegacyQuotientSpur(path3, distrib4, distrib4[0].p); + const path4 = new LegacyQuotientSpur(path3, distrib4, {trueTransform: distrib4[0].sample, inputStartIndex: 0, bestProbFromSet: distrib4[0].p}); return { paths: [rootPath, path1, path2, path3, path4], @@ -68,7 +68,7 @@ describe('SearchQuotientSpur', () => { {sample: {insert: 'o', deleteLeft: 0, id: 13 }, p: 0.2} ]; - const extendedPath = new LegacyQuotientSpur(rootPath, leadEdgeDistribution, leadEdgeDistribution[0].p); + const extendedPath = new LegacyQuotientSpur(rootPath, leadEdgeDistribution, {trueTransform: leadEdgeDistribution[0].sample, inputStartIndex: 0, bestProbFromSet: leadEdgeDistribution[0].p}); assert.equal(extendedPath.inputCount, 1); assert.isNumber(extendedPath.spaceId); @@ -97,7 +97,7 @@ describe('SearchQuotientSpur', () => { const length1Path = new LegacyQuotientSpur( rootPath, leadEdgeDistribution, - leadEdgeDistribution[0].p + {trueTransform: leadEdgeDistribution[0].sample, inputStartIndex: 0, bestProbFromSet: leadEdgeDistribution[0].p} ); const tailEdgeDistribution = [ @@ -109,7 +109,7 @@ describe('SearchQuotientSpur', () => { const length2Path = new LegacyQuotientSpur( length1Path, tailEdgeDistribution, - tailEdgeDistribution[0].p + {trueTransform: tailEdgeDistribution[0].sample, inputStartIndex: 0, bestProbFromSet: tailEdgeDistribution[0].p} ); // Verify that the prior distribution remains fully unaltered. @@ -143,7 +143,7 @@ describe('SearchQuotientSpur', () => { const length1Path = new LegacyQuotientSpur( rootPath, leadEdgeDistribution, - leadEdgeDistribution[0].p + {trueTransform: leadEdgeDistribution[0].sample, inputStartIndex: 0, bestProbFromSet: leadEdgeDistribution[0].p} ); const tailEdgeDistribution = [ @@ -155,7 +155,7 @@ describe('SearchQuotientSpur', () => { const length2Path = new LegacyQuotientSpur( length1Path, tailEdgeDistribution, - tailEdgeDistribution[0].p + {trueTransform: tailEdgeDistribution[0].sample, inputStartIndex: 0, bestProbFromSet: tailEdgeDistribution[0].p} ); // Verify that the prior distribution remains fully unaltered.