From 1df3a35a033a5ad1340a4c04b65f440b487cf201 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 9 Oct 2025 09:40:54 -0500 Subject: [PATCH] refactor(web): maintain correction-path cost thresholding We now track maximum individual probability for any one input in the full, source fat-finger distribution. For some tokenizations, the corresponding input will not be considered, but that should not affect thresholding behaviors. Build-bot: skip build:web Test-bot: skip --- .../src/main/correction/context-state.ts | 7 +- .../src/main/correction/context-token.ts | 11 ++- .../main/correction/context-tokenization.ts | 9 +- .../src/main/correction/distance-modeler.ts | 19 +++- .../context/context-token.tests.ts | 86 +++++++++++-------- .../context/context-tokenization.tests.ts | 41 +++++---- .../context/tokenization-subsets.tests.ts | 16 ++-- .../distance-modeler.tests.ts | 12 +-- 8 files changed, 126 insertions(+), 75 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts index 093d4c7a6d..575bd69753 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts @@ -192,7 +192,7 @@ export class ContextState { */ analyzeTransition( context: Context, - transformDistribution?: Distribution, + transformDistribution: Distribution, // overrides checks for token substitution that can fail for large applied suggestions. isApplyingSuggestion?: boolean ): ContextTransition { @@ -245,8 +245,11 @@ export class ContextState { // and then fold all resulting search spaces (on the final token) into one. const tokenizationAnalysis = trueInputSubset.pendingSet.get(baseTokenization); + // Determine the best probability from among ALL available inputs, before they're split + // into subsets. + const bestProb = transformDistribution.reduce((best, curr) => Math.max(best, curr.p), 0); // Should gain one per subsetBuilder.subsets entry. - const resultTokenization = baseTokenization.evaluateTransition(tokenizationAnalysis, lexicalModel, trueInput); + const resultTokenization = baseTokenization.evaluateTransition(tokenizationAnalysis, lexicalModel, trueInput, bestProb); // ------------ diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts index 60bfd0fad3..4b8927e4f9 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts @@ -24,6 +24,7 @@ import Transform = LexicalModelTypes.Transform; export interface TokenInputSource { trueTransform: Transform; inputStartIndex: number; + bestProbFromSet: number; } /** @@ -129,9 +130,10 @@ export class ContextToken { rawTransformDistributions.forEach((entry) => { this._inputRange.push({ trueTransform: entry[0].sample, - inputStartIndex: 0 + inputStartIndex: 0, + bestProbFromSet: 1 }); - this.searchSpace.addInput(entry); + this.searchSpace.addInput(entry, 1); }); } } @@ -142,7 +144,7 @@ export class ContextToken { */ addInput(inputSource: TokenInputSource, distribution: Distribution) { this._inputRange.push(inputSource); - this.searchSpace.addInput(distribution); + this.searchSpace.addInput(distribution, inputSource.bestProbFromSet); } /** @@ -350,7 +352,8 @@ export class ContextToken { backupToken = new ContextToken(constructingToken); constructingToken.addInput({ trueTransform: priorSourceInput.trueTransform, - inputStartIndex: priorSourceInput.inputStartIndex + extraCharsAdded + inputStartIndex: priorSourceInput.inputStartIndex + extraCharsAdded, + bestProbFromSet: priorSourceInput.bestProbFromSet }, tailDistribution); const lenToCommit = lenBeforeLastApply + extraCharsAdded; diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 864164d3cb..e1055964a7 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -497,12 +497,17 @@ export class ContextTokenization { * @param lexicalModel The active lexical model * @param sourceInput The Transform associated with the keystroke triggering * the transition. + * @param bestProbFromSet The probability of the single most likely input + * transform in the overall transformDistribution associated with the + * keystroke triggering theh transition. It need not be represented by the + * pendingTokenization to be built. * @returns */ evaluateTransition( pendingTokenization: PendingTokenization, lexicalModel: LexicalModel, - sourceInput: Transform + sourceInput: Transform, + bestProbFromSet: number ): ContextTokenization { const { alignment: alignment, inputs } = pendingTokenization; const sliceIndex = alignment.edgeWindow.sliceIndex; @@ -581,7 +586,7 @@ export class ContextTokenization { if(affectedToken.inputRange.length == 0 && distribution[0].sample.deleteLeft != 0) { distribution = distribution.map((mass) => ({sample: { ...mass.sample, deleteLeft: 0 }, p: mass.p })); } - affectedToken.addInput({trueTransform: sourceInput, inputStartIndex: appliedLength}, distribution); + affectedToken.addInput({trueTransform: sourceInput, inputStartIndex: appliedLength, bestProbFromSet: bestProbFromSet}, distribution); appliedLength += KMWString.length(distribution[0].sample.insert); const tokenize = determineModelTokenizer(lexicalModel); diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts index a1b346636e..2e5655bb28 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts @@ -652,6 +652,12 @@ export class SearchSpace { */ private processedEdgeSet: {[pathKey: string]: boolean} = {}; + /** + * Provides a heuristic for the base cost at each depth if the best + * individual input were taken at that level. + */ + private lowestCostAtDepth: number[]; + /** * Clone constructor. Deep-copies its internal queues, but not search nodes. * @param instance @@ -670,6 +676,7 @@ export class SearchSpace { this.rootNode = arg1.rootNode; // Re-use already-checked Nodes. this.completedPaths = [].concat(arg1.completedPaths); + this.lowestCostAtDepth = arg1.lowestCostAtDepth.slice(); this.returnedValues = {...arg1.returnedValues}; this.processedEdgeSet = {...arg1.processedEdgeSet}; @@ -688,6 +695,7 @@ export class SearchSpace { this.selectionQueue = new PriorityQueue(QUEUE_NODE_COMPARATOR); this.rootNode = new SearchNode(model.traverseFromRoot(), model.toKey ? model.toKey.bind(model) : null); this.selectionQueue.enqueue(this.rootNode); + this.lowestCostAtDepth = []; this.completedPaths = []; } @@ -722,8 +730,12 @@ export class SearchSpace { * @param inputDistribution The fat-finger distribution for the incoming keystroke (or * just the raw keystroke if corrections are disabled) */ - addInput(inputDistribution: Distribution) { - this._inputSequence.push(inputDistribution); + addInput(inputDistribution: Distribution, bestProbFromSet: number) { + const input = inputDistribution; + this._inputSequence.push(input); + const lastDepthCost = this.lowestCostAtDepth[this.lowestCostAtDepth.length - 1] ?? 0; + const logTierCost = -Math.log(bestProbFromSet); + this.lowestCostAtDepth.push(lastDepthCost + logTierCost); // Assumes that `inputDistribution` is already sorted. this.minInputCost.push(-Math.log(inputDistribution[0].p)); @@ -822,7 +834,8 @@ export class SearchSpace { // ... or even just not the then-current layer of the keyboard. // // TODO: still consider the lowest-cost individual edges for THIS specific criterion. - if(currentNode.currentCost > /* tierMinCost */ + 2.5 * SearchSpace.EDIT_DISTANCE_COST_SCALE) { + const tierMinCost = this.lowestCostAtDepth[currentNode.priorInput.length-1]; + if(currentNode.currentCost > tierMinCost + 2.5 * SearchSpace.EDIT_DISTANCE_COST_SCALE) { return unmatchedResult; } diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts index 3ff9b05798..edcb0b94c7 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts @@ -128,22 +128,25 @@ describe('ContextToken', function() { token1.addInput({ trueTransform: srcTransform, - inputStartIndex: 0 + inputStartIndex: 0, + bestProbFromSet: 1 }, [{sample: {insert: 'can', deleteLeft: 0, deleteRight: 0, id: 1}, p: 1}]); token2.addInput({ trueTransform: srcTransform, - inputStartIndex: 3 + inputStartIndex: 3, + bestProbFromSet: 1 }, [{sample: {insert: "'", deleteLeft: 0, deleteRight: 0, id: 1}, p: 1}]); token3.addInput({ trueTransform: srcTransform, - inputStartIndex: 4 + inputStartIndex: 4, + bestProbFromSet: 1 }, [{sample: {insert: 't', deleteLeft: 0, deleteRight: 0, id: 1}, p: 1}]); const merged = ContextToken.merge([token1, token2, token3], plainModel); assert.equal(merged.exampleInput, "can't"); - assert.deepEqual(merged.inputRange, [ { trueTransform: srcTransform, inputStartIndex: 0 } ]); + assert.deepEqual(merged.inputRange, [ { trueTransform: srcTransform, inputStartIndex: 0, bestProbFromSet: 1 } ]); assert.deepEqual(merged.searchSpace.inputSequence, [[{sample: srcTransform, p: 1}]]); }); @@ -168,35 +171,41 @@ describe('ContextToken', function() { token1.addInput({ trueTransform: srcTransform1, - inputStartIndex: 0 + inputStartIndex: 0, + bestProbFromSet: 1 }, [{sample: srcTransform1, p: 1}]); token1.addInput({ trueTransform: srcTransform2, - inputStartIndex: 0 + inputStartIndex: 0, + bestProbFromSet: 1 }, [{sample: {insert: 's', deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}]); token2.addInput({ trueTransform: srcTransform2, - inputStartIndex: 1 + inputStartIndex: 1, + bestProbFromSet: 1 }, [{sample: {insert: "and", deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}]); token3.addInput({ trueTransform: srcTransform2, - inputStartIndex: 4 + inputStartIndex: 4, + bestProbFromSet: 1 }, [{sample: {insert: 's', deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}]); token3.addInput({ trueTransform: srcTransform3, - inputStartIndex: 0 + inputStartIndex: 0, + bestProbFromSet: 1 }, [{sample: srcTransform3, p: 1}]); token4.addInput({ trueTransform: srcTransform4, - inputStartIndex: 0 + inputStartIndex: 0, + bestProbFromSet: 1 }, [{sample: srcTransform4, p: 1}]); const merged = ContextToken.merge(tokensToMerge, plainModel); assert.equal(merged.exampleInput, "applesandsourgrapes"); - assert.deepEqual(merged.inputRange, srcTransforms.map((t) => ({ trueTransform: t, inputStartIndex: 0 }) )); + assert.deepEqual(merged.inputRange, srcTransforms.map((t) => ({ trueTransform: t, inputStartIndex: 0, bestProbFromSet: 1 }) )); assert.deepEqual(merged.searchSpace.inputSequence, srcTransforms.map((t) => [{sample: t, p: 1}])); }); @@ -221,35 +230,41 @@ describe('ContextToken', function() { token1.addInput({ trueTransform: srcTransform1, - inputStartIndex: 0 + inputStartIndex: 0, + bestProbFromSet: 1 }, [{sample: srcTransform1, p: 1}]); token1.addInput({ trueTransform: srcTransform2, - inputStartIndex: 0 + inputStartIndex: 0, + bestProbFromSet: 1 }, [{sample: {insert: toMathematicalSMP('s'), deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}]); token2.addInput({ trueTransform: srcTransform2, - inputStartIndex: 1 + inputStartIndex: 1, + bestProbFromSet: 1 }, [{sample: {insert: toMathematicalSMP("and"), deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}]); token3.addInput({ trueTransform: srcTransform2, - inputStartIndex: 4 + inputStartIndex: 4, + bestProbFromSet: 1 }, [{sample: {insert: toMathematicalSMP('s'), deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}]); token3.addInput({ trueTransform: srcTransform3, - inputStartIndex: 0 + inputStartIndex: 0, + bestProbFromSet: 1 }, [{sample: srcTransform3, p: 1}]); token4.addInput({ trueTransform: srcTransform4, - inputStartIndex: 0 + inputStartIndex: 0, + bestProbFromSet: 1 }, [{sample: srcTransform4, p: 1}]); const merged = ContextToken.merge(tokensToMerge, plainModel); assert.equal(merged.exampleInput, toMathematicalSMP("applesandsourgrapes")); - assert.deepEqual(merged.inputRange, srcTransforms.map((t) => ({ trueTransform: t, inputStartIndex: 0 }) )); + assert.deepEqual(merged.inputRange, srcTransforms.map((t) => ({ trueTransform: t, inputStartIndex: 0, bestProbFromSet: 1 }) )); assert.deepEqual(merged.searchSpace.inputSequence, srcTransforms.map((t) => [{sample: t, p: 1}])); }); }); @@ -278,7 +293,7 @@ describe('ContextToken', function() { const tokenToSplit = new ContextToken(plainModel); for(let i = 0; i < keystrokeDistributions.length; i++) { - tokenToSplit.addInput({trueTransform: keystrokeDistributions[i][0].sample, inputStartIndex: 0}, keystrokeDistributions[i]); + tokenToSplit.addInput({trueTransform: keystrokeDistributions[i][0].sample, inputStartIndex: 0, bestProbFromSet: .75}, keystrokeDistributions[i]); }; assert.equal(tokenToSplit.sourceText, 'can\''); @@ -316,7 +331,7 @@ describe('ContextToken', function() { const tokenToSplit = new ContextToken(plainModel); for(let i = 0; i < keystrokeDistributions.length; i++) { - tokenToSplit.addInput({trueTransform: keystrokeDistributions[i][0].sample, inputStartIndex: 0}, keystrokeDistributions[i]); + tokenToSplit.addInput({trueTransform: keystrokeDistributions[i][0].sample, inputStartIndex: 0, bestProbFromSet: 1}, keystrokeDistributions[i]); }; assert.equal(tokenToSplit.sourceText, 'biglargetransform'); @@ -343,7 +358,9 @@ describe('ContextToken', function() { insert: 'biglargetransform', deleteLeft: 0, deleteRight: 0 - }, inputStartIndex: i + }, + inputStartIndex: i, + bestProbFromSet: 1 }))); assert.sameDeepOrderedMembers(resultsOfSplit.map(t => t.searchSpace.inputSequence[0]), splitTextArray.map(t => [{ sample: { insert: t, deleteLeft: 0, deleteRight: 0 }, p: 1 @@ -365,7 +382,7 @@ describe('ContextToken', function() { const tokenToSplit = new ContextToken(plainModel); for(let i = 0; i < keystrokeDistributions.length; i++) { - tokenToSplit.addInput({trueTransform: keystrokeDistributions[i][0].sample, inputStartIndex: 0}, keystrokeDistributions[i]); + tokenToSplit.addInput({trueTransform: keystrokeDistributions[i][0].sample, inputStartIndex: 0, bestProbFromSet: 1}, keystrokeDistributions[i]); }; assert.equal(tokenToSplit.exampleInput, 'largelongtransforms'); @@ -388,15 +405,15 @@ describe('ContextToken', function() { assert.equal(resultsOfSplit.length, 3); assert.sameOrderedMembers(resultsOfSplit.map(t => t.exampleInput), splitTextArray); assert.deepEqual(resultsOfSplit[0].inputRange, [ - { trueTransform: keystrokeDistributions[0][0].sample, inputStartIndex: 0 }, - { trueTransform: keystrokeDistributions[1][0].sample, inputStartIndex: 0 }, + { trueTransform: keystrokeDistributions[0][0].sample, inputStartIndex: 0, bestProbFromSet: 1 }, + { trueTransform: keystrokeDistributions[1][0].sample, inputStartIndex: 0, bestProbFromSet: 1 }, ]); assert.deepEqual(resultsOfSplit[1].inputRange, [ - { trueTransform: keystrokeDistributions[1][0].sample, inputStartIndex: 'arge'.length }, - { trueTransform: keystrokeDistributions[2][0].sample, inputStartIndex: 0 }, + { trueTransform: keystrokeDistributions[1][0].sample, inputStartIndex: 'arge'.length, bestProbFromSet: 1 }, + { trueTransform: keystrokeDistributions[2][0].sample, inputStartIndex: 0, bestProbFromSet: 1 }, ]); assert.deepEqual(resultsOfSplit[2].inputRange, [ - { trueTransform: keystrokeDistributions[2][0].sample, inputStartIndex: 'ng'.length } + { trueTransform: keystrokeDistributions[2][0].sample, inputStartIndex: 'ng'.length, bestProbFromSet: 1 } ]); assert.deepEqual(resultsOfSplit[0].searchSpace.inputSequence, [ @@ -459,7 +476,7 @@ describe('ContextToken', function() { const tokenToSplit = new ContextToken(plainModel); for(let i = 0; i < keystrokeDistributions.length; i++) { - tokenToSplit.addInput({trueTransform: keystrokeDistributions[i][0].sample, inputStartIndex: 0}, keystrokeDistributions[i]); + tokenToSplit.addInput({trueTransform: keystrokeDistributions[i][0].sample, inputStartIndex: 0, bestProbFromSet: 1}, keystrokeDistributions[i]); }; assert.equal(tokenToSplit.exampleInput, toMathematicalSMP('largelongtransforms')); @@ -482,15 +499,15 @@ describe('ContextToken', function() { assert.equal(resultsOfSplit.length, 3); assert.sameOrderedMembers(resultsOfSplit.map(t => t.exampleInput), splitTextArray); assert.deepEqual(resultsOfSplit[0].inputRange, [ - { trueTransform: keystrokeDistributions[0][0].sample, inputStartIndex: 0 }, - { trueTransform: keystrokeDistributions[1][0].sample, inputStartIndex: 0 }, + { trueTransform: keystrokeDistributions[0][0].sample, inputStartIndex: 0, bestProbFromSet: 1 }, + { trueTransform: keystrokeDistributions[1][0].sample, inputStartIndex: 0, bestProbFromSet: 1 }, ]); assert.deepEqual(resultsOfSplit[1].inputRange, [ - { trueTransform: keystrokeDistributions[1][0].sample, inputStartIndex: 'arge'.length }, - { trueTransform: keystrokeDistributions[2][0].sample, inputStartIndex: 0 }, + { trueTransform: keystrokeDistributions[1][0].sample, inputStartIndex: 'arge'.length, bestProbFromSet: 1 }, + { trueTransform: keystrokeDistributions[2][0].sample, inputStartIndex: 0, bestProbFromSet: 1 }, ]); assert.deepEqual(resultsOfSplit[2].inputRange, [ - { trueTransform: keystrokeDistributions[2][0].sample, inputStartIndex: 'ng'.length } + { trueTransform: keystrokeDistributions[2][0].sample, inputStartIndex: 'ng'.length, bestProbFromSet: 1 } ]); assert.deepEqual(resultsOfSplit[0].searchSpace.inputSequence, [ @@ -550,7 +567,8 @@ describe('preprocessInputSources', () => { const results = preprocessInputSources(transforms.map((t) => ({ trueTransform: t, - inputStartIndex: 0 + inputStartIndex: 0, + bestProbFromSet: 1 }))); assert.equal(results.length, transforms.length); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index 58cb7fb462..92fe49674f 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -36,7 +36,7 @@ function toTransformToken(text: string, transformId?: number) { let isWhitespace = text == ' '; let token = new ContextToken(plainModel); const textAsTransform = { insert: text, deleteLeft: 0, id: idSeed }; - token.addInput({trueTransform: textAsTransform, inputStartIndex: 0}, [ { sample: textAsTransform, p: 1 } ]); + token.addInput({trueTransform: textAsTransform, inputStartIndex: 0, bestProbFromSet: 1}, [ { sample: textAsTransform, p: 1 } ]); token.isWhitespace = isWhitespace; return token; } @@ -185,10 +185,11 @@ describe('ContextTokenization', function() { }, removedTokenCount: 0 }, - inputs: [{ sample: inputTransformMap, p: 1 }] + inputs: [{ sample: inputTransformMap, p: 1 }], }, plainModel, - inputTransform + inputTransform, + 1 ); assert.isOk(tokenization); @@ -234,7 +235,8 @@ describe('ContextTokenization', function() { inputs: [{ sample: inputTransformMap, p: 1 }] }, plainModel, - inputTransform + inputTransform, + 1 ); assert.isOk(tokenization); @@ -267,10 +269,11 @@ describe('ContextTokenization', function() { }, removedTokenCount: 0 }, - inputs: [{ sample: inputTransformMap, p: 1 }] + inputs: [{ sample: inputTransformMap, p: 1 }], }, plainModel, - inputTransform + inputTransform, + 1 ); assert.isOk(tokenization); @@ -310,7 +313,8 @@ describe('ContextTokenization', function() { inputs: [{ sample: inputTransformMap, p: 1 }] }, plainModel, - inputTransform + inputTransform, + 1 ); assert.isOk(tokenization); @@ -354,7 +358,8 @@ describe('ContextTokenization', function() { inputs: [{ sample: inputTransformMap, p: 1 }] }, plainModel, - inputTransform + inputTransform, + 1 ); assert.isOk(tokenization); @@ -405,7 +410,8 @@ describe('ContextTokenization', function() { inputs: [{ sample: inputTransformMap, p: 1 }] }, plainModel, - inputTransform + inputTransform, + 1 ); assert.isOk(tokenization); @@ -416,12 +422,12 @@ describe('ContextTokenization', function() { const boundaryToken = tokenization.tokens[tokenization.tokens.length-3]; const boundaryTailInput = boundaryToken.inputRange[boundaryToken.inputRange.length - 1]; - assert.deepEqual(boundaryTailInput, {trueTransform: inputTransform, inputStartIndex: 0}); + assert.deepEqual(boundaryTailInput, {trueTransform: inputTransform, inputStartIndex: 0, bestProbFromSet: 1}); // The new tail tokens should not include anything from the original tail; // the token should be replaced. - assert.deepEqual(tokenization.tokens[tokenization.tokens.length-2].inputRange, [{trueTransform: inputTransform, inputStartIndex: 0}]); - assert.deepEqual(tokenization.tokens[tokenization.tokens.length-1].inputRange, [{trueTransform: inputTransform, inputStartIndex: 1}]); + assert.deepEqual(tokenization.tokens[tokenization.tokens.length-2].inputRange, [{trueTransform: inputTransform, inputStartIndex: 0, bestProbFromSet: 1}]); + assert.deepEqual(tokenization.tokens[tokenization.tokens.length-1].inputRange, [{trueTransform: inputTransform, inputStartIndex: 1, bestProbFromSet: 1}]); const tailIndex = tokenization.tokens.length - 1; for(let i of inputTransformMap.keys()) { @@ -467,7 +473,8 @@ describe('ContextTokenization', function() { inputs: [{ sample: inputTransformMap, p: 1 }] }, plainModel, - { insert: ' ', deleteLeft: 0 } + { insert: ' ', deleteLeft: 0 }, + 1 ); assert.isOk(tokenization); @@ -530,7 +537,8 @@ describe('ContextTokenization', function() { inputs: [{ sample: inputTransformMap, p: 1 }] }, plainModel, - { insert: 't', deleteLeft: 0 } + { insert: 't', deleteLeft: 0 }, + 1 ); assert.isOk(tokenization); @@ -600,7 +608,8 @@ describe('ContextTokenization', function() { inputs: [{ sample: inputTransformMap, p: 1 }] }, plainModel, - inputTransform + inputTransform, + 1 ); assert.isOk(tokenization); @@ -622,7 +631,7 @@ describe('ContextTokenization', function() { // We've also appended a '.' to the final split-off token. Thus, we need // to account for that in the assertions below. assert.includeDeepMembers( - [...baseTokenization.tail.inputRange, { trueTransform: inputTransform, inputStartIndex: 0 }], + [...baseTokenization.tail.inputRange, { trueTransform: inputTransform, inputStartIndex: 0, bestProbFromSet: 1 }], [...tokenization.tokens[tokenization.tokens.length - 1].inputRange] ); assert.includeDeepMembers( diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts index a8348191ca..fbd013ddeb 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts @@ -173,7 +173,7 @@ describe('precomputationSubsetKeyer', function() { const token = new ContextToken(plainModel, 'da'); // source text: 'date' token.addInput( - {trueTransform: {insert: 'te', deleteLeft: 0}, inputStartIndex: 0}, + {trueTransform: {insert: 'te', deleteLeft: 0}, inputStartIndex: 0, bestProbFromSet: 1}, [{sample: {insert: 'te', deleteLeft: 0}, p: 1}] ); return token; @@ -200,7 +200,7 @@ describe('precomputationSubsetKeyer', function() { const token = new ContextToken(plainModel, 'da'); // source text: 'date' token.addInput( - {trueTransform: {insert: 'te', deleteLeft: 0}, inputStartIndex: 0}, + {trueTransform: {insert: 'te', deleteLeft: 0}, inputStartIndex: 0, bestProbFromSet: 1}, [{sample: {insert: 't', deleteLeft: 0}, p: 1}] ); return token; @@ -241,7 +241,7 @@ describe('precomputationSubsetKeyer', function() { token.isPartial = true; // source text: 'dat' token.addInput( - {trueTransform: {insert: 't', deleteLeft: 0}, inputStartIndex: 0}, + {trueTransform: {insert: 't', deleteLeft: 0}, inputStartIndex: 0, bestProbFromSet: 1}, [{sample: {insert: 'ts', deleteLeft: 0}, p: 1}] ); return token; @@ -269,7 +269,7 @@ describe('precomputationSubsetKeyer', function() { token.isPartial = true; // source text: 'dat' token.addInput( - {trueTransform: {insert: 't', deleteLeft: 0}, inputStartIndex: 0}, + {trueTransform: {insert: 't', deleteLeft: 0}, inputStartIndex: 0, bestProbFromSet: 1}, [{sample: {insert: 't', deleteLeft: 0}, p: 1}] ); return token; @@ -721,13 +721,13 @@ describe('TokenizationSubsetBuilder', function() { const fourCharTailToken = new ContextToken(baseTokenization.tail); fourCharTailToken.addInput( - {trueTransform: { insert: 'é', deleteLeft: 1 }, inputStartIndex: 0}, + {trueTransform: { insert: 'é', deleteLeft: 1 }, inputStartIndex: 0, bestProbFromSet: 1}, [{ sample: trueSourceTransform, p: .6 }] ); const fiveCharTailToken = new ContextToken(baseTokenization.tail); fiveCharTailToken.addInput( - {trueTransform: { insert: 'é', deleteLeft: 1 }, inputStartIndex: 0}, + {trueTransform: { insert: 'é', deleteLeft: 1 }, inputStartIndex: 0, bestProbFromSet: 1}, [{ sample: { insert: 's', deleteLeft: 0 }, p: .4 }] ); @@ -760,13 +760,13 @@ describe('TokenizationSubsetBuilder', function() { const twoCharTailToken = new ContextToken(baseTokenization.tail); twoCharTailToken.addInput( - {trueTransform: { insert: 'é', deleteLeft: 1 }, inputStartIndex: 0}, + {trueTransform: { insert: 'é', deleteLeft: 1 }, inputStartIndex: 0, bestProbFromSet: .6}, [{ sample: trueSourceTransform, p: .6 }] ); const threeCharTailToken = new ContextToken(baseTokenization.tail); threeCharTailToken.addInput( - {trueTransform: { insert: 'é', deleteLeft: 1 }, inputStartIndex: 0}, + {trueTransform: { insert: 'é', deleteLeft: 1 }, inputStartIndex: 0, bestProbFromSet: .6}, [{ sample: { insert: 'a', deleteLeft: 0 }, p: .4 }] ); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/distance-modeler.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/distance-modeler.tests.ts index da9ce9acef..7e1657e77b 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/distance-modeler.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/distance-modeler.tests.ts @@ -1255,9 +1255,9 @@ describe('Correction Distance Modeler', function() { {sample: {insert: 'n', deleteLeft: 0}, p: 0.25} ]; - searchSpace.addInput(synthDistribution1); - searchSpace.addInput(synthDistribution2); - searchSpace.addInput(synthDistribution3); + searchSpace.addInput(synthDistribution1, 1); + searchSpace.addInput(synthDistribution2, .75); + searchSpace.addInput(synthDistribution3, .25); let iter = searchSpace.getBestMatches(buildTestTimer()); // disables the correction-search timeout. await checkRepeatableResults_teh(iter); @@ -1285,9 +1285,9 @@ describe('Correction Distance Modeler', function() { {sample: {insert: 'n', deleteLeft: 0}, p: 0.25} ]; - searchSpace.addInput(synthDistribution1); - searchSpace.addInput(synthDistribution2); - searchSpace.addInput(synthDistribution3); + searchSpace.addInput(synthDistribution1, 1); + searchSpace.addInput(synthDistribution2, .75); + searchSpace.addInput(synthDistribution3, .25); let iter = searchSpace.getBestMatches(buildTestTimer()); // disables the correction-search timeout. await checkRepeatableResults_teh(iter);