From 3ae0e4cb8e5c38b0e648c39ad4c9d0fa8643f6a8 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 2 Apr 2026 11:47:40 -0500 Subject: [PATCH 1/8] refactor(web): extract token-search result duplicate-filtering method Build-bot: skip build:web Test-bot: skip --- .../src/main/correction/distance-modeler.ts | 53 +++++++++---------- .../main/correction/token-result-mapping.ts | 25 +++++++++ .../worker-thread/src/main/predict-helpers.ts | 4 +- .../correction-search/getBestMatches.tests.ts | 10 ++-- 4 files changed, 58 insertions(+), 34 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts index 50ccbed73f..3b38110cd9 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts @@ -6,7 +6,7 @@ import { LexicalModelTypes } from '@keymanapp/common-types'; import { ClassicalDistanceCalculation } from './classical-calculation.js'; import { ExecutionTimer, STANDARD_TIME_BETWEEN_DEFERS } from './execution-timer.js'; import { SearchQuotientNode } from './search-quotient-node.js'; -import { TokenResultMapping } from './token-result-mapping.js'; +import { initTokenResultFilterer, TokenResultMapping } from './token-result-mapping.js'; import { subsetByChar, subsetByInterval, mergeSubset, TransformSubset } from '../transform-subsets.js'; import TransformUtils from '../transformUtils.js'; @@ -572,6 +572,18 @@ export class SearchNode { } } +/** + * Searches for the best available token corrections from among the provided + * SearchSpaces, ending after the configured timer has elapsed or all available + * corrections have been enumerated. + * @param searchModules + * @param timer + * @returns + */ +export const getBestTokenMatches = (searchModules: SearchQuotientNode[], timer?: ExecutionTimer) => { + return getBestMatches(searchModules, timer, initTokenResultFilterer()); +} + /** * Searches for the best available corrections from among the provided * SearchSpaces, ending after the configured timer has elapsed or all available @@ -580,7 +592,13 @@ export class SearchNode { * @param timer * @returns */ -export async function *getBestMatches(searchModules: SearchQuotientNode[], timer: ExecutionTimer): AsyncGenerator { +export async function *getBestMatches( + searchModules: SearchQuotientNode[], + timer: ExecutionTimer, + filter?: (searchResult: TokenResultMapping) => boolean +): AsyncGenerator { + filter ??= () => true; + const spaceQueue = new PriorityQueue((a, b) => a.currentCost - b.currentCost); // Stage 1 - if we already have extracted results, build a queue just for them @@ -595,14 +613,14 @@ export async function *getBestMatches(searchModules: SearchQuotientNode[], timer // With potential prior results re-queued, NOW enqueue. (Not before - the heap may reheapify!) spaceQueue.enqueueAll(searchModules); - let currentReturns: {[resultKey: string]: SearchNode} = {}; - // Stage 2: the fun part; actually searching! do { const entry: TokenResultMapping = timer.time(() => { if((priorResultsQueue.peek()?.totalCost ?? Number.POSITIVE_INFINITY) <= spaceQueue.peek().currentCost) { const result = priorResultsQueue.dequeue(); - currentReturns[result.node.resultKey] = result.node; + + // Just pass it through the filter, even if it _was_ already filtered once before. + filter(result); return result; } @@ -613,27 +631,8 @@ export async function *getBestMatches(searchModules: SearchQuotientNode[], timer if(newResult.type == 'none') { return null; } else if(newResult.type == 'complete') { - const node = newResult.mapping.node; - - // Is the entry a reasonable result? - if(node.isFullReplacement) { - // If the entry's 'match' fully replaces the input string, we consider it - // unreasonable and ignore it. Also, if we've reached this point... - // we can(?) assume that everything thereafter is as well. - return null; - } - - // As we can't guarantee a monotonically-increasing cost during the search - - // due to effects from keystrokes with deleteLeft > 0 - it's technically - // possible to find a lower-cost path later in such cases. - // - // If it occurs, we should re-emit it - it'll show up earlier in the - // suggestions that way, as it should. - if((currentReturns[node.resultKey]?.currentCost ?? Number.MAX_VALUE) > node.currentCost) { - currentReturns[node.resultKey] = node; - // Do not track yielded time. - return newResult.mapping; - } + const mapping = newResult.mapping; + return filter(mapping) ? mapping : null; } return null; @@ -651,4 +650,4 @@ export async function *getBestMatches(searchModules: SearchQuotientNode[], timer } while(!timer.elapsed && spaceQueue.peek().currentCost < Number.POSITIVE_INFINITY); return null; -} +} \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts index d177b54bb0..83d174eb01 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts @@ -6,6 +6,31 @@ import LexiconTraversal = LexicalModelTypes.LexiconTraversal; import ProbabilityMass = LexicalModelTypes.ProbabilityMass; import Transform = LexicalModelTypes.Transform; +export function initTokenResultFilterer() { + const priorReturns: Map = new Map(); + + const closure = (searchResult: TokenResultMapping) => { + const node = searchResult.node; + + if(node.isFullReplacement) { + // If the entry's 'match' fully replaces the input string, we consider it + // unreasonable and ignore it. Also, if we've reached this point... + // we can(?) assume that everything thereafter is as well. + return false; + } + + if((priorReturns.get(node.resultKey)?.currentCost ?? Number.MAX_VALUE) >= searchResult.totalCost) { + priorReturns.set(node.resultKey, node); + + return true; + } else { + return false; + } + }; + + return closure; +} + export class TokenResultMapping { readonly node: SearchNode; diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index c8d1204ac6..9c5892424f 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -12,7 +12,7 @@ import { ContextTransition } from './correction/context-transition.js'; import { ExecutionTimer } from './correction/execution-timer.js'; import ModelCompositor from './model-compositor.js'; import { getBestMatches } from './correction/distance-modeler.js'; -import { TokenResultMapping } from './correction/token-result-mapping.js'; +import { initTokenResultFilterer, TokenResultMapping } from './correction/token-result-mapping.js'; const searchForProperty = defaultWordbreaker.searchForProperty; @@ -534,7 +534,7 @@ export async function correctAndEnumerate( let rawPredictions: CorrectionPredictionTuple[] = []; let bestCorrectionCost: number; const correctionPredictionMap: Record> = {}; - for await(const match of getBestMatches(searchModules, timer)) { + for await(const match of getBestMatches(searchModules, timer, initTokenResultFilterer())) { // Corrections obtained: now to predict from them! const tokenization = tokenizations.find(t => t.spaceId == match.spaceId); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/getBestMatches.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/getBestMatches.tests.ts index 88271dbbcd..ff5a8289ae 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/getBestMatches.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/getBestMatches.tests.ts @@ -10,7 +10,7 @@ import { assert } from 'chai'; import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; -import { correction, getBestMatches, LegacyQuotientSpur, models, LegacyQuotientRoot, TokenResultMapping } from '@keymanapp/lm-worker/test-index'; +import { correction, getBestTokenMatches, LegacyQuotientSpur, models, LegacyQuotientRoot, TokenResultMapping } from '@keymanapp/lm-worker/test-index'; import TrieModel = models.TrieModel; @@ -97,7 +97,7 @@ describe('getBestMatches', () => { const searchSpace = new LegacyQuotientRoot(testModel); const timer = buildTestTimer(); - const iter = getBestMatches([searchSpace], timer); + const iter = getBestTokenMatches([searchSpace], timer); // While there's no input, insertion operations can produce suggestions. const resultState = await iter.next(); @@ -152,7 +152,7 @@ describe('getBestMatches', () => { assert.notEqual(searchPath2.spaceId, searchPath1.spaceId); assert.notEqual(searchPath3.spaceId, searchPath2.spaceId); - const iter = getBestMatches([searchPath3], buildTestTimer()); // disables the correction-search timeout. + const iter = getBestTokenMatches([searchPath3], buildTestTimer()); // disables the correction-search timeout. await checkRepeatableResults_teh(iter); }); @@ -186,12 +186,12 @@ describe('getBestMatches', () => { assert.notEqual(searchPath2.spaceId, searchPath1.spaceId); assert.notEqual(searchPath3.spaceId, searchPath2.spaceId); - const iter = getBestMatches([searchPath3], buildTestTimer()); // disables the correction-search timeout. + const iter = getBestTokenMatches([searchPath3], buildTestTimer()); // disables the correction-search timeout. await checkRepeatableResults_teh(iter); // The key: do we get the same results the second time? // Reset the iterator first... - const iter2 = getBestMatches([searchPath3], buildTestTimer()); // disables the correction-search timeout. + const iter2 = getBestTokenMatches([searchPath3], buildTestTimer()); // disables the correction-search timeout. await checkRepeatableResults_teh(iter2); }); }); From 2710b261d86920b0d6ce8342b6546a4f615b4027 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 20 Apr 2026 16:52:25 -0500 Subject: [PATCH 2/8] fix(web): undo accidental sign-change in extracted filter method --- .../worker-thread/src/main/correction/token-result-mapping.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts index e0ea942911..ea79758cd5 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts @@ -29,7 +29,7 @@ export function initTokenResultFilterer() { return false; } - if((priorReturns.get(node.resultKey)?.currentCost ?? Number.MAX_VALUE) >= searchResult.totalCost) { + if((priorReturns.get(node.resultKey)?.currentCost ?? Number.MAX_VALUE) > searchResult.totalCost) { priorReturns.set(node.resultKey, node); return true; From 6abf6ed08c87b883d8ae276fbf43c696b1dbac32 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Wed, 22 Apr 2026 09:08:46 -0500 Subject: [PATCH 3/8] fix(web): ensure newly-mapping result is returned from clusters --- .../src/main/correction/search-quotient-cluster.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-cluster.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-cluster.ts index ad5b58918c..b3201834da 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-cluster.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-cluster.ts @@ -142,11 +142,16 @@ export class SearchQuotientCluster implements SearchQuotientNode { const currentResult = bestPath.handleNextNode(); this.selectionQueue.enqueue(bestPath); + let finalResult = currentResult; if(currentResult.type == 'complete') { - this.completedPaths.push(new TokenResultMapping(currentResult.mapping, this.spaceId)); + finalResult = { + ...currentResult, + mapping: new TokenResultMapping(currentResult.mapping, this.spaceId) + }; + this.completedPaths.push(finalResult.mapping); } - return currentResult; + return finalResult; } public get previousResults(): TokenResultMapping[] { From 50cf4b997e140c481128447ae59c5da5fb4f2019 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Wed, 22 Apr 2026 09:09:08 -0500 Subject: [PATCH 4/8] fix(web): require helper argument required in main version of function --- .../worker-thread/src/main/correction/distance-modeler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts index 37de6f9c49..5408a79bb6 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts @@ -582,7 +582,7 @@ export class SearchNode { * @param timer * @returns */ -export const getBestTokenMatches = (searchModules: SearchQuotientNode[], timer?: ExecutionTimer) => { +export const getBestTokenMatches = (searchModules: SearchQuotientNode[], timer: ExecutionTimer) => { return getBestMatches(searchModules, timer, initTokenResultFilterer()); } From 4dae9255e0166e2f4c09236db405b7d254951fcf Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Fri, 24 Apr 2026 20:40:15 +0700 Subject: [PATCH 5/8] change(web): Apply EB suggestions from code review Co-authored-by: Eberhard Beilharz --- .../worker-thread/src/main/correction/legacy-quotient-spur.ts | 2 +- .../predictive-text/worker-thread/src/main/predict-helpers.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts index dadf5bc239..147efc3e52 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts @@ -51,7 +51,7 @@ export class LegacyQuotientSpur extends SearchQuotientSpur { return new LegacyQuotientSpur(parentNode, inputs, inputSource) as this; } - protected buildEdgesFromResults(priorResults: ReadonlyArray) { + protected buildEdgesFromResults(priorResults: ReadonlyArray): SearchNode[] { // With a newly-available input, we can extend new input-dependent paths from // our previously-reached 'extractedResults' nodes. let outboundNodes = priorResults.map((result) => { diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 4b7bde5bae..4a915b74b9 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -534,7 +534,7 @@ export async function correctAndEnumerate( let rawPredictions: CorrectionPredictionTuple[] = []; let bestCorrectionCost: number; const correctionPredictionMap: Record> = {}; - for await(const match of getBestMatches(searchModules, timer, initTokenResultFilterer())) { + for await(const match of getBestTokenMatches(searchModules, timer)) { // Corrections obtained: now to predict from them! const tokenization = tokenizations.find(t => t.spaceId == match.spaceId); From f0c7084dc228da710ce67b3ba85a744f0513ed91 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Fri, 24 Apr 2026 08:52:47 -0500 Subject: [PATCH 6/8] change(web): re-filter previously returned results --- .../src/main/correction/distance-modeler.ts | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts index 5408a79bb6..fdbd8f92f0 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts @@ -621,9 +621,25 @@ export async function *getBestMatches( if((priorResultsQueue.peek()?.totalCost ?? Number.POSITIVE_INFINITY) <= spaceQueue.peek().currentCost) { const result = priorResultsQueue.dequeue(); - // Just pass it through the filter, even if it _was_ already filtered once before. - filter(result); - return result; + // There's no guarantee that the filter closure is the same instance as + // before. + // + // As a filter function may contain caching and/or deduplication + // components, we pass pre-existing results through the filter so that + // it may reconstruct related state and thus cache/deduplicate new + // results based upon old results. + // + // See `initTokenResultFilterer()`, which maintains a map used for + // deduplication. + // + // As these _are_ pre-existing results, we know that they previously + // passed through the filter with a `true` response. However, as #14366 + // isn't implemented, it IS technically possible that a lower-cost + // result was found after a higher-cost result in some cases; therefore + // there is a chance such a duplicate may exist. On that basis, + // re-filtering even for prior results is reasonably motivated at this + // time. + return filter(result) ? result : null; } let lowestCostSource = spaceQueue.dequeue(); From 14a86afeb45582eb1e6f67b981dff4f5680991a7 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Fri, 24 Apr 2026 09:26:02 -0500 Subject: [PATCH 7/8] change(web): fix imports after suggested change --- .../predictive-text/worker-thread/src/main/predict-helpers.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 4a915b74b9..c16022c227 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -11,8 +11,8 @@ import { ContextState, determineContextSlideTransform } from './correction/conte import { ContextTransition } from './correction/context-transition.js'; import { ExecutionTimer } from './correction/execution-timer.js'; import ModelCompositor from './model-compositor.js'; -import { getBestMatches } from './correction/distance-modeler.js'; -import { initTokenResultFilterer, TokenResultMapping } from './correction/token-result-mapping.js'; +import { getBestTokenMatches } from './correction/distance-modeler.js'; +import { TokenResultMapping } from './correction/token-result-mapping.js'; const searchForProperty = defaultWordbreaker.searchForProperty; From d54f0cfef35b5f969e4290fec8341961f1418a4d Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Fri, 24 Apr 2026 10:32:32 -0500 Subject: [PATCH 8/8] change(web): store cost in deduplication object, not actual results --- .../src/main/correction/token-result-mapping.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts index 650630fb5c..fef8b26622 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts @@ -18,7 +18,7 @@ import ProbabilityMass = LexicalModelTypes.ProbabilityMass; import Transform = LexicalModelTypes.Transform; export function initTokenResultFilterer() { - const priorReturns: Map = new Map(); + const priorReturnCosts: Map = new Map(); const closure = (searchResult: TokenResultMapping) => { if(searchResult.isFullReplacement) { @@ -28,8 +28,8 @@ export function initTokenResultFilterer() { return false; } - if((priorReturns.get(searchResult.matchString)?.totalCost ?? Number.MAX_VALUE) > searchResult.totalCost) { - priorReturns.set(searchResult.matchString, searchResult); + if((priorReturnCosts.get(searchResult.matchString) ?? Number.MAX_VALUE) > searchResult.totalCost) { + priorReturnCosts.set(searchResult.matchString, searchResult.totalCost); return true; } else {