From 9fc7c23a5887df1a3ba2c5dfd36ee878c13bafdd Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Fri, 17 Apr 2026 11:25:50 -0500 Subject: [PATCH] change(web): TokenizationCorrector should use quotient-nodes internally for queue This, plus the upcoming QuotientNodeFinalizer class, will support result forwarding in multi-tokenization contexts during multi-token & multi-tokenization correction. --- .../main/correction/tokenization-corrector.ts | 99 ++++++++++--------- .../tokenization-corrector.tests.ts | 40 ++++---- 2 files changed, 71 insertions(+), 68 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts index 5b7813e865..a1d3e70f68 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts @@ -12,7 +12,8 @@ import { PriorityQueue } from "@keymanapp/web-utils"; import { ContextToken } from "./context-token.js"; import { CorrectionSearchable, PathResult } from "./correction-searchable.js"; -import { ContextTokenization } from "../test-index.js"; +import { ContextTokenization } from "./context-tokenization.js"; +import { SearchQuotientNode } from "./search-quotient-node.js"; import { TokenizationResultMapping } from "./tokenization-result-mapping.js"; // PathResult needs to be generic: @@ -30,43 +31,44 @@ export class TokenizationCorrector implements CorrectionSearchable; - private tokenCostMap: Map; - private _lockedTokenResults: Map; + private selectionQueue: PriorityQueue; + private tokenCostMap: Map; + private _lockedTokenResults: Map; private lastTotalCost: number; private handleHasBeenCalled: boolean = false; get currentCost(): number { - const token = this.selectionQueue.peek(); - if(!token) { + const correctable = this.selectionQueue.peek(); + if(!correctable) { return this.lastTotalCost; } - return this.getUpdatedTotalCost(token, token.searchModule.currentCost); + return this.getUpdatedTotalCost(correctable, correctable.currentCost); }; get orderedTokens(): ReadonlyArray { return this.tokenization.tokens.slice(-this.tailCorrectionLength); } - get lockedTokens(): ReadonlyArray { - return this._lockedTokens; + get uncorrectableTokens(): ReadonlyArray { + return this.orderedTokens.filter((t) => this._uncorrectables.find((c) => c.spaceId == t.spaceId)); } - get boundTokens(): ReadonlyArray { - return this._boundTokens; + get correctableTokens(): ReadonlyArray { + return this.orderedTokens.filter((t) => this._correctables.find((c) => c.spaceId == t.spaceId)); } - get unboundToken(): ContextToken { - return this._unboundToken; + get predictableToken(): ContextToken { + return this.orderedTokens.find((t) => this._predictable?.spaceId == t.spaceId); } get lockedTokenResults(): ReadonlyMap { - return this._lockedTokenResults; + return new Map([...this._lockedTokenResults.entries()] + .map((tuple) => [this.orderedTokens.find((t) => t.searchModule == tuple[0]), tuple[1]])); } // Will have actual result sequences. @@ -96,24 +98,25 @@ export class TokenizationCorrector implements CorrectionSearchable { + const searchModule = token.searchModule; if(!filterClosure(token)) { - this._lockedTokens.push(token); + this._uncorrectables.push(searchModule); } else if(index == tailCorrectionLength - 1) { - this._unboundToken = token; + this._predictable = searchModule; } else { - this._boundTokens.push(token); + this._correctables.push(searchModule); } }); this._lockedTokenResults = new Map(); - const lockedTokens = this._lockedTokens; - lockedTokens.forEach((t) => { - const lockedResult = t.searchModule.bestExample; - this._lockedTokenResults.set(t, { + const uncorrectables = this._uncorrectables; + uncorrectables.forEach((uncorrectable) => { + const lockedResult = uncorrectable.bestExample; + this._lockedTokenResults.set(uncorrectable, { matchString: lockedResult.text, inputSamplingCost: 0, knownCost: -Math.log(lockedResult.p), @@ -121,20 +124,20 @@ export class TokenizationCorrector implements CorrectionSearchable accum - Math.log(curr.searchModule.bestExample.p), 0); - const tokenCostMap = this.tokenCostMap = new Map(); + let totalCost = uncorrectables.reduce((accum, curr) => accum - Math.log(curr.bestExample.p), 0); + const tokenCostMap = this.tokenCostMap = new Map(); - const tokensToQueue = this._boundTokens.concat(this.unboundToken ?? []); - tokensToQueue.forEach((t) => { - totalCost += t.searchModule.currentCost; - tokenCostMap.set(t, t.searchModule.currentCost); + const correctablesToQueue = this._correctables.concat(this.predictableToken?.searchModule ?? []); + correctablesToQueue.forEach((t) => { + totalCost += t.currentCost; + tokenCostMap.set(t.spaceId, t.currentCost); }); this.lastTotalCost = totalCost; // Compute a weighting for each token's search space based the increase in // tokenization cost that it represents. - const tokenUpdateCost = (token: ContextToken) => token.searchModule.currentCost - (tokenCostMap.get(token) ?? 0) + const tokenUpdateCost = (searchModule: SearchQuotientNode) => searchModule.currentCost - (tokenCostMap.get(searchModule.spaceId) ?? 0) this.selectionQueue = new PriorityQueue((a, b) => { const aUpdateCost = tokenUpdateCost(a); const bUpdateCost = tokenUpdateCost(b); @@ -144,15 +147,15 @@ export class TokenizationCorrector implements CorrectionSearchable this._lockedTokenResults.get(t)), this); + return new TokenizationResultMapping(this.orderedTokens.map((t) => this._lockedTokenResults.get(t.searchModule)), this); } handleNextNode(): PathResult { @@ -167,8 +170,8 @@ export class TokenizationCorrector implements CorrectionSearchable correction-string map with the obtained result. - this._lockedTokenResults.set(tokenToUpdate, tokenResult.mapping); + this._lockedTokenResults.set(correctableToUpdate, tokenResult.mapping); // If we have a correction for all components in need of correction, allow // searching for alternative corrections for the 'unbound' token. - if(this._boundTokens.length == 0 && this._unboundToken) { - this.selectionQueue.enqueue(this._unboundToken); + if(this._correctables.length == 0 && this._predictable) { + this.selectionQueue.enqueue(this._predictable); } const correctionResults = this.collateResults(); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts index 987bee7f2e..4ca7a94343 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts @@ -205,9 +205,9 @@ describe('TokenizationCorrector', () => { fixture.filter ); - assert.sameOrderedMembers(instance.lockedTokens.slice(), []); - assert.sameOrderedMembers(instance.boundTokens.slice(), []); - assert.equal(instance.unboundToken, tokenization.tail); + assert.sameOrderedMembers(instance.uncorrectableTokens.slice(), []); + assert.sameOrderedMembers(instance.correctableTokens.slice(), []); + assert.equal(instance.predictableToken, tokenization.tail); }); it('constructs correctly from a single uncorrectable token', () => { @@ -221,9 +221,9 @@ describe('TokenizationCorrector', () => { fixture.filter ); - assert.sameOrderedMembers(instance.lockedTokens.slice(), [tokenization.tail]); - assert.sameOrderedMembers(instance.boundTokens.slice(), []); - assert.equal(instance.unboundToken, undefined); + assert.sameOrderedMembers(instance.uncorrectableTokens.slice(), [tokenization.tail]); + assert.sameOrderedMembers(instance.correctableTokens.slice(), []); + assert.equal(instance.predictableToken, undefined); }); it('constructs from multiple tokens, with the middle one uncorrectable', () => { @@ -238,9 +238,9 @@ describe('TokenizationCorrector', () => { fixture.filter ); - assert.sameOrderedMembers(instance.lockedTokens.slice(), [tokenization.tokens[tokenCount-2]]); - assert.sameOrderedMembers(instance.boundTokens.slice(), [tokenization.tokens[tokenCount-3]]); - assert.equal(instance.unboundToken, tokenization.tail); + assert.sameOrderedMembers(instance.uncorrectableTokens.slice(), [tokenization.tokens[tokenCount-2]]); + assert.sameOrderedMembers(instance.correctableTokens.slice(), [tokenization.tokens[tokenCount-3]]); + assert.equal(instance.predictableToken, tokenization.tail); }); it('constructs from multiple tokens, ignoring the first due to bounds', () => { @@ -255,9 +255,9 @@ describe('TokenizationCorrector', () => { fixture.filter ); - assert.sameOrderedMembers(instance.lockedTokens.slice(), [tokenization.tokens[tokenCount-2]]); - assert.sameOrderedMembers(instance.boundTokens.slice(), []); - assert.equal(instance.unboundToken, tokenization.tail); + assert.sameOrderedMembers(instance.uncorrectableTokens.slice(), [tokenization.tokens[tokenCount-2]]); + assert.sameOrderedMembers(instance.correctableTokens.slice(), []); + assert.equal(instance.predictableToken, tokenization.tail); }); it('constructs correctly when the final token is uncorrectable', () => { @@ -271,9 +271,9 @@ describe('TokenizationCorrector', () => { fixture.filter ); - assert.sameOrderedMembers(instance.lockedTokens.slice(), [tokenization.tail]); - assert.sameOrderedMembers(instance.boundTokens.slice(), [tokenization.tokens[0]]); - assert.equal(instance.unboundToken, undefined); + assert.sameOrderedMembers(instance.uncorrectableTokens.slice(), [tokenization.tail]); + assert.sameOrderedMembers(instance.correctableTokens.slice(), [tokenization.tokens[0]]); + assert.equal(instance.predictableToken, undefined); }); }); @@ -304,9 +304,9 @@ describe('TokenizationCorrector', () => { assert.sameOrderedMembers(tokenResults.map((r) => r.matchString), ['theref']); // Now that an entry has been found, verify the corrector's state. - assert.isOk(instance.unboundToken); // should not become bound or locked. - assert.isTrue(instance.lockedTokenResults.has(instance.unboundToken)); - assert.equal(instance.lockedTokenResults.get(instance.unboundToken), tokenResults[0]); + assert.isOk(instance.predictableToken); // should not become bound or locked. + assert.isTrue(instance.lockedTokenResults.has(instance.predictableToken)); + assert.equal(instance.lockedTokenResults.get(instance.predictableToken), tokenResults[0]); } searchResult = instance.handleNextNode(); @@ -349,8 +349,8 @@ describe('TokenizationCorrector', () => { } // Now that an entry has been found, verify the corrector's state. - assert.isOk(instance.unboundToken); // should not become bound or locked. - assert.isTrue(instance.lockedTokenResults.has(instance.unboundToken)); + assert.isOk(instance.predictableToken); // should not become bound or locked. + assert.isTrue(instance.lockedTokenResults.has(instance.predictableToken)); for(let i=0; i < firstResults.length; i++) { assert.equal(instance.lockedTokenResults.get(instance.orderedTokens[i]), firstResults[i]); }