diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts index aa64a603df..37e9d1d00b 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts @@ -12,7 +12,6 @@ import { LexicalModelTypes } from '@keymanapp/common-types'; import { CorrectionResultMapping } from './correction-result-mapping.js'; import { SearchNode, TraversableToken } from "./distance-modeler.js"; -import { TokenResult } from './tokenization-corrector.js'; // Circular type reference; do not actually require direct use of the prototype // or constructor! @@ -46,7 +45,7 @@ export function initTokenResultFilterer() { return closure; } -export class TokenResultMapping implements CorrectionResultMapping, TokenResult { +export class TokenResultMapping implements CorrectionResultMapping { readonly matchingSpace: SearchQuotientNode; private readonly node: SearchNode; readonly spaceId: number; diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts deleted file mode 100644 index 444ea9ee85..0000000000 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts +++ /dev/null @@ -1,361 +0,0 @@ -/* - * Keyman is copyright (C) SIL Global. MIT License. - * - * Created by jahorton on 2026-04-02 - * - * This file defines the `TokenizationCorrector` class, which is used to - * prioritize optimal multi-token corrections (and predictions) within the - * predictive-text correction-search engine. - */ - -import { PriorityQueue } from "keyman/common/web-utils"; - -import { ContextToken } from "./context-token.js"; -import { CorrectionSearchable, PathResult } from "./correction-searchable.js"; -import { ContextTokenization } from "./context-tokenization.js"; -import { QuotientNodeFinalizer } from "./quotient-node-finalizer.js"; -import { TokenizationResultMapping } from "./tokenization-result-mapping.js"; -import { EDIT_DISTANCE_COST_SCALE } from "./distance-modeler.js"; -import { MAX_EDIT_THRESHOLD_FACTOR } from "./search-quotient-spur.js"; - -// PathResult needs to be generic: -// - a result for correcting a single Token - "TokenResult"? -// - a result for completing correction for a full Tokenization - "TokenizationResult"? - -/** - * Implements an interface (extended by TokenResultMapping) that represents the - * form (and related probability data) of a token to be utilized for generation - * of predictions. - * - * Notably, this can be instantiated directly from a token without use of - * correction-search while still adhering to an interface compatible with - * correction results. - */ -export type TokenResult = { - matchString: string, - inputSamplingCost: number, - knownCost: number, - totalCost: number -} - -/** - * This class is the focal point for support of whitespace and word-boundary - * correction. It uses the SearchQuotientNode search-spaces of an existing - * tokenization's tokens to optimally prioritize the correction process among - * all correctable tokens, generating corrections for the full represented - * range. - */ -export class TokenizationCorrector implements CorrectionSearchable, TokenizationResultMapping> { - public readonly tokenization: ContextTokenization; - private readonly tailCorrectionLength: number; - - // public read-only via properties - private readonly _uncorrectables: QuotientNodeFinalizer[]; - private readonly _correctables: QuotientNodeFinalizer[]; - private _predictable?: QuotientNodeFinalizer; - private _generatedTokenResults: Map; - private _previousResults: TokenizationResultMapping[] = []; - - // fully private - private selectionQueue: PriorityQueue; - private tokenCostMap: Map; - private tokenLookupMap: Map; - private lastTotalCost: number; - private handleHasBeenCalled: boolean = false; - - get currentCost(): number { - const correctable = this.selectionQueue.peek(); - if(!correctable) { - return this.lastTotalCost; - } - - return this.getUpdatedTotalCost(correctable, correctable.currentCost); - }; - - /** - * Returns the tokens contributing in some manner to correction-search and its weightings. - */ - get orderedTokens(): ReadonlyArray { - return this.tokenization.tokens.slice(-this.tailCorrectionLength); - } - - /** - * Returns the tokens, in order, that are considered "uncorrectable"; correction-search will not - * perform no further text-correction on them. - * - * Note that some tokens may have started out as "correctable" initially, becoming "uncorrectable" after - * their first viable correction was found. - * - * Other tokens may have been labeled "uncorrectable" from the start. - */ - get uncorrectableTokens(): ReadonlyArray { - return this._uncorrectables.map((c) => this.tokenLookupMap.get(c.spaceId)); - } - - /** - * Returns the tokens, in order, that are considered "correctable". - * - * Correction-search will search for only the first viable correction for each - * and will penalize any additional codepoints not already within the token - * that prove necesssary to match a valid lexical entry. - */ - get correctableTokens(): ReadonlyArray { - return this._correctables.map((c) => this.tokenLookupMap.get(c.spaceId)); - } - - /** - * Returns the token, if it exists, that is considered "predictable". - * - * Correction-search will search for any number of corrections for this token - * provided that the list of "correctables" has been exhausted. If the list - * of "correctables" still has entries, once an initial correction is found - * for this token, correction will be suspended until the correctables list - * is empty. - */ - get predictableToken(): ContextToken { - return this.tokenLookupMap.get(this._predictable?.spaceId); - } - - /** - * Returns the current map of token-to-corrections that has been determined thus far. - * - * Tokens initially considered "uncorrectable" will have valid, pre-set entries. - */ - get generatedTokenResults(): ReadonlyMap { - return new Map([...this._generatedTokenResults.entries()] - .map((tuple) => [this.tokenLookupMap.get(tuple[0]), tuple[1]])); - } - - // Will have actual result sequences. - // - // Once we have an actual answer for all non-locked tokens, the first entry - // should appear. The only variation among results, after that, should be the - // correction for the last token. - // - // Results may be a clone of the lockedTokenResults map. The owning - // tokenization may then flesh out the ordering of the entries to build the - // proper corrections / predictions. - get previousResults(): TokenizationResultMapping[] { - return this._previousResults; - }; - - /** - * Constructs an instance of TokenizationCorrector for finding corrections for - * correctable tokens within the specified section of an existing - * ContextTokenization. - * @param tokenization The tokenization pattern under consideration, - * containing tokens that may be correctable - * @param tailCorrectionLength The length, in tokens, at the end of the - * tokenization pattern that should be considered for correction - * @param filterClosure A closure that indicates via boolean whether to permit correction - * for each token - */ - constructor( - tokenization: ContextTokenization, - tailCorrectionLength: number, - filterClosure: (token: ContextToken) => boolean - ) { - this.tokenization = tokenization; - this.tailCorrectionLength = tailCorrectionLength; - - if(tailCorrectionLength < 1) { - throw new Error(`Length for correction near tail may not be ${tailCorrectionLength} - it must be a positive number.`); - } else if(tailCorrectionLength > tokenization.tokens.length) { - throw new Error(`Tail correction length must not extend actual token count - ${tailCorrectionLength} > ${tokenization.tokens.length}`); - } - - const orderedTokens = this.orderedTokens; - - this._uncorrectables = []; - this._correctables = []; - - this.tokenLookupMap = new Map(); - - orderedTokens.forEach((token, index) => { - // New issue: this mangles the space IDs! We almost certainly need some - // sort of proper map to the source token. - const searchModule = new QuotientNodeFinalizer(token.searchModule, index == orderedTokens.length - 1); - this.tokenLookupMap.set(searchModule.spaceId, token); - if(!filterClosure(token)) { - this._uncorrectables.push(searchModule); - } else if(index == tailCorrectionLength - 1) { - // The sole assignment case for this field. It may only be assigned for - // the final token, and only if its text is of a form considered - // correctable by the filter. - this._predictable = searchModule; - } else { - this._correctables.push(searchModule); - } - }); - - this._generatedTokenResults = new Map(); - const uncorrectables = this._uncorrectables; - uncorrectables.forEach((uncorrectable) => { - const lockedResult = uncorrectable.bestExample; - this._generatedTokenResults.set(uncorrectable.spaceId, { - matchString: lockedResult.text, - inputSamplingCost: -Math.log(lockedResult.p), - knownCost: 0, - totalCost: -Math.log(lockedResult.p) - }); - }); - - let totalCost = uncorrectables.reduce((accum, curr) => accum - Math.log(curr.bestExample.p), 0); - const tokenCostMap = this.tokenCostMap = new Map(); - - const correctablesToQueue = this._correctables.concat(this._predictable ?? []); - correctablesToQueue.forEach((t) => { - totalCost += t.currentCost; - tokenCostMap.set(t.spaceId, t.currentCost); - }); - - this.lastTotalCost = totalCost; - - // Compute a weighting for each token's search space based the increase in - // tokenization cost that it represents. - const tokenUpdateCost = (searchModule: QuotientNodeFinalizer) => searchModule.currentCost - (tokenCostMap.get(searchModule.spaceId) ?? 0) - this.selectionQueue = new PriorityQueue((a, b) => { - const aUpdateCost = tokenUpdateCost(a); - const bUpdateCost = tokenUpdateCost(b); - - // Division or subtraction, we get the same effect for ordering: the - // operands are all positive. Subtraction is computationally less costly. - return aUpdateCost - bUpdateCost; - }); - - this.selectionQueue.enqueueAll(correctablesToQueue); - } - - private getUpdatedTotalCost(updatedCorrectable: QuotientNodeFinalizer, tokenCost: number): number { - return this.lastTotalCost + tokenCost - (this.tokenCostMap.get(updatedCorrectable.spaceId) ?? 0); - } - - /** - * Converts the internal 'generated token results' map into the proper Tokenization-correction return type. - */ - private collateResults(): TokenizationResultMapping { - // The tokenLookupMap was constructed in the same ordering as the tokens; we can iterate the keys - // or entries to keep everything in order. - const results = [...this.tokenLookupMap.keys()].map((spaceId) => this._generatedTokenResults.get(spaceId)) - return new TokenizationResultMapping(results, this); - } - - // The actual method used to iteratively search for tokenization-level corrections. - handleNextNode(): PathResult { - // Notable states: - // 1. Unbound tokens have not yet been "locked" - no valid correction has yet been found. - // - Variation: the final, "unbound" token may be locked while awaiting this case. - // - If so, remember the corresponding matchString! - // 2. An unbound token may become "locked" - a workable correction is found. - // - Remember the correction's matchString / correction! - // 3. The **last** unbound token may finally become "locked". - // - If final "unbound" token is locked, unlock it! - // - Produce first search result! - // 4. Unbound token is unlocked, but all others are locked. - - if(this.selectionQueue.count == 0) { - // If we reach this point, the tokenization has exhausted its search space. - if(this.handleHasBeenCalled) { - return { type: 'none' }; - } else { - // It is possible that the editable tokenization range exists entirely of - // tokens considered to be uncorrectable. - this.handleHasBeenCalled = true; - const results = this.collateResults(); - this._previousResults.push(results); - return { - 'type': 'complete', - cost: this.lastTotalCost, - mapping: results - }; - } - } - - this.handleHasBeenCalled = true; - - const correctableToUpdate = this.selectionQueue.dequeue(); - const tokenResult = correctableToUpdate?.handleNextNode(); - - const delistCorrectable = () => { - if(correctableToUpdate != this._predictable) { - // Lock the 'correctable' token now that either a valid correction for - // it has been found or all possible corrections are exhausted. We only - // consider a single correction for most of a tokenization's tokens, - // generally only allowing correction variation for the last represented - // token. - this._correctables.splice(this._correctables.indexOf(correctableToUpdate), 1); - this._uncorrectables.push(correctableToUpdate); - } - } - - if(tokenResult.type == 'none') { - // Transition the node from 'correctable' to 'uncorrectable' - we were - // unable to find valid corrections for it. - const lockedResult = correctableToUpdate.bestExample; - this._generatedTokenResults.set(correctableToUpdate.spaceId, { - matchString: lockedResult.text, - inputSamplingCost: -Math.log(lockedResult.p), - knownCost: MAX_EDIT_THRESHOLD_FACTOR, // we'll use the same threshold at which further search is terminated. - totalCost: -Math.log(lockedResult.p) + MAX_EDIT_THRESHOLD_FACTOR * EDIT_DISTANCE_COST_SCALE - }); - - // We can make no further predictions if we've exhausted all search options. - if(correctableToUpdate == this._predictable) { - this._uncorrectables.push(correctableToUpdate); - delete this._predictable; - } else { - delistCorrectable(); - } - } else if(tokenResult.type == 'complete') { - // Note that at this stage, we do not requeue the 'predictable' - other - // correctables may exist and need their first corrections before we look - // for other corrective variations of the 'predictable'. - delistCorrectable(); - - // Either way, update the token -> correction-string map with the obtained result. - this._generatedTokenResults.set(correctableToUpdate.spaceId, tokenResult.mapping); - } - - const resultCost = tokenResult.type != 'none' ? tokenResult.cost : this._generatedTokenResults.get(correctableToUpdate.spaceId).totalCost; - - // Update the cost associated with the token. - const tokenizationCost = this.lastTotalCost = this.getUpdatedTotalCost(correctableToUpdate, resultCost); - this.tokenCostMap.set(correctableToUpdate.spaceId, resultCost); - - // If we haven't found a valid correction for the token with lowest-cost update, - // just requeue it and keep searching until we find one. - if(tokenResult.type == 'intermediate') { - this.selectionQueue.enqueue(correctableToUpdate); - - // Needs to return the 'proper' type of result. - return { - type: 'intermediate', - cost: tokenizationCost - }; - } - - // If we have a correction for all components in need of correction, then - // search for alternative corrections for the 'predictable' token - even if - // we previously stopped searching for more because we found its first - // correction before finding one for at least one other 'correctable'. - if(this._correctables.length == 0 && this.selectionQueue.count == 0 && this._predictable) { - this.selectionQueue.enqueue(this._predictable); - } - - const correctionResults = this.collateResults(); - if(correctionResults.matchedResult.findIndex((c) => c == undefined) != -1) { - return { - type: 'intermediate', - cost: tokenizationCost - }; - } - - // Determine the proper return type and construct the proper return object accordingly. - this._previousResults.push(correctionResults); - return { - type: 'complete', - cost: tokenizationCost, - mapping: correctionResults - }; - } -} \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-result-mapping.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-result-mapping.ts deleted file mode 100644 index 32e0fb48fc..0000000000 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-result-mapping.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { CorrectionResultMapping } from "./correction-result-mapping.js"; -import { TokenizationCorrector, TokenResult } from './tokenization-corrector.js'; - -export class TokenizationResultMapping implements CorrectionResultMapping> { - readonly matchingSpace: TokenizationCorrector; - readonly matchedResult: ReadonlyArray; - - constructor(tokenization: TokenResult[], corrector: TokenizationCorrector) { - this.matchingSpace = corrector; - this.matchedResult = tokenization; - } - - get spaceId(): number { - return this.matchingSpace.tokenization.spaceId; - } - - // /** - // * Gets the number of Damerau-Levenshtein edits needed to reach the node's - // * matchString from the output induced by the input sequence used to reach it. - // * - // * (This is scaled by `SearchSpace.EDIT_DISTANCE_COST_SCALE` when included in - // * `totalCost`.) - // */ - // get knownCost(): number { - // return this.node.editCount; - // } - - // /** - // * Gets the "input sampling cost" of the edge, which should be considered as the - // * negative log-likelihood of the input path taken to reach the node. - // */ - // get inputSamplingCost(): number { - // return this.node.inputSamplingCost; - // } - - /** - * Gets the "total cost" of the edge, which should be considered as the - * negative log-likelihood of the input path taken to reach the node - * multiplied by the 'probability' induced by needed Damerau-Levenshtein edits - * to the resulting output. - */ - get totalCost(): number { - return this.matchedResult.reduce((total, curr) => total + curr.totalCost, 0); - } -} \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts b/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts index bcaf4692bf..41229d0526 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts @@ -16,11 +16,9 @@ export * from './correction/legacy-quotient-spur.js'; export * from './correction/quotient-node-finalizer.js'; export * from './correction/search-quotient-root.js'; export { ExtendedEditOperation, SegmentableDistanceCalculation } from './correction/segmentable-calculation.js'; -export * from './correction/tokenization-corrector.js'; export * from './correction/tokenization-subsets.js'; export * from './correction/transition-helpers.js'; export * from './correction/token-result-mapping.js'; -export * from './correction/tokenization-result-mapping.js'; export { determinePunctuationFromModel, determineModelWordbreaker, determineModelTokenizer, detectCurrentCasing diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts deleted file mode 100644 index 711f494f08..0000000000 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts +++ /dev/null @@ -1,468 +0,0 @@ -/* - * Keyman is copyright (C) SIL Global. MIT License. - * - * Created by jahorton on 2026-04-14 - * - * This file defines tests for the `TokenizationCorrector` class, which is used - * to prioritize optimal multi-token corrections (and predictions) within the - * predictive-text correction-search engine. - */ - -import { assert } from 'chai'; - -import { LexicalModelTypes } from '@keymanapp/common-types'; -import { default as defaultBreaker } from '@keymanapp/models-wordbreakers'; -import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; - -import { - ContextToken, - ContextTokenization, - correctionValidForAutoSelect, - ExecutionTimer, - generateSubsetId, - getBestMatches, - LegacyQuotientSpur, - models, - PathInputProperties, - PathResult, - SearchQuotientNode, - SearchQuotientRoot, - TokenizationCorrector, - TokenResult, - TokenizationResultMapping -} from '@keymanapp/lm-worker/test-index'; - -import Distribution = LexicalModelTypes.Distribution; -import TrieModel = models.TrieModel; -import Transform = LexicalModelTypes.Transform; - -const plainModel = new TrieModel( - jsonFixture('models/tries/english-1000'), { - languageUsesCasing: true, - wordBreaker: defaultBreaker - } -); - -function buildTestTimer() { - return new ExecutionTimer(Number.MAX_VALUE, Number.MAX_VALUE); -} - -function buildFixture_therefore() { - let ID_SEED = 11; - - const distributionSrc: [string, number][][] = [ - [ ['t', 1] ], - [ ['h', 1] ], - [ ['e', 0.6] ], - [ [' ', 0.8], ['r', 0.2] ], - [ ['e', 1] ], - [ ['f', 1] ] - ]; - - const distributions: Distribution>[] = distributionSrc.map((tupleArray) => { - const transitionId = ID_SEED++; - return tupleArray.map((tuple) => { - return { - p: tuple[1], - sample: { - insert: tuple[0], - deleteLeft: 0, - deleteRight: 0, - id: transitionId - } - } - }); - }); - - // Assumes that the first entry in each distribution is the most likely. - const inputSources: PathInputProperties[] = distributions.map((dist) => { - return { - subsetId: generateSubsetId(), - segment: {start: 0, transitionId: dist[0].sample.id}, - bestProbFromSet: dist[0].p - }; - }) - - const therefTokens: ContextToken[] = []; // as in "therefore" - const the_efTokens: ContextToken[] = []; // as in "the effect" - - // TODO: Use SubstitutionQuotientSpur instead! - let firstTokenNode: SearchQuotientNode = new SearchQuotientRoot(plainModel); - for(let i=0; i < 3; i++) { - firstTokenNode = new LegacyQuotientSpur(firstTokenNode, distributions[i], inputSources[i]); - } - - the_efTokens.push(new ContextToken(firstTokenNode, false)); - - firstTokenNode = new LegacyQuotientSpur(firstTokenNode, [distributions[3][1]], { - ...inputSources[3], - subsetId: generateSubsetId() - }); - - // whitespace token alternate - using the ' ' input instead. - const whitespaceToken = new ContextToken( - new LegacyQuotientSpur( - new SearchQuotientRoot(plainModel), - [distributions[3][0]], - { ...inputSources[3], subsetId: generateSubsetId() } - ), false - ); - whitespaceToken.isWhitespace = true; - the_efTokens.push(whitespaceToken); - - let secondTokenNode: SearchQuotientNode = new SearchQuotientRoot(plainModel); - for(let i=4; i < distributions.length; i++) { - firstTokenNode = new LegacyQuotientSpur(firstTokenNode, distributions[i], { - ...inputSources[i], - subsetId: generateSubsetId() - }); - secondTokenNode = new LegacyQuotientSpur(secondTokenNode, distributions[i], { - ...inputSources[i], - subsetId: generateSubsetId() - }) - } - - therefTokens.push(new ContextToken(firstTokenNode)); - the_efTokens.push(new ContextToken(secondTokenNode)); - - return { - filter: (token: ContextToken) => correctionValidForAutoSelect(token.exampleInput), - theref: new ContextTokenization(therefTokens), - the_ef: new ContextTokenization(the_efTokens) - } -} - -function buildFixture_terminalWhitespace() { - let ID_SEED = 11; - - const distributionSrc: [string, number][][] = [ - [ ['s', 1] ], - [ ['p', 1] ], - [ ['a', 1] ], - [ ['c', 1] ], - [ ['e', 1] ], - [ [' ', 1] ], - ]; - - const distributions: Distribution>[] = distributionSrc.map((tupleArray) => { - const transitionId = ID_SEED++; - return tupleArray.map((tuple) => { - return { - p: tuple[1], - sample: { - insert: tuple[0], - deleteLeft: 0, - deleteRight: 0, - id: transitionId - } - } - }); - }); - - // Assumes that the first entry in each distribution is the most likely. - const inputSources: PathInputProperties[] = distributions.map((dist) => { - return { - subsetId: generateSubsetId(), - segment: {start: 0, transitionId: dist[0].sample.id}, - bestProbFromSet: dist[0].p - }; - }) - - const fullTokens: ContextToken[] = []; - const lastToken: ContextToken[] = []; - - // TODO: Use SubstitutionQuotientSpur instead! - let firstTokenNode: SearchQuotientNode = new SearchQuotientRoot(plainModel); - for(let i=0; i < 5; i++) { - firstTokenNode = new LegacyQuotientSpur(firstTokenNode, distributions[i], inputSources[i]); - } - - fullTokens.push(new ContextToken(firstTokenNode, false)); - - // whitespace token alternate - using the ' ' input instead. - const whitespaceToken = new ContextToken( - new LegacyQuotientSpur( - new SearchQuotientRoot(plainModel), - distributions[5], - inputSources[5], - ), false - ); - whitespaceToken.isWhitespace = true; - fullTokens.push(whitespaceToken); - lastToken.push(whitespaceToken); - - return { - filter: (token: ContextToken) => correctionValidForAutoSelect(token.exampleInput), - wordThenSpace: new ContextTokenization(fullTokens), - spaceOnly: new ContextTokenization(lastToken) - } -} - -describe('TokenizationCorrector', () => { - describe('constructor', () => { - it('constructs correctly from a single correctable token', () => { - const fixture = buildFixture_therefore(); - - const tokenization = fixture.theref; - - const instance = new TokenizationCorrector( - tokenization, - 1, - fixture.filter - ); - - assert.sameOrderedMembers(instance.uncorrectableTokens.slice(), []); - assert.sameOrderedMembers(instance.correctableTokens.slice(), []); - assert.equal(instance.predictableToken, tokenization.tail); - }); - - it('constructs correctly from a single uncorrectable token', () => { - const fixture = buildFixture_terminalWhitespace(); - - const tokenization = fixture.spaceOnly; - - const instance = new TokenizationCorrector( - tokenization, - tokenization.tokens.length, - fixture.filter - ); - - assert.sameOrderedMembers(instance.uncorrectableTokens.slice(), [tokenization.tail]); - assert.sameOrderedMembers(instance.correctableTokens.slice(), []); - assert.equal(instance.predictableToken, undefined); - }); - - it('constructs from multiple tokens, with the middle one uncorrectable', () => { - const fixture = buildFixture_therefore(); - - const tokenization = fixture.the_ef; - const tokenCount = tokenization.tokens.length; - - const instance = new TokenizationCorrector( - tokenization, - tokenCount, - fixture.filter - ); - - assert.sameOrderedMembers(instance.uncorrectableTokens.slice(), [tokenization.tokens[tokenCount-2]]); - assert.sameOrderedMembers(instance.correctableTokens.slice(), [tokenization.tokens[tokenCount-3]]); - assert.equal(instance.predictableToken, tokenization.tail); - }); - - it('constructs from multiple tokens, ignoring the first due to bounds', () => { - const fixture = buildFixture_therefore(); - - const tokenization = fixture.the_ef; - const tokenCount = tokenization.tokens.length; - - const instance = new TokenizationCorrector( - tokenization, - tokenCount-1, - fixture.filter - ); - - assert.sameOrderedMembers(instance.uncorrectableTokens.slice(), [tokenization.tokens[tokenCount-2]]); - assert.sameOrderedMembers(instance.correctableTokens.slice(), []); - assert.equal(instance.predictableToken, tokenization.tail); - }); - - it('constructs correctly when the final token is uncorrectable', () => { - const fixture = buildFixture_terminalWhitespace(); - - const tokenization = fixture.wordThenSpace; - - const instance = new TokenizationCorrector( - tokenization, - tokenization.tokens.length, - fixture.filter - ); - - assert.sameOrderedMembers(instance.uncorrectableTokens.slice(), [tokenization.tail]); - assert.sameOrderedMembers(instance.correctableTokens.slice(), [tokenization.tokens[0]]); - assert.equal(instance.predictableToken, undefined); - }); - }); - - describe('handleNextNode', () => { - it('finds corrections for a single correctable token', () => { - const fixture = buildFixture_therefore(); - - const tokenization = fixture.theref; - - const instance = new TokenizationCorrector( - tokenization, - 1, - fixture.filter - ); - - let searchResult: PathResult; - do { - searchResult = instance.handleNextNode(); - } while(searchResult.type == 'intermediate'); - - assert.equal(searchResult.type, 'complete'); - if(searchResult.type == 'complete') { - const mapping = searchResult.mapping; - const tokenResults = mapping.matchedResult; - assert.isNotNaN(searchResult.cost); - assert.equal(searchResult.cost, searchResult.mapping.totalCost); - assert.equal(tokenResults.length, 1); - assert.sameOrderedMembers(tokenResults.map((r) => r.matchString), ['theref']); - - // Now that an entry has been found, verify the corrector's state. - assert.isOk(instance.predictableToken); // should not become bound or locked. - assert.isTrue(instance.generatedTokenResults.has(instance.predictableToken)); - assert.equal(instance.generatedTokenResults.get(instance.predictableToken), tokenResults[0]); - } - - searchResult = instance.handleNextNode(); - // There should be more results that may be found. - assert.notEqual(searchResult.type, 'none'); - - do { - searchResult = instance.handleNextNode(); - } while(searchResult.type == 'intermediate'); - - assert.notEqual(searchResult.type, 'none'); - }); - - it('finds corrections for a group of tokens with two correctable', () => { - const fixture = buildFixture_therefore(); - - const tokenization = fixture.the_ef; - - const instance = new TokenizationCorrector( - tokenization, - 3, - fixture.filter - ); - - let searchResult: PathResult; - do { - searchResult = instance.handleNextNode(); - } while(searchResult.type == 'intermediate'); - - assert.equal(searchResult.type, 'complete'); - let firstResults: ReadonlyArray; - if(searchResult.type == 'complete') { - const mapping = searchResult.mapping; - const tokenResults = mapping.matchedResult; - firstResults = tokenResults; - assert.isNotNaN(searchResult.cost); - assert.equal(searchResult.cost, searchResult.mapping.totalCost); - assert.equal(tokenResults.length, 3); - assert.sameOrderedMembers(tokenResults.map((r) => r.matchString), ['the', ' ', 'ef']); - } - - // Now that an entry has been found, verify the corrector's state. - assert.isOk(instance.predictableToken); // should not become bound or locked. - assert.isTrue(instance.generatedTokenResults.has(instance.predictableToken)); - for(let i=0; i < firstResults.length; i++) { - assert.equal(instance.generatedTokenResults.get(instance.orderedTokens[i]), firstResults[i]); - } - - searchResult = instance.handleNextNode(); - // There should be more results that may be found. - assert.notEqual(searchResult.type, 'none'); - - do { - searchResult = instance.handleNextNode(); - if(searchResult.type == 'complete') { - const mapping = searchResult.mapping; - const tokenResults = mapping.matchedResult; - - // Verify that the first (bound) token is not altered further. - // It should receive no further correction attempts. - assert.equal(tokenResults[0], firstResults[0]); - assert.equal(tokenResults[1], firstResults[1]); - assert.notEqual(tokenResults[2], firstResults[2]); - } - } while(searchResult.type != 'none'); - }); - - it('immediately returns a single result when the only represented token is uncorrectable', () => { - const fixture = buildFixture_terminalWhitespace(); - - const tokenization = fixture.spaceOnly; - - const instance = new TokenizationCorrector( - tokenization, - tokenization.tokens.length, - fixture.filter - ); - - const searchResult = instance.handleNextNode(); - assert.equal(searchResult.type, 'complete'); - if(searchResult.type == 'complete') { - assert.equal(searchResult.mapping.matchedResult[0].matchString, ' '); - } - - const nilResult = instance.handleNextNode(); - assert.equal(nilResult.type, 'none'); - }); - - it('returns a single result when the final token is uncorrectable', () => { - const fixture = buildFixture_terminalWhitespace(); - - const tokenization = fixture.wordThenSpace; - - const instance = new TokenizationCorrector( - tokenization, - tokenization.tokens.length, - fixture.filter - ); - - let searchResult: PathResult; - do { - searchResult = instance.handleNextNode(); - } while(searchResult.type == 'intermediate'); - - assert.equal(searchResult.type, 'complete'); - if(searchResult.type == 'complete') { - assert.equal(searchResult.mapping.matchedResult[0].matchString, 'space'); - assert.equal(searchResult.mapping.matchedResult[1].matchString, ' '); - } - - const nilResult = instance.handleNextNode(); - assert.equal(nilResult.type, 'none'); - }); - - describe('with getBestMatches()', () => { - it('finds results from each of two tokenization variants sharing lower-level SearchQuotientNodes', async () => { - const fixture = buildFixture_therefore(); - - const tokenizations = [fixture.theref, fixture.the_ef]; - const correctors = tokenizations.map((t) => new TokenizationCorrector(t, t.tokens.length, fixture.filter)); - - let haveSeenSingleTokenCorrection = false; - let haveSeenThreeTokenCorrection = false; - for await(let phraseMatch of getBestMatches< - ReadonlyArray, - TokenizationResultMapping, - TokenizationCorrector - >(correctors, buildTestTimer())) { - - if(phraseMatch.matchedResult.length == 1) { - if(!haveSeenSingleTokenCorrection) { - assert.sameOrderedMembers(phraseMatch.matchedResult.map((t) => t.matchString), ['theref' /* -ore */]); - } - - haveSeenSingleTokenCorrection = true; - } else if(phraseMatch.matchedResult.length == 3) { - if(!haveSeenThreeTokenCorrection) { - assert.sameOrderedMembers(phraseMatch.matchedResult.map((t) => t.matchString), ['the', ' ', 'ef' /* -fort */]); - } - haveSeenThreeTokenCorrection = true; - } - - if(haveSeenSingleTokenCorrection && haveSeenThreeTokenCorrection) { - break; - } - } - - assert.isTrue(haveSeenSingleTokenCorrection, 'A single-token correction was expected but not found'); - assert.isTrue(haveSeenThreeTokenCorrection, 'A three-token correction was expected but not found'); - }); - }); - }); -}); \ No newline at end of file