erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-09-19 06:07:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

Merge pull request #16581 from keymanapp/change/web/remove-tokenization-corrector

Browse source 
change(web): remove TokenizationCorrector class only used in epic/boundary-correction
This commit is contained in:
Joshua Horton 2026-09-18 20:08:35 +07:00 • committed by GitHub
commit 00a9b66b01
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 1 additions and 878 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
web/src/engine/predictive-text/worker-thread/src/main/correction/token-result-mapping.ts
View file

@ -12,7 +12,6 @@ import { LexicalModelTypes } from '@keymanapp/common-types';
import { CorrectionResultMapping } from './correction-result-mapping.js';
import { SearchNode, TraversableToken } from "./distance-modeler.js";
import { TokenResult } from './tokenization-corrector.js';
// Circular type reference; do not actually require direct use of the prototype
// or constructor!
@ -46,7 +45,7 @@ export function initTokenResultFilterer() {
return closure;
}
export class TokenResultMapping implements CorrectionResultMapping<SearchNode>, TokenResult {
export class TokenResultMapping implements CorrectionResultMapping<SearchNode> {
readonly matchingSpace: SearchQuotientNode;
private readonly node: SearchNode;
readonly spaceId: number;

361
web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts
View file

@ -1,361 +0,0 @@
/*
* Keyman is copyright (C) SIL Global. MIT License.
*
* Created by jahorton on 2026-04-02
*
* This file defines the `TokenizationCorrector` class, which is used to
* prioritize optimal multi-token corrections (and predictions) within the
* predictive-text correction-search engine.
*/
import { PriorityQueue } from "keyman/common/web-utils";
import { ContextToken } from "./context-token.js";
import { CorrectionSearchable, PathResult } from "./correction-searchable.js";
import { ContextTokenization } from "./context-tokenization.js";
import { QuotientNodeFinalizer } from "./quotient-node-finalizer.js";
import { TokenizationResultMapping } from "./tokenization-result-mapping.js";
import { EDIT_DISTANCE_COST_SCALE } from "./distance-modeler.js";
import { MAX_EDIT_THRESHOLD_FACTOR } from "./search-quotient-spur.js";
// PathResult needs to be generic:
// - a result for correcting a single Token - "TokenResult"?
// - a result for completing correction for a full Tokenization - "TokenizationResult"?
/**
* Implements an interface (extended by TokenResultMapping) that represents the
* form (and related probability data) of a token to be utilized for generation
* of predictions.
*
* Notably, this can be instantiated directly from a token without use of
* correction-search while still adhering to an interface compatible with
* correction results.
*/
export type TokenResult = {
matchString: string,
inputSamplingCost: number,
knownCost: number,
totalCost: number
}
/**
* This class is the focal point for support of whitespace and word-boundary
* correction. It uses the SearchQuotientNode search-spaces of an existing
* tokenization's tokens to optimally prioritize the correction process among
* all correctable tokens, generating corrections for the full represented
* range.
*/
export class TokenizationCorrector implements CorrectionSearchable<ReadonlyArray<TokenResult>, TokenizationResultMapping> {
public readonly tokenization: ContextTokenization;
private readonly tailCorrectionLength: number;
// public read-only via properties
private readonly _uncorrectables: QuotientNodeFinalizer[];
private readonly _correctables: QuotientNodeFinalizer[];
private _predictable?: QuotientNodeFinalizer;
private _generatedTokenResults: Map<number, TokenResult>;
private _previousResults: TokenizationResultMapping[] = [];
// fully private
private selectionQueue: PriorityQueue<QuotientNodeFinalizer>;
private tokenCostMap: Map<number, number>;
private tokenLookupMap: Map<number, ContextToken>;
private lastTotalCost: number;
private handleHasBeenCalled: boolean = false;
get currentCost(): number {
const correctable = this.selectionQueue.peek();
if(!correctable) {
return this.lastTotalCost;
}
return this.getUpdatedTotalCost(correctable, correctable.currentCost);
};
/**
* Returns the tokens contributing in some manner to correction-search and its weightings.
*/
get orderedTokens(): ReadonlyArray<ContextToken> {
return this.tokenization.tokens.slice(-this.tailCorrectionLength);
}
/**
* Returns the tokens, in order, that are considered "uncorrectable"; correction-search will not
* perform no further text-correction on them.
*
* Note that some tokens may have started out as "correctable" initially, becoming "uncorrectable" after
* their first viable correction was found.
*
* Other tokens may have been labeled "uncorrectable" from the start.
*/
get uncorrectableTokens(): ReadonlyArray<ContextToken> {
return this._uncorrectables.map((c) => this.tokenLookupMap.get(c.spaceId));
}
/**
* Returns the tokens, in order, that are considered "correctable".
*
* Correction-search will search for only the first viable correction for each
* and will penalize any additional codepoints not already within the token
* that prove necesssary to match a valid lexical entry.
*/
get correctableTokens(): ReadonlyArray<ContextToken> {
return this._correctables.map((c) => this.tokenLookupMap.get(c.spaceId));
}
/**
* Returns the token, if it exists, that is considered "predictable".
*
* Correction-search will search for any number of corrections for this token
* provided that the list of "correctables" has been exhausted. If the list
* of "correctables" still has entries, once an initial correction is found
* for this token, correction will be suspended until the correctables list
* is empty.
*/
get predictableToken(): ContextToken {
return this.tokenLookupMap.get(this._predictable?.spaceId);
}
/**
* Returns the current map of token-to-corrections that has been determined thus far.
*
* Tokens initially considered "uncorrectable" will have valid, pre-set entries.
*/
get generatedTokenResults(): ReadonlyMap<ContextToken, TokenResult> {
return new Map([...this._generatedTokenResults.entries()]
.map((tuple) => [this.tokenLookupMap.get(tuple[0]), tuple[1]]));
}
// Will have actual result sequences.
//
// Once we have an actual answer for all non-locked tokens, the first entry
// should appear. The only variation among results, after that, should be the
// correction for the last token.
//
// Results may be a clone of the lockedTokenResults map. The owning
// tokenization may then flesh out the ordering of the entries to build the
// proper corrections / predictions.
get previousResults(): TokenizationResultMapping[] {
return this._previousResults;
};
/**
* Constructs an instance of TokenizationCorrector for finding corrections for
* correctable tokens within the specified section of an existing
* ContextTokenization.
* @param tokenization The tokenization pattern under consideration,
* containing tokens that may be correctable
* @param tailCorrectionLength The length, in tokens, at the end of the
* tokenization pattern that should be considered for correction
* @param filterClosure A closure that indicates via boolean whether to permit correction
* for each token
*/
constructor(
tokenization: ContextTokenization,
tailCorrectionLength: number,
filterClosure: (token: ContextToken) => boolean
) {
this.tokenization = tokenization;
this.tailCorrectionLength = tailCorrectionLength;
if(tailCorrectionLength < 1) {
throw new Error(`Length for correction near tail may not be ${tailCorrectionLength} - it must be a positive number.`);
} else if(tailCorrectionLength > tokenization.tokens.length) {
throw new Error(`Tail correction length must not extend actual token count - ${tailCorrectionLength} > ${tokenization.tokens.length}`);
}
const orderedTokens = this.orderedTokens;
this._uncorrectables = [];
this._correctables = [];
this.tokenLookupMap = new Map();
orderedTokens.forEach((token, index) => {
// New issue: this mangles the space IDs! We almost certainly need some
// sort of proper map to the source token.
const searchModule = new QuotientNodeFinalizer(token.searchModule, index == orderedTokens.length - 1);
this.tokenLookupMap.set(searchModule.spaceId, token);
if(!filterClosure(token)) {
this._uncorrectables.push(searchModule);
} else if(index == tailCorrectionLength - 1) {
// The sole assignment case for this field. It may only be assigned for
// the final token, and only if its text is of a form considered
// correctable by the filter.
this._predictable = searchModule;
} else {
this._correctables.push(searchModule);
}
});
this._generatedTokenResults = new Map();
const uncorrectables = this._uncorrectables;
uncorrectables.forEach((uncorrectable) => {
const lockedResult = uncorrectable.bestExample;
this._generatedTokenResults.set(uncorrectable.spaceId, {
matchString: lockedResult.text,
inputSamplingCost: -Math.log(lockedResult.p),
knownCost: 0,
totalCost: -Math.log(lockedResult.p)
});
});
let totalCost = uncorrectables.reduce((accum, curr) => accum - Math.log(curr.bestExample.p), 0);
const tokenCostMap = this.tokenCostMap = new Map<number, number>();
const correctablesToQueue = this._correctables.concat(this._predictable ?? []);
correctablesToQueue.forEach((t) => {
totalCost += t.currentCost;
tokenCostMap.set(t.spaceId, t.currentCost);
});
this.lastTotalCost = totalCost;
// Compute a weighting for each token's search space based the increase in
// tokenization cost that it represents.
const tokenUpdateCost = (searchModule: QuotientNodeFinalizer) => searchModule.currentCost - (tokenCostMap.get(searchModule.spaceId) ?? 0)
this.selectionQueue = new PriorityQueue((a, b) => {
const aUpdateCost = tokenUpdateCost(a);
const bUpdateCost = tokenUpdateCost(b);
// Division or subtraction, we get the same effect for ordering: the
// operands are all positive. Subtraction is computationally less costly.
return aUpdateCost - bUpdateCost;
});
this.selectionQueue.enqueueAll(correctablesToQueue);
}
private getUpdatedTotalCost(updatedCorrectable: QuotientNodeFinalizer, tokenCost: number): number {
return this.lastTotalCost + tokenCost - (this.tokenCostMap.get(updatedCorrectable.spaceId) ?? 0);
}
/**
* Converts the internal 'generated token results' map into the proper Tokenization-correction return type.
*/
private collateResults(): TokenizationResultMapping {
// The tokenLookupMap was constructed in the same ordering as the tokens; we can iterate the keys
// or entries to keep everything in order.
const results = [...this.tokenLookupMap.keys()].map((spaceId) => this._generatedTokenResults.get(spaceId))
return new TokenizationResultMapping(results, this);
}
// The actual method used to iteratively search for tokenization-level corrections.
handleNextNode(): PathResult<TokenizationResultMapping> {
// Notable states:
// 1. Unbound tokens have not yet been "locked" - no valid correction has yet been found.
// - Variation: the final, "unbound" token may be locked while awaiting this case.
// - If so, remember the corresponding matchString!
// 2. An unbound token may become "locked" - a workable correction is found.
// - Remember the correction's matchString / correction!
// 3. The **last** unbound token may finally become "locked".
// - If final "unbound" token is locked, unlock it!
// - Produce first search result!
// 4. Unbound token is unlocked, but all others are locked.
if(this.selectionQueue.count == 0) {
// If we reach this point, the tokenization has exhausted its search space.
if(this.handleHasBeenCalled) {
return { type: 'none' };
} else {
// It is possible that the editable tokenization range exists entirely of
// tokens considered to be uncorrectable.
this.handleHasBeenCalled = true;
const results = this.collateResults();
this._previousResults.push(results);
return {
'type': 'complete',
cost: this.lastTotalCost,
mapping: results
};
}
}
this.handleHasBeenCalled = true;
const correctableToUpdate = this.selectionQueue.dequeue();
const tokenResult = correctableToUpdate?.handleNextNode();
const delistCorrectable = () => {
if(correctableToUpdate != this._predictable) {
// Lock the 'correctable' token now that either a valid correction for
// it has been found or all possible corrections are exhausted. We only
// consider a single correction for most of a tokenization's tokens,
// generally only allowing correction variation for the last represented
// token.
this._correctables.splice(this._correctables.indexOf(correctableToUpdate), 1);
this._uncorrectables.push(correctableToUpdate);
}
}
if(tokenResult.type == 'none') {
// Transition the node from 'correctable' to 'uncorrectable' - we were
// unable to find valid corrections for it.
const lockedResult = correctableToUpdate.bestExample;
this._generatedTokenResults.set(correctableToUpdate.spaceId, {
matchString: lockedResult.text,
inputSamplingCost: -Math.log(lockedResult.p),
knownCost: MAX_EDIT_THRESHOLD_FACTOR, // we'll use the same threshold at which further search is terminated.
totalCost: -Math.log(lockedResult.p) + MAX_EDIT_THRESHOLD_FACTOR * EDIT_DISTANCE_COST_SCALE
});
// We can make no further predictions if we've exhausted all search options.
if(correctableToUpdate == this._predictable) {
this._uncorrectables.push(correctableToUpdate);
delete this._predictable;
} else {
delistCorrectable();
}
} else if(tokenResult.type == 'complete') {
// Note that at this stage, we do not requeue the 'predictable' - other
// correctables may exist and need their first corrections before we look
// for other corrective variations of the 'predictable'.
delistCorrectable();
// Either way, update the token -> correction-string map with the obtained result.
this._generatedTokenResults.set(correctableToUpdate.spaceId, tokenResult.mapping);
}
const resultCost = tokenResult.type != 'none' ? tokenResult.cost : this._generatedTokenResults.get(correctableToUpdate.spaceId).totalCost;
// Update the cost associated with the token.
const tokenizationCost = this.lastTotalCost = this.getUpdatedTotalCost(correctableToUpdate, resultCost);
this.tokenCostMap.set(correctableToUpdate.spaceId, resultCost);
// If we haven't found a valid correction for the token with lowest-cost update,
// just requeue it and keep searching until we find one.
if(tokenResult.type == 'intermediate') {
this.selectionQueue.enqueue(correctableToUpdate);
// Needs to return the 'proper' type of result.
return {
type: 'intermediate',
cost: tokenizationCost
};
}
// If we have a correction for all components in need of correction, then
// search for alternative corrections for the 'predictable' token - even if
// we previously stopped searching for more because we found its first
// correction before finding one for at least one other 'correctable'.
if(this._correctables.length == 0 && this.selectionQueue.count == 0 && this._predictable) {
this.selectionQueue.enqueue(this._predictable);
}
const correctionResults = this.collateResults();
if(correctionResults.matchedResult.findIndex((c) => c == undefined) != -1) {
return {
type: 'intermediate',
cost: tokenizationCost
};
}
// Determine the proper return type and construct the proper return object accordingly.
this._previousResults.push(correctionResults);
return {
type: 'complete',
cost: tokenizationCost,
mapping: correctionResults
};
}
}

45
web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-result-mapping.ts
View file

@ -1,45 +0,0 @@
import { CorrectionResultMapping } from "./correction-result-mapping.js";
import { TokenizationCorrector, TokenResult } from './tokenization-corrector.js';
export class TokenizationResultMapping implements CorrectionResultMapping<ReadonlyArray<TokenResult>> {
readonly matchingSpace: TokenizationCorrector;
readonly matchedResult: ReadonlyArray<TokenResult>;
constructor(tokenization: TokenResult[], corrector: TokenizationCorrector) {
this.matchingSpace = corrector;
this.matchedResult = tokenization;
}
get spaceId(): number {
return this.matchingSpace.tokenization.spaceId;
}
// /**
// * Gets the number of Damerau-Levenshtein edits needed to reach the node's
// * matchString from the output induced by the input sequence used to reach it.
// *
// * (This is scaled by `SearchSpace.EDIT_DISTANCE_COST_SCALE` when included in
// * `totalCost`.)
// */
// get knownCost(): number {
// return this.node.editCount;
// }
// /**
// * Gets the "input sampling cost" of the edge, which should be considered as the
// * negative log-likelihood of the input path taken to reach the node.
// */
// get inputSamplingCost(): number {
// return this.node.inputSamplingCost;
// }
/**
* Gets the "total cost" of the edge, which should be considered as the
* negative log-likelihood of the input path taken to reach the node
* multiplied by the 'probability' induced by needed Damerau-Levenshtein edits
* to the resulting output.
*/
get totalCost(): number {
return this.matchedResult.reduce((total, curr) => total + curr.totalCost, 0);
}
}

2
web/src/engine/predictive-text/worker-thread/src/main/test-index.ts
View file

@ -16,11 +16,9 @@ export * from './correction/legacy-quotient-spur.js';
export * from './correction/quotient-node-finalizer.js';
export * from './correction/search-quotient-root.js';
export { ExtendedEditOperation, SegmentableDistanceCalculation } from './correction/segmentable-calculation.js';
export * from './correction/tokenization-corrector.js';
export * from './correction/tokenization-subsets.js';
export * from './correction/transition-helpers.js';
export * from './correction/token-result-mapping.js';
export * from './correction/tokenization-result-mapping.js';
export {
determinePunctuationFromModel, determineModelWordbreaker,
determineModelTokenizer, detectCurrentCasing

468
web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts
View file

@ -1,468 +0,0 @@
/*
* Keyman is copyright (C) SIL Global. MIT License.
*
* Created by jahorton on 2026-04-14
*
* This file defines tests for the `TokenizationCorrector` class, which is used
* to prioritize optimal multi-token corrections (and predictions) within the
* predictive-text correction-search engine.
*/
import { assert } from 'chai';
import { LexicalModelTypes } from '@keymanapp/common-types';
import { default as defaultBreaker } from '@keymanapp/models-wordbreakers';
import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs';
import {
ContextToken,
ContextTokenization,
correctionValidForAutoSelect,
ExecutionTimer,
generateSubsetId,
getBestMatches,
LegacyQuotientSpur,
models,
PathInputProperties,
PathResult,
SearchQuotientNode,
SearchQuotientRoot,
TokenizationCorrector,
TokenResult,
TokenizationResultMapping
} from '@keymanapp/lm-worker/test-index';
import Distribution = LexicalModelTypes.Distribution;
import TrieModel = models.TrieModel;
import Transform = LexicalModelTypes.Transform;
const plainModel = new TrieModel(
jsonFixture('models/tries/english-1000'), {
languageUsesCasing: true,
wordBreaker: defaultBreaker
}
);
function buildTestTimer() {
return new ExecutionTimer(Number.MAX_VALUE, Number.MAX_VALUE);
}
function buildFixture_therefore() {
let ID_SEED = 11;
const distributionSrc: [string, number][][] = [
[ ['t', 1] ],
[ ['h', 1] ],
[ ['e', 0.6] ],
[ [' ', 0.8], ['r', 0.2] ],
[ ['e', 1] ],
[ ['f', 1] ]
];
const distributions: Distribution<Required<Transform>>[] = distributionSrc.map((tupleArray) => {
const transitionId = ID_SEED++;
return tupleArray.map((tuple) => {
return {
p: tuple[1],
sample: {
insert: tuple[0],
deleteLeft: 0,
deleteRight: 0,
id: transitionId
}
}
});
});
// Assumes that the first entry in each distribution is the most likely.
const inputSources: PathInputProperties[] = distributions.map((dist) => {
return {
subsetId: generateSubsetId(),
segment: {start: 0, transitionId: dist[0].sample.id},
bestProbFromSet: dist[0].p
};
})
const therefTokens: ContextToken[] = []; // as in "therefore"
const the_efTokens: ContextToken[] = []; // as in "the effect"
// TODO: Use SubstitutionQuotientSpur instead!
let firstTokenNode: SearchQuotientNode = new SearchQuotientRoot(plainModel);
for(let i=0; i < 3; i++) {
firstTokenNode = new LegacyQuotientSpur(firstTokenNode, distributions[i], inputSources[i]);
}
the_efTokens.push(new ContextToken(firstTokenNode, false));
firstTokenNode = new LegacyQuotientSpur(firstTokenNode, [distributions[3][1]], {
...inputSources[3],
subsetId: generateSubsetId()
});
// whitespace token alternate - using the ' ' input instead.
const whitespaceToken = new ContextToken(
new LegacyQuotientSpur(
new SearchQuotientRoot(plainModel),
[distributions[3][0]],
{ ...inputSources[3], subsetId: generateSubsetId() }
), false
);
whitespaceToken.isWhitespace = true;
the_efTokens.push(whitespaceToken);
let secondTokenNode: SearchQuotientNode = new SearchQuotientRoot(plainModel);
for(let i=4; i < distributions.length; i++) {
firstTokenNode = new LegacyQuotientSpur(firstTokenNode, distributions[i], {
...inputSources[i],
subsetId: generateSubsetId()
});
secondTokenNode = new LegacyQuotientSpur(secondTokenNode, distributions[i], {
...inputSources[i],
subsetId: generateSubsetId()
})
}
therefTokens.push(new ContextToken(firstTokenNode));
the_efTokens.push(new ContextToken(secondTokenNode));
return {
filter: (token: ContextToken) => correctionValidForAutoSelect(token.exampleInput),
theref: new ContextTokenization(therefTokens),
the_ef: new ContextTokenization(the_efTokens)
}
}
function buildFixture_terminalWhitespace() {
let ID_SEED = 11;
const distributionSrc: [string, number][][] = [
[ ['s', 1] ],
[ ['p', 1] ],
[ ['a', 1] ],
[ ['c', 1] ],
[ ['e', 1] ],
[ [' ', 1] ],
];
const distributions: Distribution<Required<Transform>>[] = distributionSrc.map((tupleArray) => {
const transitionId = ID_SEED++;
return tupleArray.map((tuple) => {
return {
p: tuple[1],
sample: {
insert: tuple[0],
deleteLeft: 0,
deleteRight: 0,
id: transitionId
}
}
});
});
// Assumes that the first entry in each distribution is the most likely.
const inputSources: PathInputProperties[] = distributions.map((dist) => {
return {
subsetId: generateSubsetId(),
segment: {start: 0, transitionId: dist[0].sample.id},
bestProbFromSet: dist[0].p
};
})
const fullTokens: ContextToken[] = [];
const lastToken: ContextToken[] = [];
// TODO: Use SubstitutionQuotientSpur instead!
let firstTokenNode: SearchQuotientNode = new SearchQuotientRoot(plainModel);
for(let i=0; i < 5; i++) {
firstTokenNode = new LegacyQuotientSpur(firstTokenNode, distributions[i], inputSources[i]);
}
fullTokens.push(new ContextToken(firstTokenNode, false));
// whitespace token alternate - using the ' ' input instead.
const whitespaceToken = new ContextToken(
new LegacyQuotientSpur(
new SearchQuotientRoot(plainModel),
distributions[5],
inputSources[5],
), false
);
whitespaceToken.isWhitespace = true;
fullTokens.push(whitespaceToken);
lastToken.push(whitespaceToken);
return {
filter: (token: ContextToken) => correctionValidForAutoSelect(token.exampleInput),
wordThenSpace: new ContextTokenization(fullTokens),
spaceOnly: new ContextTokenization(lastToken)
}
}
describe('TokenizationCorrector', () => {
describe('constructor', () => {
it('constructs correctly from a single correctable token', () => {
const fixture = buildFixture_therefore();
const tokenization = fixture.theref;
const instance = new TokenizationCorrector(
tokenization,
1,
fixture.filter
);
assert.sameOrderedMembers(instance.uncorrectableTokens.slice(), []);
assert.sameOrderedMembers(instance.correctableTokens.slice(), []);
assert.equal(instance.predictableToken, tokenization.tail);
});
it('constructs correctly from a single uncorrectable token', () => {
const fixture = buildFixture_terminalWhitespace();
const tokenization = fixture.spaceOnly;
const instance = new TokenizationCorrector(
tokenization,
tokenization.tokens.length,
fixture.filter
);
assert.sameOrderedMembers(instance.uncorrectableTokens.slice(), [tokenization.tail]);
assert.sameOrderedMembers(instance.correctableTokens.slice(), []);
assert.equal(instance.predictableToken, undefined);
});
it('constructs from multiple tokens, with the middle one uncorrectable', () => {
const fixture = buildFixture_therefore();
const tokenization = fixture.the_ef;
const tokenCount = tokenization.tokens.length;
const instance = new TokenizationCorrector(
tokenization,
tokenCount,
fixture.filter
);
assert.sameOrderedMembers(instance.uncorrectableTokens.slice(), [tokenization.tokens[tokenCount-2]]);
assert.sameOrderedMembers(instance.correctableTokens.slice(), [tokenization.tokens[tokenCount-3]]);
assert.equal(instance.predictableToken, tokenization.tail);
});
it('constructs from multiple tokens, ignoring the first due to bounds', () => {
const fixture = buildFixture_therefore();
const tokenization = fixture.the_ef;
const tokenCount = tokenization.tokens.length;
const instance = new TokenizationCorrector(
tokenization,
tokenCount-1,
fixture.filter
);
assert.sameOrderedMembers(instance.uncorrectableTokens.slice(), [tokenization.tokens[tokenCount-2]]);
assert.sameOrderedMembers(instance.correctableTokens.slice(), []);
assert.equal(instance.predictableToken, tokenization.tail);
});
it('constructs correctly when the final token is uncorrectable', () => {
const fixture = buildFixture_terminalWhitespace();
const tokenization = fixture.wordThenSpace;
const instance = new TokenizationCorrector(
tokenization,
tokenization.tokens.length,
fixture.filter
);
assert.sameOrderedMembers(instance.uncorrectableTokens.slice(), [tokenization.tail]);
assert.sameOrderedMembers(instance.correctableTokens.slice(), [tokenization.tokens[0]]);
assert.equal(instance.predictableToken, undefined);
});
});
describe('handleNextNode', () => {
it('finds corrections for a single correctable token', () => {
const fixture = buildFixture_therefore();
const tokenization = fixture.theref;
const instance = new TokenizationCorrector(
tokenization,
1,
fixture.filter
);
let searchResult: PathResult<TokenizationResultMapping>;
do {
searchResult = instance.handleNextNode();
} while(searchResult.type == 'intermediate');
assert.equal(searchResult.type, 'complete');
if(searchResult.type == 'complete') {
const mapping = searchResult.mapping;
const tokenResults = mapping.matchedResult;
assert.isNotNaN(searchResult.cost);
assert.equal(searchResult.cost, searchResult.mapping.totalCost);
assert.equal(tokenResults.length, 1);
assert.sameOrderedMembers(tokenResults.map((r) => r.matchString), ['theref']);
// Now that an entry has been found, verify the corrector's state.
assert.isOk(instance.predictableToken); // should not become bound or locked.
assert.isTrue(instance.generatedTokenResults.has(instance.predictableToken));
assert.equal(instance.generatedTokenResults.get(instance.predictableToken), tokenResults[0]);
}
searchResult = instance.handleNextNode();
// There should be more results that may be found.
assert.notEqual(searchResult.type, 'none');
do {
searchResult = instance.handleNextNode();
} while(searchResult.type == 'intermediate');
assert.notEqual(searchResult.type, 'none');
});
it('finds corrections for a group of tokens with two correctable', () => {
const fixture = buildFixture_therefore();
const tokenization = fixture.the_ef;
const instance = new TokenizationCorrector(
tokenization,
3,
fixture.filter
);
let searchResult: PathResult<TokenizationResultMapping>;
do {
searchResult = instance.handleNextNode();
} while(searchResult.type == 'intermediate');
assert.equal(searchResult.type, 'complete');
let firstResults: ReadonlyArray<TokenResult>;
if(searchResult.type == 'complete') {
const mapping = searchResult.mapping;
const tokenResults = mapping.matchedResult;
firstResults = tokenResults;
assert.isNotNaN(searchResult.cost);
assert.equal(searchResult.cost, searchResult.mapping.totalCost);
assert.equal(tokenResults.length, 3);
assert.sameOrderedMembers(tokenResults.map((r) => r.matchString), ['the', ' ', 'ef']);
}
// Now that an entry has been found, verify the corrector's state.
assert.isOk(instance.predictableToken); // should not become bound or locked.
assert.isTrue(instance.generatedTokenResults.has(instance.predictableToken));
for(let i=0; i < firstResults.length; i++) {
assert.equal(instance.generatedTokenResults.get(instance.orderedTokens[i]), firstResults[i]);
}
searchResult = instance.handleNextNode();
// There should be more results that may be found.
assert.notEqual(searchResult.type, 'none');
do {
searchResult = instance.handleNextNode();
if(searchResult.type == 'complete') {
const mapping = searchResult.mapping;
const tokenResults = mapping.matchedResult;
// Verify that the first (bound) token is not altered further.
// It should receive no further correction attempts.
assert.equal(tokenResults[0], firstResults[0]);
assert.equal(tokenResults[1], firstResults[1]);
assert.notEqual(tokenResults[2], firstResults[2]);
}
} while(searchResult.type != 'none');
});
it('immediately returns a single result when the only represented token is uncorrectable', () => {
const fixture = buildFixture_terminalWhitespace();
const tokenization = fixture.spaceOnly;
const instance = new TokenizationCorrector(
tokenization,
tokenization.tokens.length,
fixture.filter
);
const searchResult = instance.handleNextNode();
assert.equal(searchResult.type, 'complete');
if(searchResult.type == 'complete') {
assert.equal(searchResult.mapping.matchedResult[0].matchString, ' ');
}
const nilResult = instance.handleNextNode();
assert.equal(nilResult.type, 'none');
});
it('returns a single result when the final token is uncorrectable', () => {
const fixture = buildFixture_terminalWhitespace();
const tokenization = fixture.wordThenSpace;
const instance = new TokenizationCorrector(
tokenization,
tokenization.tokens.length,
fixture.filter
);
let searchResult: PathResult<TokenizationResultMapping>;
do {
searchResult = instance.handleNextNode();
} while(searchResult.type == 'intermediate');
assert.equal(searchResult.type, 'complete');
if(searchResult.type == 'complete') {
assert.equal(searchResult.mapping.matchedResult[0].matchString, 'space');
assert.equal(searchResult.mapping.matchedResult[1].matchString, ' ');
}
const nilResult = instance.handleNextNode();
assert.equal(nilResult.type, 'none');
});
describe('with getBestMatches()', () => {
it('finds results from each of two tokenization variants sharing lower-level SearchQuotientNodes', async () => {
const fixture = buildFixture_therefore();
const tokenizations = [fixture.theref, fixture.the_ef];
const correctors = tokenizations.map((t) => new TokenizationCorrector(t, t.tokens.length, fixture.filter));
let haveSeenSingleTokenCorrection = false;
let haveSeenThreeTokenCorrection = false;
for await(let phraseMatch of getBestMatches<
ReadonlyArray<TokenResult>,
TokenizationResultMapping,
TokenizationCorrector
>(correctors, buildTestTimer())) {
if(phraseMatch.matchedResult.length == 1) {
if(!haveSeenSingleTokenCorrection) {
assert.sameOrderedMembers(phraseMatch.matchedResult.map((t) => t.matchString), ['theref' /* -ore */]);
}
haveSeenSingleTokenCorrection = true;
} else if(phraseMatch.matchedResult.length == 3) {
if(!haveSeenThreeTokenCorrection) {
assert.sameOrderedMembers(phraseMatch.matchedResult.map((t) => t.matchString), ['the', ' ', 'ef' /* -fort */]);
}
haveSeenThreeTokenCorrection = true;
}
if(haveSeenSingleTokenCorrection && haveSeenThreeTokenCorrection) {
break;
}
}
assert.isTrue(haveSeenSingleTokenCorrection, 'A single-token correction was expected but not found');
assert.isTrue(haveSeenThreeTokenCorrection, 'A three-token correction was expected but not found');
});
});
});
});