From fa828d4710c4e59c7ed9cc119480c9f153c373e4 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Wed, 8 Oct 2025 08:31:00 -0500 Subject: [PATCH] feat(web): add correction source search-space identification When we start supporting more than one "space" for correction-searches, we may need to know which "space" (tokenization) a suggestion arose from. This way, we have a path forward for applying tokenization-dependent behaviors that may be required. Build-bot: skip build:web Test-bot: skip --- .../src/main/correction/context-token.ts | 17 +- .../main/correction/context-tokenization.ts | 7 + .../src/main/correction/distance-modeler.ts | 48 +++-- .../main/correction/search-quotient-node.ts | 9 +- .../main/correction/search-quotient-spur.ts | 22 ++- .../distance-modeler.tests.ts | 173 ++++++++++++------ .../correction-search/getBestMatches.tests.ts | 28 ++- 7 files changed, 203 insertions(+), 101 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts index 5387c0da41..7658c5f8ff 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts @@ -100,8 +100,7 @@ export class ContextToken { constructor(param: ContextToken | LexicalModel, rawText?: string, isPartial?: boolean) { if(param instanceof ContextToken) { const priorToken = param; - this.isWhitespace = priorToken.isWhitespace; - this.isPartial = priorToken.isPartial; + Object.assign(this, priorToken); // We need to construct a separate search space from other token copies. // @@ -109,12 +108,6 @@ export class ContextToken { // we need to ensure that only fully-utilized keystrokes are considered. this._searchModule = priorToken.searchModule; this._inputRange = priorToken._inputRange.slice(); - - // Preserve any annotated applied-suggestion transition ID data; it's useful - // for delayed reversion operations. - if(priorToken.appliedTransitionId !== undefined) { - this.appliedTransitionId = priorToken.appliedTransitionId - } } else { const model = param; @@ -166,6 +159,14 @@ export class ContextToken { return this.exampleInput == ''; } + /** + * Gets the unique identifier that may be used to match this ContextToken with + * a correction-search result. + */ + get spaceId(): number { + return this.searchSpace.spaceId; + } + /** * Gets a compact string-based representation of `inputRange` that * maps compatible token source ranges to each other. diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 2e7ef7c7f0..75eb72952c 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -149,6 +149,13 @@ export class ContextTokenization { return this.tokens[this.tokens.length - 1]; } + /** + * Returns the unique correction-search space identifier corresponding to this tokenization. + */ + get spaceId(): number { + return this.tail.spaceId; + } + /** * Returns plain-text strings representing the most probable representation for all * tokens represented by this tokenization instance. diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts index b9d674ecca..473bb60b20 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts @@ -127,13 +127,20 @@ export class SearchNode { */ private _inputCost?: number; - constructor(rootTraversal: LexiconTraversal, toKey?: (arg0: string) => string); - constructor(node: SearchNode); - constructor(rootTraversal: LexiconTraversal | SearchNode, toKey?: (arg0: string) => string) { - toKey = toKey || (x => x); + /** + * A unique identifier corresponding to the earliest SearchPath containing + * the correction-search graph edge represented by this instance. + * + * Corresponding search results will be tagged with this, which can be used + * to identify the result's original source tokenization. + */ + readonly spaceId: number; - if(rootTraversal instanceof SearchNode) { - const priorNode = rootTraversal; + constructor(rootTraversal: LexiconTraversal, spaceId: number, toKey?: (arg0: string) => string); + constructor(node: SearchNode, spaceId?: number); + constructor(param1: LexiconTraversal | SearchNode, spaceId?: number, toKey?: ((arg0: string) => string)) { + if(param1 instanceof SearchNode) { + const priorNode = param1; Object.assign(this, priorNode); if(this.partialEdge) { @@ -145,11 +152,16 @@ export class SearchNode { // Do NOT copy over _inputCost; this is a helper-constructor for methods // building new nodes... which will have a different cost. delete this._inputCost; + + // This is unique at each level, though it will reuse a previous ID if no new + // one is provided (say, for 'insert' edits). + this.spaceId = spaceId ?? priorNode.spaceId; } else { this.calculation = new ClassicalDistanceCalculation(); - this.matchedTraversals = [rootTraversal]; + this.matchedTraversals = [param1]; this.priorInput = []; - this.toKey = toKey; + this.toKey = toKey || (x => x); + this.spaceId = spaceId; } } @@ -423,17 +435,17 @@ export class SearchNode { * represented lexicon prefix - be it due to not adding one (deletions) or * due to not being the same character, all mismatching cases are merged into * one, reducing the rate of expansion for the search graph. - * @param inputDistribution + * @param input * @param isSubstitution * @returns */ - private setupSubsetProcessing(inputDistribution: Distribution, isSubstitution: boolean ) { + private setupSubsetProcessing(dist: Distribution, isSubstitution: boolean, edgeId: number) { if(this.hasPartialInput) { throw new Error("Invalid state: will not take new input while still processing Transform subset"); } const edges: SearchNode[] = []; - const subsets = subsetByInterval(inputDistribution); + const subsets = subsetByInterval(dist); for(let dl = 0; dl < subsets.length; dl++) { const dlSubset = subsets[dl]; @@ -451,7 +463,7 @@ export class SearchNode { continue; } - const node = new SearchNode(this); + const node = new SearchNode(this, edgeId); node.calculation = edgeCalc; node.partialEdge = { doSubsetMatching: isSubstitution, @@ -480,8 +492,8 @@ export class SearchNode { * @returns An array of SearchNodes corresponding to search paths that skip the next * input keystroke. */ - buildDeletionEdges(inputDistribution: Distribution): SearchNode[] { - return this.setupSubsetProcessing(inputDistribution, false); + buildDeletionEdges(dist: Distribution, edgeId: number): SearchNode[] { + return this.setupSubsetProcessing(dist, false, edgeId); } /** @@ -493,12 +505,12 @@ export class SearchNode { * @returns An array of SearchNodes corresponding to search paths that match or * replace the next currently-unprocessed input. */ - buildSubstitutionEdges(inputDistribution: Distribution): SearchNode[] { + buildSubstitutionEdges(dist: Distribution, edgeId: number): SearchNode[] { // Note: due to the batching approach used via TransformSubsets, // substitutions are _not_ adequately represented by one 'insertion' + one // 'deletion' step. Explicit substitution / match-oriented processing is // required. - return this.setupSubsetProcessing(inputDistribution, true); + return this.setupSubsetProcessing(dist, true, edgeId); } /** @@ -598,6 +610,10 @@ export class SearchResult { get finalTraversal(): LexiconTraversal { return this.resultNode.currentTraversal; } + + get spaceId(): number { + return this.resultNode.spaceId; + } } // Current best guesstimate of how compositor will retrieve ideal corrections. diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-node.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-node.ts index 590d1b7965..c563eb2f92 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-node.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-node.ts @@ -27,7 +27,8 @@ type IntermediateSearchPath = { type CompleteSearchPath = { type: 'complete', cost: number, - finalNode: SearchNode + finalNode: SearchNode, + spaceId: number } export type PathResult = NullPath | IntermediateSearchPath | CompleteSearchPath; @@ -37,6 +38,12 @@ export type PathResult = NullPath | IntermediateSearchPath | CompleteSearchPath; * for predictive-text corrections. */ export interface SearchQuotientNode { + /** + * Returns an identifier uniquely identifying this search-batching structure + * by correction-search results. + */ + readonly spaceId: number; + /** * Retrieves the lowest-cost / lowest-distance edge from the batcher's search * area, checks its validity as a correction to the input text, and reports on diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts index 40dddf48dc..6f0e75c1a5 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts @@ -12,7 +12,7 @@ import { QueueComparator as Comparator, KMWString, PriorityQueue } from '@keyman import { LexicalModelTypes } from '@keymanapp/common-types'; import { EDIT_DISTANCE_COST_SCALE, SearchNode, SearchResult } from './distance-modeler.js'; -import { PathResult, SearchQuotientNode } from './search-quotient-node.js'; +import { generateSpaceSeed, PathResult, SearchQuotientNode } from './search-quotient-node.js'; import Distribution = LexicalModelTypes.Distribution; import LexicalModel = LexicalModelTypes.LexicalModel; @@ -33,6 +33,7 @@ export class SearchQuotientSpur implements SearchQuotientNode { readonly rootPath: SearchQuotientSpur; private parentPath: SearchQuotientSpur; + readonly spaceId: number; // We use an array and not a PriorityQueue b/c batch-heapifying at a single point in time // is cheaper than iteratively building a priority queue. @@ -79,7 +80,9 @@ export class SearchQuotientSpur implements SearchQuotientNode { * @param model */ constructor(model: LexicalModel); - constructor(arg1: LexicalModel | SearchQuotientSpur) { + constructor(arg1: SearchQuotientSpur|LexicalModel) { + this.spaceId = generateSpaceSeed(); + if(arg1 instanceof SearchQuotientSpur) { const parentSpace = arg1; this.lowestCostAtDepth = parentSpace.lowestCostAtDepth.slice(); @@ -94,7 +97,7 @@ export class SearchQuotientSpur implements SearchQuotientNode { throw new Error("The provided model does not implement the `traverseFromRoot` function, which is needed to support robust correction searching."); } - const rootNode = new SearchNode(model.traverseFromRoot(), model.toKey ? model.toKey.bind(model) : null); + const rootNode = new SearchNode(model.traverseFromRoot(), this.spaceId, model.toKey ? model.toKey.bind(model) : null); this.selectionQueue.enqueue(rootNode); this.lowestCostAtDepth = []; this.rootPath = this; @@ -168,8 +171,8 @@ export class SearchQuotientSpur implements SearchQuotientNode { // our previously-reached 'extractedResults' nodes. let newlyAvailableEdges: SearchNode[] = []; let batches = this.completedPaths?.map(function(node) { - let deletions = node.buildDeletionEdges(input); - let substitutions = node.buildSubstitutionEdges(input); + let deletions = node.buildDeletionEdges(input, childSpace.spaceId); + let substitutions = node.buildSubstitutionEdges(input, childSpace.spaceId); const batch = deletions.concat(substitutions); @@ -210,9 +213,9 @@ export class SearchQuotientSpur implements SearchQuotientNode { let deletionEdges: SearchNode[] = []; if(!substitutionsOnly) { - deletionEdges = currentNode.buildDeletionEdges(this.inputs); + deletionEdges = currentNode.buildDeletionEdges(this.inputs, this.spaceId); } - const substitutionEdges = currentNode.buildSubstitutionEdges(this.inputs); + const substitutionEdges = currentNode.buildSubstitutionEdges(this.inputs, this.spaceId); let batch = deletionEdges.concat(substitutionEdges); // Skip the queue for the first pass; there will ALWAYS be at least one pass, @@ -249,7 +252,7 @@ export class SearchQuotientSpur implements SearchQuotientNode { return { ...result, type: 'intermediate' - } as PathResult; + } as PathResult } let currentNode = this.selectionQueue.dequeue(); @@ -319,7 +322,8 @@ export class SearchQuotientSpur implements SearchQuotientNode { return { type: 'complete', cost: currentNode.currentCost, - finalNode: currentNode + finalNode: currentNode, + spaceId: this.spaceId }; } diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/distance-modeler.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/distance-modeler.tests.ts index c72917bed7..abf9402f70 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/distance-modeler.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/distance-modeler.tests.ts @@ -33,6 +33,8 @@ const toKey = (s: string) => testModel.toKey(s); */ const FIRST_CHAR_VARIANTS = 24; +let SEARCH_EDGE_SEED = 0; + function assertEdgeChars(edge: correction.SearchNode, input: string, match: string) { assert.isTrue(edgeHasChars(edge, input, match)); } @@ -46,7 +48,7 @@ function edgeHasChars(edge: correction.SearchNode, input: string, match: string) } function findEdgesWithChars(edgeArray: correction.SearchNode[], match: string) { - let results = edgeArray.filter(function(value) { + const results = edgeArray.filter((value) => { return value.calculation.lastMatchEntry == match; }); @@ -55,7 +57,8 @@ function findEdgesWithChars(edgeArray: correction.SearchNode[], match: string) { } function fetchCommonTENode() { - const rootNode = new SearchNode(testModel.traverseFromRoot(), toKey); + const rootSeed = SEARCH_EDGE_SEED++; + const rootNode = new SearchNode(testModel.traverseFromRoot(), rootSeed, toKey); // Establish desired source node: prefix 'te'. const firstLayerTransforms: Distribution = [{ @@ -71,14 +74,16 @@ function fetchCommonTENode() { }, p: 0.25 }]; + const firstLayerSeed = SEARCH_EDGE_SEED++; const firstLayerNodes = rootNode - .buildSubstitutionEdges(firstLayerTransforms) + .buildSubstitutionEdges(firstLayerTransforms, firstLayerSeed) .flatMap(node => node.processSubsetEdge()); assert.isAbove(firstLayerNodes.length, FIRST_CHAR_VARIANTS); firstLayerNodes.sort((a, b) => a.currentCost - b.currentCost); const tNode = firstLayerNodes[0]; assert.equal(tNode.resultKey, 't'); + assert.equal(tNode.spaceId, firstLayerSeed); assert.sameDeepMembers(tNode.priorInput, [firstLayerTransforms[0]]); assert.isFalse(tNode.hasPartialInput); @@ -95,8 +100,9 @@ function fetchCommonTENode() { }, p: 0.2 }]; + const secondLayerSeed = SEARCH_EDGE_SEED++; const secondLayerNodes = tNode - .buildSubstitutionEdges(secondLayerTransforms) + .buildSubstitutionEdges(secondLayerTransforms, secondLayerSeed) .flatMap(node => node.processSubsetEdge()); // The field narrows down at this point, but still has a decent number // of variants (11). @@ -107,6 +113,7 @@ function fetchCommonTENode() { assert.equal(teNode.resultKey, 'te'); assert.equal(teNode.editCount, 0); + assert.equal(teNode.spaceId, secondLayerSeed); return teNode; } @@ -114,7 +121,8 @@ function fetchCommonTENode() { describe('Correction Distance Modeler', () => { describe('SearchNode', () => { it('constructs a fresh instance from a traversal + keyingFunction', () => { - const rootNode = new SearchNode(testModel.traverseFromRoot(), toKey); + const rootSeed = SEARCH_EDGE_SEED++; + const rootNode = new SearchNode(testModel.traverseFromRoot(), rootSeed, toKey); assert.equal(rootNode.resultKey, ''); assert.equal(rootNode.editCount, 0); @@ -127,11 +135,12 @@ describe('Correction Distance Modeler', () => { assert.isUndefined(rootNode.calculation.lastInputEntry); assert.isUndefined(rootNode.calculation.lastMatchEntry); assert.equal(rootNode.toKey, toKey); + assert.equal(rootNode.spaceId, rootSeed); }); describe('supports the cloning constructor pattern', () => { it('properly deep-copies the root node', () => { - const originalNode = new SearchNode(testModel.traverseFromRoot(), toKey); + const originalNode = new SearchNode(testModel.traverseFromRoot(), SEARCH_EDGE_SEED++, toKey); const clonedNode = new SearchNode(originalNode); // Root node properties; may as well re-assert 'em. @@ -147,6 +156,7 @@ describe('Correction Distance Modeler', () => { assert.isUndefined(clonedNode.calculation.lastInputEntry); assert.isUndefined(clonedNode.calculation.lastMatchEntry); assert.equal(clonedNode.toKey, toKey); + assert.equal(clonedNode.spaceId, originalNode.spaceId); // Avoid aliasing for properties holding mutable objects assert.notEqual(clonedNode.priorInput, originalNode.priorInput); @@ -157,7 +167,8 @@ describe('Correction Distance Modeler', () => { }); it('properly deep-copies fully-processed nodes later in the search path', () => { - const rootNode = new SearchNode(testModel.traverseFromRoot(), toKey); + const rootSeed = SEARCH_EDGE_SEED++; + const rootNode = new SearchNode(testModel.traverseFromRoot(), rootSeed, toKey); // Establish desired source node: prefix 'te'. const firstLayerTransforms: Distribution = [{ @@ -174,7 +185,7 @@ describe('Correction Distance Modeler', () => { p: 0.25 }]; const firstLayerNodes = rootNode - .buildSubstitutionEdges(firstLayerTransforms) + .buildSubstitutionEdges(firstLayerTransforms, SEARCH_EDGE_SEED++) .flatMap(node => node.processSubsetEdge()); assert.isAbove(firstLayerNodes.length, FIRST_CHAR_VARIANTS); firstLayerNodes.sort((a, b) => a.currentCost - b.currentCost); @@ -197,8 +208,9 @@ describe('Correction Distance Modeler', () => { }, p: 0.2 }]; + const secondSpaceId = SEARCH_EDGE_SEED++; const secondLayerNodes = tNode - .buildSubstitutionEdges(secondLayerTransforms) + .buildSubstitutionEdges(secondLayerTransforms, secondSpaceId) .flatMap(node => node.processSubsetEdge()); assert.isAbove(secondLayerNodes.length, 10); secondLayerNodes.sort((a, b) => a.currentCost - b.currentCost); @@ -220,6 +232,7 @@ describe('Correction Distance Modeler', () => { assert.equal(node.calculation.lastMatchEntry, 'e'); assert.equal((node.currentTraversal as TrieTraversal).prefix, 'te'); assert.equal(node.toKey, toKey); + assert.equal(node.spaceId, secondSpaceId); } assertSourceNodeProps(teNode); @@ -238,7 +251,8 @@ describe('Correction Distance Modeler', () => { }); it('properly deep-copies partially-processed edges later in the search path', () => { - const rootNode = new SearchNode(testModel.traverseFromRoot(), toKey); + const rootSeed = SEARCH_EDGE_SEED++; + const rootNode = new SearchNode(testModel.traverseFromRoot(), rootSeed, toKey); // Establish desired source node: prefix 'te', with 'e' the first letter of // a multi-char insert Transform. @@ -256,7 +270,7 @@ describe('Correction Distance Modeler', () => { p: 0.25 }]; const firstLayerNodes = rootNode - .buildSubstitutionEdges(firstLayerTransforms) + .buildSubstitutionEdges(firstLayerTransforms, SEARCH_EDGE_SEED++) .flatMap(node => node.processSubsetEdge()); assert.isAbove(firstLayerNodes.length, FIRST_CHAR_VARIANTS); firstLayerNodes.sort((a, b) => a.currentCost - b.currentCost); @@ -279,8 +293,10 @@ describe('Correction Distance Modeler', () => { }, p: 0.2 }]; + + const secondLayerId = SEARCH_EDGE_SEED++; const secondLayerNodes = tNode - .buildSubstitutionEdges(secondLayerTransforms) + .buildSubstitutionEdges(secondLayerTransforms, secondLayerId) .flatMap(node => node.processSubsetEdge()); assert.isAbove(secondLayerNodes.length, 10); secondLayerNodes.sort((a, b) => a.currentCost - b.currentCost); @@ -302,6 +318,7 @@ describe('Correction Distance Modeler', () => { assert.equal(node.calculation.lastMatchEntry, 'e'); assert.equal((node.currentTraversal as TrieTraversal).prefix, 'te'); assert.equal(node.toKey, toKey); + assert.equal(node.spaceId, secondLayerId); } assertSourceNodeProps(teNode); @@ -321,25 +338,27 @@ describe('Correction Distance Modeler', () => { }); // Consider adding more, deeper? - it('builds insertion edges based on lexicon, from root', function() { - let rootTraversal = testModel.traverseFromRoot(); + it('builds insertion edges based on lexicon, from root', () => { + const rootTraversal = testModel.traverseFromRoot(); assert.isNotEmpty(rootTraversal); - let rootNode = new correction.SearchNode(rootTraversal); + const rootSeed = SEARCH_EDGE_SEED++; + const rootNode = new correction.SearchNode(rootTraversal, rootSeed); assert.equal(rootNode.calculation.getHeuristicFinalCost(), 0); - let edges = rootNode.buildInsertionEdges(); + const edges = rootNode.buildInsertionEdges(); assert.isAbove(edges.length, 0); let expectedChildCount = 0; - for(let child of rootTraversal.children()) { + for(const child of rootTraversal.children()) { expectedChildCount++; - let childEdge = edges.filter(value => value.calculation.lastMatchEntry == child.char)[0]; + const childEdge = edges.filter(value => value.calculation.lastMatchEntry == child.char)[0]; assert.isOk(childEdge); assert.isEmpty(childEdge.priorInput); assert.isEmpty(childEdge.calculation.inputSequence); assert.isAbove(childEdge.currentCost, 0); + assert.equal(childEdge.spaceId, rootSeed); } assert.equal(edges.length, expectedChildCount); @@ -363,13 +382,15 @@ describe('Correction Distance Modeler', () => { ]; it('step 1: batches deletion edge(s) for input transforms', () => { - let rootTraversal = testModel.traverseFromRoot(); + const rootTraversal = testModel.traverseFromRoot(); assert.isNotEmpty(rootTraversal); - let rootNode = new correction.SearchNode(rootTraversal); + const rootSeed = SEARCH_EDGE_SEED++; + const rootNode = new correction.SearchNode(rootTraversal, rootSeed); assert.equal(rootNode.calculation.getHeuristicFinalCost(), 0); - const subsetNodes = rootNode.buildDeletionEdges(synthDistribution); + const subsetSeed = SEARCH_EDGE_SEED++; + const subsetNodes = rootNode.buildDeletionEdges(synthDistribution, subsetSeed); assert.equal(subsetNodes.length, 4); subsetNodes.sort((a, b) => a.currentCost - b.currentCost); const expectedCosts = [0.5, .25, 0.15, 0.1].map(x => -Math.log(x)) @@ -378,6 +399,7 @@ describe('Correction Distance Modeler', () => { assert.isTrue(subsetNodes[i].hasPartialInput); // From root: the deleteLeft 1 entries have nothing to delete. assert.equal((subsetNodes[i].currentTraversal as TrieTraversal).prefix, ''); + assert.equal(subsetNodes[i].spaceId, subsetSeed); // Allow a little value wiggle due to double-precision limitations. assert.approximately(subsetNodes[i].inputSamplingCost, expectedCosts[i], 1e-8); @@ -388,15 +410,18 @@ describe('Correction Distance Modeler', () => { it('step 2: first processing layer resolves zero + one char inserts', () => { // From "step 1" above, assertions removed - let rootTraversal = testModel.traverseFromRoot(); - let rootNode = new correction.SearchNode(rootTraversal); - const subsetNodes = rootNode.buildDeletionEdges(synthDistribution); + const rootTraversal = testModel.traverseFromRoot(); + const rootSeed = SEARCH_EDGE_SEED++; + const rootNode = new correction.SearchNode(rootTraversal, rootSeed); + const subsetSeed = SEARCH_EDGE_SEED++; + const subsetNodes = rootNode.buildDeletionEdges(synthDistribution, subsetSeed); subsetNodes.sort((a, b) => a.currentCost - b.currentCost); const processedNodes = subsetNodes.flatMap(n => n.processSubsetEdge()); // All delete-oriented Transform subsets condense down to a single // transform (and edge) each. assert.equal(processedNodes.length, 4); + processedNodes.forEach((n) => assert.equal(n.spaceId, subsetSeed)); // Sorted index 0: 1 insert - should be processed already, in a single // step. @@ -431,9 +456,11 @@ describe('Correction Distance Modeler', () => { it('step 3: second processing layer resolves two char inserts', () => { // From "steps 0, 1" above, assertions removed - let rootTraversal = testModel.traverseFromRoot(); - let rootNode = new correction.SearchNode(rootTraversal); - const subsetNodes = rootNode.buildDeletionEdges(synthDistribution); + const rootTraversal = testModel.traverseFromRoot(); + const rootSeed = SEARCH_EDGE_SEED++; + const rootNode = new correction.SearchNode(rootTraversal, rootSeed); + const subsetSeed = SEARCH_EDGE_SEED++; + const subsetNodes = rootNode.buildDeletionEdges(synthDistribution, subsetSeed); subsetNodes.sort((a, b) => a.currentCost - b.currentCost); // Two nodes were unprocessed at the end of the last step; we handle @@ -450,6 +477,7 @@ describe('Correction Distance Modeler', () => { assert.isFalse(processedNode.hasPartialInput); assert.isFalse(processedNode.hasPartialInput); assert.equal(processedNode.editCount, 2); + assert.equal(processedNode.spaceId, subsetSeed); assert.equal(processedNode.calculation.inputSequence.length, 2); assert.equal(processedNode.calculation.lastInputEntry, SENTINEL_CODE_UNIT); assert.equal(processedNode.inputSamplingCost, step2Nodes[index].inputSamplingCost); @@ -481,7 +509,8 @@ describe('Correction Distance Modeler', () => { const teNode = fetchCommonTENode(); assert.equal(teNode.calculation.getHeuristicFinalCost(), 0); - const subsetNodes = teNode.buildDeletionEdges(synthDistribution); + const subsetSeed = SEARCH_EDGE_SEED++; + const subsetNodes = teNode.buildDeletionEdges(synthDistribution, subsetSeed); assert.equal(subsetNodes.length, 4); subsetNodes.sort((a, b) => a.currentCost - b.currentCost); @@ -492,6 +521,8 @@ describe('Correction Distance Modeler', () => { assert.isTrue(subsetNodes[i].hasPartialInput); // From a 'te' prefix, the deleteLeft 1 entries do have something to delete. assert.equal((subsetNodes[i].currentTraversal as TrieTraversal).prefix, (i == 0 || i == 2) ? 'te' : 't'); + assert.notEqual(subsetNodes[i].spaceId, teNode.spaceId); + assert.equal(subsetNodes[i].spaceId, subsetSeed); // Allow a little value wiggle due to double-precision limitations. assert.approximately(subsetNodes[i].inputSamplingCost, expectedCosts[i], 1e-8); @@ -503,13 +534,16 @@ describe('Correction Distance Modeler', () => { it('step 2: first processing layer resolves zero + one char inserts', () => { // From "step 1" above, assertions removed const teNode = fetchCommonTENode(); - const subsetNodes = teNode.buildDeletionEdges(synthDistribution); + const subsetSeed = SEARCH_EDGE_SEED++; + const subsetNodes = teNode.buildDeletionEdges(synthDistribution, subsetSeed); subsetNodes.sort((a, b) => a.currentCost - b.currentCost); const processedNodes = subsetNodes.flatMap(n => n.processSubsetEdge()); // All delete-oriented Transform subsets condense down to a single // transform (and edge) each. assert.equal(processedNodes.length, 4); + processedNodes.forEach(n => assert.notEqual(n.spaceId, teNode.spaceId)); + processedNodes.forEach(n => assert.equal(n.spaceId, subsetSeed)); // Sorted index 0: 1 insert - should be processed already, in a single // step. @@ -545,7 +579,8 @@ describe('Correction Distance Modeler', () => { it('step 3: second processing layer resolves two char inserts', () => { const teNode = fetchCommonTENode(); - const subsetNodes = teNode.buildDeletionEdges(synthDistribution); + const subsetSeed = SEARCH_EDGE_SEED++; + const subsetNodes = teNode.buildDeletionEdges(synthDistribution, subsetSeed); subsetNodes.sort((a, b) => a.currentCost - b.currentCost); // Two nodes were unprocessed at the end of the last step; we handle @@ -562,6 +597,8 @@ describe('Correction Distance Modeler', () => { assert.isFalse(processedNode.hasPartialInput); assert.isFalse(processedNode.hasPartialInput); assert.equal(processedNode.editCount, 2); + assert.notEqual(processedNode.spaceId, teNode.spaceId); + assert.equal(processedNode.spaceId, subsetSeed); assert.equal(processedNode.calculation.inputSequence.length, 4 - dl); assert.equal(processedNode.calculation.lastInputEntry, SENTINEL_CODE_UNIT); assert.equal(processedNode.inputSamplingCost, baseNode.inputSamplingCost); @@ -592,13 +629,14 @@ describe('Correction Distance Modeler', () => { ]; it('step 1: batches substitution edge(s) for input transforms', () => { - let rootTraversal = testModel.traverseFromRoot(); + const rootTraversal = testModel.traverseFromRoot(); assert.isNotEmpty(rootTraversal); - let rootNode = new correction.SearchNode(rootTraversal); + const rootNode = new correction.SearchNode(rootTraversal, SEARCH_EDGE_SEED++); assert.equal(rootNode.calculation.getHeuristicFinalCost(), 0); - const subsetNodes = rootNode.buildSubstitutionEdges(synthDistribution); + const subsetSeed = SEARCH_EDGE_SEED++; + const subsetNodes = rootNode.buildSubstitutionEdges(synthDistribution, subsetSeed); assert.equal(subsetNodes.length, 4); subsetNodes.sort((a, b) => a.currentCost - b.currentCost); const expectedCosts = [0.5, .25, 0.15, 0.1].map(x => -Math.log(x)) @@ -607,6 +645,7 @@ describe('Correction Distance Modeler', () => { assert.isTrue(subsetNodes[i].hasPartialInput); // From root: the deleteLeft 1 entries have nothing to insert. assert.equal((subsetNodes[i].currentTraversal as TrieTraversal).prefix, ''); + assert.equal(subsetNodes[i].spaceId, subsetSeed); // Allow a little value wiggle due to double-precision limitations. assert.approximately(subsetNodes[i].inputSamplingCost, expectedCosts[i], 1e-8); @@ -617,15 +656,18 @@ describe('Correction Distance Modeler', () => { it('step 2: first processing layer resolves zero + one char inserts', () => { // From "step 1" above, assertions removed - let rootTraversal = testModel.traverseFromRoot(); - let rootNode = new correction.SearchNode(rootTraversal); - const subsetNodes = rootNode.buildSubstitutionEdges(synthDistribution); + const rootTraversal = testModel.traverseFromRoot(); + const rootNode = new correction.SearchNode(rootTraversal, SEARCH_EDGE_SEED++); + const subsetSeed = SEARCH_EDGE_SEED++; + const subsetNodes = rootNode.buildSubstitutionEdges(synthDistribution, subsetSeed); subsetNodes.sort((a, b) => a.currentCost - b.currentCost); + subsetNodes.forEach(n => assert.equal(n.spaceId, subsetSeed)); // Set 0: set for ins 1, dl 0 const ins1_dl0 = subsetNodes[0].processSubsetEdge(); // 3 transforms went in... but 1 ('x') had no lexical match. assert.equal(ins1_dl0.length, FIRST_CHAR_VARIANTS + 3 - 1); + ins1_dl0.forEach(n => assert.equal(n.spaceId, subsetSeed)); ins1_dl0.sort((a, b) => a.currentCost - b.currentCost); ins1_dl0.forEach(n => assert.isFalse(n.hasPartialInput)); // all fully-processed. @@ -673,6 +715,7 @@ describe('Correction Distance Modeler', () => { // No inserts, so no insert variants are possible. assert.equal(ins0_dl1.length, 1); + ins0_dl1.forEach(n => assert.equal(n.spaceId, subsetSeed)); assert.isFalse(ins0_dl1[0].hasPartialInput); // No insert string => no sentinel char to delete. @@ -691,6 +734,7 @@ describe('Correction Distance Modeler', () => { ins2_dl1.sort((a, b) => a.currentCost - b.currentCost); // only one char is processed at this stage. ins2_dl1.forEach(n => assert.isTrue(n.hasPartialInput)); + ins2_dl1.forEach(n => assert.equal(n.spaceId, subsetSeed)); assert.equal(ins2_dl1[0].calculation.lastInputEntry, 't'); assert.equal(ins2_dl1[0].calculation.lastMatchEntry, 't'); @@ -722,6 +766,7 @@ describe('Correction Distance Modeler', () => { ins2_dl0.sort((a, b) => a.currentCost - b.currentCost); // only one char is processed at this stage. ins2_dl0.forEach(n => assert.isTrue(n.hasPartialInput)); + ins2_dl0.forEach(n => assert.equal(n.spaceId, subsetSeed)); assert.equal(ins2_dl0[0].calculation.lastInputEntry, 'c'); assert.equal(ins2_dl0[0].calculation.lastMatchEntry, 'c'); @@ -745,9 +790,10 @@ describe('Correction Distance Modeler', () => { it('step 3: second processing layer resolves two char inserts', () => { // From "steps 0, 1" above, assertions removed - let rootTraversal = testModel.traverseFromRoot(); - let rootNode = new correction.SearchNode(rootTraversal); - const subsetNodes = rootNode.buildSubstitutionEdges(synthDistribution); + const rootTraversal = testModel.traverseFromRoot(); + const rootNode = new correction.SearchNode(rootTraversal, SEARCH_EDGE_SEED++); + const subsetSeed = SEARCH_EDGE_SEED++; + const subsetNodes = rootNode.buildSubstitutionEdges(synthDistribution, subsetSeed); subsetNodes.sort((a, b) => a.currentCost - b.currentCost); @@ -762,6 +808,9 @@ describe('Correction Distance Modeler', () => { // All should be finished now! fin_in2_dl1.forEach(n => assert.isFalse(n.hasPartialInput)); + fin_in2_dl1.forEach(n => assert.equal(n.spaceId, subsetSeed)); + fin_in2_dl0.forEach(n => assert.equal(n.spaceId, subsetSeed)); + fin_in2_dl1.sort((a, b) => a.currentCost - b.currentCost); fin_in2_dl0.sort((a, b) => a.currentCost - b.currentCost); @@ -837,7 +886,7 @@ describe('Correction Distance Modeler', () => { const teNode = fetchCommonTENode(); assert.equal(teNode.calculation.getHeuristicFinalCost(), 0); - const subsetNodes = teNode.buildSubstitutionEdges(synthDistribution); + const subsetNodes = teNode.buildSubstitutionEdges(synthDistribution, SEARCH_EDGE_SEED++); assert.equal(subsetNodes.length, 4); subsetNodes.sort((a, b) => a.currentCost - b.currentCost); const expectedCosts = [0.5, .25, 0.15, 0.1].map(x => -Math.log(x) + teNode.currentCost); @@ -857,7 +906,7 @@ describe('Correction Distance Modeler', () => { it('step 2: first processing layer resolves zero + one char inserts', () => { // From "step 1" above, assertions removed const teNode = fetchCommonTENode(); - const subsetNodes = teNode.buildSubstitutionEdges(synthDistribution); + const subsetNodes = teNode.buildSubstitutionEdges(synthDistribution, SEARCH_EDGE_SEED++); subsetNodes.sort((a, b) => a.currentCost - b.currentCost); // Set 0: set for ins 1, dl 0 @@ -986,7 +1035,7 @@ describe('Correction Distance Modeler', () => { it('step 3: second processing layer resolves two char inserts', () => { // From "steps 0, 1" above, assertions removed const teNode = fetchCommonTENode(); - const subsetNodes = teNode.buildSubstitutionEdges(synthDistribution); + const subsetNodes = teNode.buildSubstitutionEdges(synthDistribution, SEARCH_EDGE_SEED++); subsetNodes.sort((a, b) => a.currentCost - b.currentCost); // ************ @@ -1054,60 +1103,67 @@ describe('Correction Distance Modeler', () => { }); }); - it('Small integration test: "teh" => "ten", "the"', function() { + it('Small integration test: "teh" => "ten", "the"', () => { // The combinatorial effect here is a bit much to fully test. - let rootTraversal = testModel.traverseFromRoot(); + const rootTraversal = testModel.traverseFromRoot(); assert.isNotEmpty(rootTraversal); - let rootNode = new correction.SearchNode(rootTraversal); + const rootSeed = SEARCH_EDGE_SEED++; + const rootNode = new correction.SearchNode(rootTraversal, rootSeed); assert.equal(rootNode.calculation.getHeuristicFinalCost(), 0); // VERY artificial distributions. - let synthDistribution1 = [ + const synthDistribution1 = [ {sample: {insert: 't', deleteLeft: 0}, p: 1} // Transform, probability ]; - let synthDistribution2 = [ + const synthDistribution2 = [ {sample: {insert: 'e', deleteLeft: 0}, p: 0.75}, // Transform, probability {sample: {insert: 'h', deleteLeft: 0}, p: 0.25} ]; - let synthDistribution3 = [ + const synthDistribution3 = [ {sample: {insert: 'h', deleteLeft: 0}, p: 0.75}, // Transform, probability {sample: {insert: 'n', deleteLeft: 0}, p: 0.25} ]; - let layer1Edges = rootNode.buildSubstitutionEdges(synthDistribution1) + const layer1Id = SEARCH_EDGE_SEED++; + const layer1Edges = rootNode.buildSubstitutionEdges(synthDistribution1, layer1Id) // No 2+ inserts here; we're fine with just one call. .flatMap(e => e.processSubsetEdge()); const layer1Queue = new PriorityQueue(QUEUE_NODE_COMPARATOR, layer1Edges); - let tEdge = layer1Queue.dequeue(); + const tEdge = layer1Queue.dequeue(); assertEdgeChars(tEdge, 't', 't'); + assert.equal(tEdge.spaceId, layer1Id); // would be obtained by the token after one input. - let layer2Edges = tEdge.buildSubstitutionEdges(synthDistribution2) + const layer2Id = SEARCH_EDGE_SEED++; + const layer2Edges = tEdge.buildSubstitutionEdges(synthDistribution2, layer2Id) // No 2+ inserts here; we're fine with just one call. .flatMap(e => e.processSubsetEdge()); const layer2Queue = new PriorityQueue(QUEUE_NODE_COMPARATOR, layer2Edges); - let eEdge = layer2Queue.dequeue(); + const eEdge = layer2Queue.dequeue(); assertEdgeChars(eEdge, 'e', 'e'); + assert.equal(eEdge.spaceId, layer2Id); - let hEdge = layer2Queue.dequeue(); + const hEdge = layer2Queue.dequeue(); assertEdgeChars(hEdge, 'h', 'h'); + assert.equal(hEdge.spaceId, layer2Id); // Needed for a proper e <-> h transposition. - let ehEdge = findEdgesWithChars(layer2Edges, 'h')[0]; + const ehEdge = findEdgesWithChars(layer2Edges, 'h')[0]; assert.isOk(ehEdge); // Final round: we'll use three nodes and throw all of their results into the same priority queue. - let layer3eEdges = eEdge.buildSubstitutionEdges(synthDistribution3) + const layer3Id = SEARCH_EDGE_SEED++; + const layer3eEdges = eEdge.buildSubstitutionEdges(synthDistribution3, layer3Id) // No 2+ inserts here; we're fine with just one call. .flatMap(e => e.processSubsetEdge()); - let layer3hEdges = hEdge.buildSubstitutionEdges(synthDistribution3) + const layer3hEdges = hEdge.buildSubstitutionEdges(synthDistribution3, layer3Id) .flatMap(e => e.processSubsetEdge()); - let layer3ehEdges = ehEdge.buildSubstitutionEdges(synthDistribution3) + const layer3ehEdges = ehEdge.buildSubstitutionEdges(synthDistribution3, layer3Id) .flatMap(e => e.processSubsetEdge()); const layer3Queue = new PriorityQueue(QUEUE_NODE_COMPARATOR, layer3eEdges.concat(layer3hEdges).concat(layer3ehEdges)); @@ -1118,12 +1174,14 @@ describe('Correction Distance Modeler', () => { } while(bestEdge.currentTraversal.entries.length == 0); assertEdgeChars(bestEdge, 'n', 'n'); // 'ten' - perfect edit distance of 0, though less-likely input sequence. + assert.equal(bestEdge.spaceId, layer3Id); // No cost assumptions here. var sibling1; do { sibling1 = layer3Queue.dequeue(); } while(sibling1.currentTraversal.entries.length == 0); + assert.equal(sibling1.spaceId, layer3Id); // Both have a raw edit distance of 1 while using the same input-sequence root. ('th') let tenFlag = edgeHasChars(sibling1, SENTINEL_CODE_UNIT, 'n'); // subs out the 'h' entirely. Could also occur with 'a', but is too unlikely. @@ -1136,6 +1194,7 @@ describe('Correction Distance Modeler', () => { do { sibling2 = layer3Queue.dequeue(); } while(sibling2.currentTraversal.entries.length == 0); + assert.equal(sibling2.spaceId, layer3Id); tenFlag = tenFlag || edgeHasChars(sibling2, SENTINEL_CODE_UNIT, 'n'); theFlag = theFlag || edgeHasChars(sibling2, SENTINEL_CODE_UNIT, 'e'); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/getBestMatches.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/getBestMatches.tests.ts index 621679cfe7..75582eee33 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/getBestMatches.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/getBestMatches.tests.ts @@ -126,11 +126,15 @@ describe('getBestMatches', () => { {sample: {insert: 'n', deleteLeft: 0}, p: 0.25} ]; - searchPath = searchPath.addInput(synthInput1, 1); - searchPath = searchPath.addInput(synthInput2, .75); - searchPath = searchPath.addInput(synthInput3, .75); + const searchPath1 = searchPath.addInput(synthInput1, 1); + const searchPath2 = searchPath1.addInput(synthInput2, .75); + const searchPath3 = searchPath2.addInput(synthInput3, .75); - const iter = getBestMatches(searchPath, buildTestTimer()); // disables the correction-search timeout. + assert.notEqual(searchPath1.spaceId, searchPath.spaceId); + assert.notEqual(searchPath2.spaceId, searchPath1.spaceId); + assert.notEqual(searchPath3.spaceId, searchPath2.spaceId); + + const iter = getBestMatches(searchPath3, buildTestTimer()); // disables the correction-search timeout. await checkRepeatableResults_teh(iter); }); @@ -139,7 +143,7 @@ describe('getBestMatches', () => { const rootTraversal = testModel.traverseFromRoot(); assert.isNotEmpty(rootTraversal); - let searchSpace = new SearchQuotientSpur(testModel); + let searchPath = new SearchQuotientSpur(testModel); // VERY artificial distributions. const synthInput1 = [ @@ -156,16 +160,20 @@ describe('getBestMatches', () => { {sample: {insert: 'n', deleteLeft: 0}, p: 0.25} ]; - searchSpace = searchSpace.addInput(synthInput1, 1); - searchSpace = searchSpace.addInput(synthInput2, .75); - searchSpace = searchSpace.addInput(synthInput3, .75); + const searchPath1 = searchPath.addInput(synthInput1, 1); + const searchPath2 = searchPath1.addInput(synthInput2, .75); + const searchPath3 = searchPath2.addInput(synthInput3, .75); - const iter = getBestMatches(searchSpace, buildTestTimer()); // disables the correction-search timeout. + assert.notEqual(searchPath1.spaceId, searchPath.spaceId); + assert.notEqual(searchPath2.spaceId, searchPath1.spaceId); + assert.notEqual(searchPath3.spaceId, searchPath2.spaceId); + + const iter = getBestMatches(searchPath3, buildTestTimer()); // disables the correction-search timeout. await checkRepeatableResults_teh(iter); // The key: do we get the same results the second time? // Reset the iterator first... - const iter2 = getBestMatches(searchSpace, buildTestTimer()); // disables the correction-search timeout. + const iter2 = getBestMatches(searchPath3, buildTestTimer()); // disables the correction-search timeout. await checkRepeatableResults_teh(iter2); });