From 8ca4b8b3cd9c878cd50545e837ddb7134d64560c Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 4 Sep 2025 11:19:31 -0500 Subject: [PATCH] feat(web): start support for calculating 'split', 'merge' edits --- .../main/correction/classical-calculation.ts | 63 ++++---- .../correction/segmentable-calculation.ts | 134 ++++++++++++++++++ .../worker-thread/src/main/test-index.ts | 1 + .../segmentable-calculation.tests.ts | 79 +++++++++++ 4 files changed, 250 insertions(+), 27 deletions(-) create mode 100644 web/src/engine/predictive-text/worker-thread/src/main/correction/segmentable-calculation.ts create mode 100644 web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/segmentable-calculation.tests.ts diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/classical-calculation.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/classical-calculation.ts index 8d8823ee86..152c9cf2cf 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/classical-calculation.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/classical-calculation.ts @@ -619,47 +619,56 @@ export class ClassicalDistanceCalculation { return trimmedInstance; } + // Inputs add an extra row / first index entry. // Inputs add an extra row / first index entry. addInputChar(token: TUnit): ClassicalDistanceCalculation { - const returnBuffer = new ClassicalDistanceCalculation(this); - - // Insert a row, even if we don't actually do anything with it yet. - // Initialize all entries with Number.MAX_VALUE, as `undefined` use leads to JS math issues. - const row = Array(2 * returnBuffer.diagonalWidth + 1).fill(Number.MAX_VALUE); - returnBuffer.resolvedDistances[returnBuffer.inputSequence.length] = row; - - returnBuffer.inputSequence.push(token); - - // If there isn't a 'match' entry yet, there are no values to compute. Exit immediately. - if(returnBuffer.matchSequence.length == 0) { - return returnBuffer; - } - - forNewIndices(returnBuffer, true, function(r, c, diagIndex) { - row[diagIndex] = ClassicalDistanceCalculation.initialCostAt(returnBuffer, r, c); - }); - + const returnBuffer = new ClassicalDistanceCalculation(this); + returnBuffer._addInputChar(token); return returnBuffer; } - addMatchChar(token: TUnit): ClassicalDistanceCalculation { - const returnBuffer = new ClassicalDistanceCalculation(this); - returnBuffer.matchSequence.push(token); + protected _addInputChar(token: TUnit) { + // Insert a row, even if we don't actually do anything with it yet. + // Initialize all entries with Number.MAX_VALUE, as `undefined` use leads to JS math issues. + const row = Array(2 * this.diagonalWidth + 1).fill(Number.MAX_VALUE); + this.resolvedDistances[this.inputSequence.length] = row; + this.inputSequence.push(token); // If there isn't a 'match' entry yet, there are no values to compute. Exit immediately. - if(returnBuffer.inputSequence.length == 0) { - return returnBuffer; + if(this.matchSequence.length == 0) { + return; } - forNewIndices(returnBuffer, false, (r, c, diagIndex) => { - const row = returnBuffer.resolvedDistances[r]; - // Since diagIndex is from the perspective of the row, it must be inverted to properly index the column. - row[diagIndex] = ClassicalDistanceCalculation.initialCostAt(returnBuffer, r, c); + forNewIndices(this, true, (r, c, diagIndex) => { + row[diagIndex] = ClassicalDistanceCalculation.initialCostAt(this, r, c); }); + return; + } + + addMatchChar(token: TUnit): ClassicalDistanceCalculation { + let returnBuffer = new ClassicalDistanceCalculation(this); + returnBuffer._addMatchChar(token); return returnBuffer; } + protected _addMatchChar(token: TUnit) { + this.matchSequence.push(token); + + // If there isn't a 'match' entry yet, there are no values to compute. Exit immediately. + if(this.inputSequence.length == 0) { + return; + } + + forNewIndices(this, false, (r, c, diagIndex) => { + var row = this.resolvedDistances[r]; + // Since diagIndex is from the perspective of the row, it must be inverted to properly index the column. + row[diagIndex] = ClassicalDistanceCalculation.initialCostAt(this, r, c); + }); + + return; + } + public increaseMaxDistance(): ClassicalDistanceCalculation { let returnBuffer = new ClassicalDistanceCalculation(this); returnBuffer.diagonalWidth++; diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/segmentable-calculation.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/segmentable-calculation.ts new file mode 100644 index 0000000000..797f9182e1 --- /dev/null +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/segmentable-calculation.ts @@ -0,0 +1,134 @@ +import { ClassicalDistanceCalculation, EditOperation, forNewIndices } from './classical-calculation.js'; + +/** + * The human-readable names for legal edit-operation edges on a merge-split + * supporting edit path. + */ +export type ExtendedEditOperation = 'merge' | 'split' | EditOperation; + +/** + * This is an optimized edit-distance calculation engine that supports two + * additional edit-distance operations on strings: 'merge' and 'split'. + * + * - 'merge' allows two or more input tokens to be directly combined, with no further + * edits, into a single match token. + * - 'split' allows one input token to be directly split, with no further edits, into + * two or more match tokens. + */ +export class SegmentableDistanceCalculation extends ClassicalDistanceCalculation { + /** + * Constructs a new calculation object instance. + */ + constructor(); + /** + * Clones an already-existing instance, aliasing old data only where safe. + * @param other + */ + constructor(other: SegmentableDistanceCalculation); + constructor(other?: SegmentableDistanceCalculation) { + super(other); + } + + private static getMergeSplitParent( + buffer: SegmentableDistanceCalculation, + r: number, + c: number + ): [number, number] { + const mergeTarget = buffer.matchSequence[c]; + const splitTarget = buffer.inputSequence[r]; + + // Block any operations where the initial tokens are identical. + // Other operations will be cheaper. Also, block cases where 'parents' are impossible. + if(r < 0 || c < 0 || splitTarget == mergeTarget) { + return [-1, -1]; + } + + + // Merge checks + let mergedInputs = ''; + let lastMergeIndex = -1; + for(let i = r; i >= 0; i--) { + mergedInputs = buffer.inputSequence[i] + mergedInputs; + if(mergedInputs == mergeTarget) { + lastMergeIndex = i; + break; + } else if(mergedInputs.length > mergeTarget.length) { + break; + } + } + + // Split checks + let mergedMatches = ''; + let lastSplitIndex = -1; + for(let i = c; i >= 0; i--) { + mergedMatches = buffer.matchSequence[i] + mergedMatches; + if(mergedMatches == splitTarget) { + lastSplitIndex = i; + break; + } else if(mergedMatches.length > splitTarget.length) { + break; + } + } + + return [lastMergeIndex, lastSplitIndex]; + } + + private static selectInitialCostAt( + buffer: SegmentableDistanceCalculation, + r: number, + c: number + ) { + const baseCost = buffer.getCostAt(r, c); + let mergeCost = Number.MAX_VALUE; + let splitCost = Number.MAX_VALUE; + + const [lastMergeIndex, lastSplitIndex] = SegmentableDistanceCalculation.getMergeSplitParent(buffer, r, c); + mergeCost = lastMergeIndex == -1 ? mergeCost : (buffer.getCostAt(lastMergeIndex-1, c-1) + 1); + splitCost = lastSplitIndex == -1 ? splitCost : (buffer.getCostAt(r-1, lastSplitIndex-1) + 1); + + return Math.min(baseCost, mergeCost, splitCost); + } + + addInputChar(token: string): SegmentableDistanceCalculation { + const returnBuffer = new SegmentableDistanceCalculation(this); + returnBuffer._addInputChar(token); + + // If there isn't a 'match' entry yet, there are no values to compute. Exit immediately. + if(this.matchSequence.length == 0) { + return returnBuffer; + } + + // Also do new edit types! + const r = returnBuffer.inputSequence.length - 1; + const row = returnBuffer.resolvedDistances[r]; + forNewIndices(returnBuffer, true, (r, c, diagIndex) => { + row[diagIndex] = SegmentableDistanceCalculation.selectInitialCostAt(returnBuffer, r, c); + }); + + return returnBuffer; + } + + addMatchChar(token: string): SegmentableDistanceCalculation { + const returnBuffer = new SegmentableDistanceCalculation(this); + returnBuffer._addMatchChar(token); + + // If there isn't an 'input' entry yet, there are no values to compute. Exit immediately. + if(this.matchSequence.length == 0) { + return returnBuffer; + } + + // Also do new edit types! + forNewIndices(returnBuffer, false, (r, c, diagIndex) => { + const row = returnBuffer.resolvedDistances[r]; + // Since diagIndex is from the perspective of the row, it must be inverted to properly index the column. + row[diagIndex] = SegmentableDistanceCalculation.selectInitialCostAt(returnBuffer, r, c); + }); + + return returnBuffer; + } + + // TODO: diagonal expansion + // TODO: edit path with 'split', 'merge' handling // progress: infrastructure is likely prepped? + // TODO: visualization +} + diff --git a/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts b/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts index 052851dde7..436543ada1 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts @@ -5,6 +5,7 @@ export { ContextTokenization } from './correction/context-tokenization.js'; export { ContextTracker } from './correction/context-tracker.js'; export { ContextTransition } from './correction/context-transition.js'; export * from './correction/alignment-helpers.js'; +export { SegmentableDistanceCalculation } from './correction/segmentable-calculation.js'; export * as correction from './correction/index.js'; export * from './model-helpers.js'; export * as models from './models/index.js'; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/segmentable-calculation.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/segmentable-calculation.tests.ts new file mode 100644 index 0000000000..1f3d39c89a --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/segmentable-calculation.tests.ts @@ -0,0 +1,79 @@ +import { assert } from 'chai'; +import { SegmentableDistanceCalculation } from '@keymanapp/lm-worker/test-index'; + +describe('Split/merge aware edit-distance calculation', () => { + it("a,b,c -> a,b,d = 1", () => { + let calc = new SegmentableDistanceCalculation(); + calc.diagonalWidth = 2; + + ['a', 'b', 'c'].forEach(c => calc = calc.addInputChar(c)); + ['a', 'b', 'd'].forEach(c => calc = calc.addMatchChar(c)); + + assert.equal(calc.getFinalCost(), 1); + }); + + it("a,b,c -> a,bc = 1", () => { + let calc = new SegmentableDistanceCalculation(); + calc.diagonalWidth = 2; + + ['a', 'b', 'c'].forEach(c => calc = calc.addInputChar(c)); + ['a', 'bc'].forEach(c => calc = calc.addMatchChar(c)); + + assert.equal(calc.getFinalCost(), 1); + }); + + it("ab,c,d -> a,b,cd = 2", () => { + let calc = new SegmentableDistanceCalculation(); + calc.diagonalWidth = 2; + + ['ab', 'c', 'd'].forEach(c => calc = calc.addInputChar(c)); + ['a', 'b', 'cd'].forEach(c => calc = calc.addMatchChar(c)); + + assert.equal(calc.getFinalCost(), 2); + }); + + it("a,bc,d -> a,b,c,d = 1", () => { + let calc = new SegmentableDistanceCalculation(); + calc.diagonalWidth = 2; + + ['a', 'bc', 'd'].forEach(c => calc = calc.addInputChar(c)); + ['a', 'b', 'c', 'd'].forEach(c => calc = calc.addMatchChar(c)); + + assert.equal(calc.getFinalCost(), 1); + }); + + it("a,b,c,d -> a,bc,d = 1", () => { + let calc = new SegmentableDistanceCalculation(); + calc.diagonalWidth = 2; + + ['a', 'b', 'c', 'd'].forEach(c => calc = calc.addInputChar(c)); + ['a', 'bc', 'd'].forEach(c => calc = calc.addMatchChar(c)); + + assert.equal(calc.getFinalCost(), 1); + }); + + it("ab,d,c,e,g,h -> a,b,c,d,f,gh = 4", () => { + let calc = new SegmentableDistanceCalculation(); + calc.diagonalWidth = 4; + + // 'ab' => 'a', 'b' + // d,c transposed + // e => f substitution + // 'g', 'h' => 'gh' + + ['ab', 'd', 'c', 'e', 'g', 'h'].forEach(c => calc = calc.addInputChar(c)); + ['a', 'b', 'c', 'd', 'f', 'gh'].forEach(c => calc = calc.addMatchChar(c)); + + assert.equal(calc.getFinalCost(), 4); + }); + + it("abc,d,f,g,h -> a,b,c,d,fgh = 2", () => { + let calc = new SegmentableDistanceCalculation(); + calc.diagonalWidth = 4; + + ['abc', 'd', 'f', 'g', 'h'].forEach(c => calc = calc.addInputChar(c)); + ['a', 'b', 'c', 'd', 'fgh'].forEach(c => calc = calc.addMatchChar(c)); + + assert.equal(calc.getFinalCost(), 2); + }); +}); \ No newline at end of file