feat(web): start support for calculating 'split', 'merge' edits

web/src/engine/predictive-text/worker-thread/src/main/correction/classical-calculation.ts

@@ -619,47 +619,56 @@ export class ClassicalDistanceCalculation<TUnit = string> {
     return trimmedInstance;
   }
 
+  // Inputs add an extra row / first index entry.
   // Inputs add an extra row / first index entry.
   addInputChar(token: TUnit): ClassicalDistanceCalculation<TUnit> {
-    const returnBuffer = new ClassicalDistanceCalculation(this);
-
-    // Insert a row, even if we don't actually do anything with it yet.
-    // Initialize all entries with Number.MAX_VALUE, as `undefined` use leads to JS math issues.
-    const row = Array(2 * returnBuffer.diagonalWidth + 1).fill(Number.MAX_VALUE);
-    returnBuffer.resolvedDistances[returnBuffer.inputSequence.length] = row;
-
-    returnBuffer.inputSequence.push(token);
-
-    // If there isn't a 'match' entry yet, there are no values to compute.  Exit immediately.
-    if(returnBuffer.matchSequence.length == 0) {
-      return returnBuffer;
-    }
-
-    forNewIndices(returnBuffer, true, function(r, c, diagIndex) {
-      row[diagIndex] = ClassicalDistanceCalculation.initialCostAt(returnBuffer, r, c);
-    });
-
+    const returnBuffer = new ClassicalDistanceCalculation<TUnit>(this);
+    returnBuffer._addInputChar(token);
     return returnBuffer;
   }
 
-  addMatchChar(token: TUnit): ClassicalDistanceCalculation<TUnit> {
-    const returnBuffer = new ClassicalDistanceCalculation(this);
-    returnBuffer.matchSequence.push(token);
+  protected _addInputChar(token: TUnit) {
+    // Insert a row, even if we don't actually do anything with it yet.
+    // Initialize all entries with Number.MAX_VALUE, as `undefined` use leads to JS math issues.
+    const row = Array(2 * this.diagonalWidth + 1).fill(Number.MAX_VALUE);
+    this.resolvedDistances[this.inputSequence.length] = row;
+    this.inputSequence.push(token);
 
     // If there isn't a 'match' entry yet, there are no values to compute.  Exit immediately.
-    if(returnBuffer.inputSequence.length == 0) {
-      return returnBuffer;
+    if(this.matchSequence.length == 0) {
+      return;
     }
 
-    forNewIndices(returnBuffer, false, (r, c, diagIndex) => {
-      const row = returnBuffer.resolvedDistances[r];
-      // Since diagIndex is from the perspective of the row, it must be inverted to properly index the column.
-      row[diagIndex] = ClassicalDistanceCalculation.initialCostAt(returnBuffer, r, c);
+    forNewIndices(this, true, (r, c, diagIndex) => {
+      row[diagIndex] = ClassicalDistanceCalculation.initialCostAt(this, r, c);
     });
 
+    return;
+  }
+
+  addMatchChar(token: TUnit): ClassicalDistanceCalculation<TUnit> {
+    let returnBuffer = new ClassicalDistanceCalculation(this);
+    returnBuffer._addMatchChar(token);
     return returnBuffer;
   }
 
+  protected _addMatchChar(token: TUnit) {
+    this.matchSequence.push(token);
+
+    // If there isn't a 'match' entry yet, there are no values to compute.  Exit immediately.
+    if(this.inputSequence.length == 0) {
+      return;
+    }
+
+    forNewIndices(this, false, (r, c, diagIndex) => {
+      var row = this.resolvedDistances[r];
+      // Since diagIndex is from the perspective of the row, it must be inverted to properly index the column.
+      row[diagIndex] = ClassicalDistanceCalculation.initialCostAt(this, r, c);
+    });
+
+    return;
+  }
+
   public increaseMaxDistance(): ClassicalDistanceCalculation<TUnit> {
     let returnBuffer = new ClassicalDistanceCalculation(this);
     returnBuffer.diagonalWidth++;

web/src/engine/predictive-text/worker-thread/src/main/correction/segmentable-calculation.ts

@@ -0,0 +1,134 @@
+import { ClassicalDistanceCalculation, EditOperation, forNewIndices } from './classical-calculation.js';
+
+/**
+ * The human-readable names for legal edit-operation edges on a merge-split
+ * supporting edit path.
+ */
+export type ExtendedEditOperation = 'merge' | 'split' | EditOperation;
+
+/**
+ * This is an optimized edit-distance calculation engine that supports two
+ * additional edit-distance operations on strings:  'merge' and 'split'.
+ *
+ * - 'merge' allows two or more input tokens to be directly combined, with no further
+ * edits, into a single match token.
+ * - 'split' allows one input token to be directly split, with no further edits, into
+ * two or more match tokens.
+ */
+export class SegmentableDistanceCalculation extends ClassicalDistanceCalculation<string> {
+  /**
+   * Constructs a new calculation object instance.
+   */
+  constructor();
+  /**
+   * Clones an already-existing instance, aliasing old data only where safe.
+   * @param other
+   */
+  constructor(other: SegmentableDistanceCalculation);
+  constructor(other?: SegmentableDistanceCalculation) {
+    super(other);
+  }
+
+  private static getMergeSplitParent(
+    buffer: SegmentableDistanceCalculation,
+    r: number,
+    c: number
+  ): [number, number] {
+    const mergeTarget = buffer.matchSequence[c];
+    const splitTarget = buffer.inputSequence[r];
+
+    // Block any operations where the initial tokens are identical.
+    // Other operations will be cheaper.  Also, block cases where 'parents' are impossible.
+    if(r < 0 || c < 0 || splitTarget == mergeTarget) {
+      return [-1, -1];
+    }
+
+
+    // Merge checks
+    let mergedInputs = '';
+    let lastMergeIndex = -1;
+    for(let i = r; i >= 0; i--) {
+      mergedInputs = buffer.inputSequence[i] + mergedInputs;
+      if(mergedInputs == mergeTarget) {
+        lastMergeIndex = i;
+        break;
+      } else if(mergedInputs.length > mergeTarget.length) {
+        break;
+      }
+    }
+
+    // Split checks
+    let mergedMatches = '';
+    let lastSplitIndex = -1;
+    for(let i = c; i >= 0; i--) {
+      mergedMatches = buffer.matchSequence[i] + mergedMatches;
+      if(mergedMatches == splitTarget) {
+        lastSplitIndex = i;
+        break;
+      } else if(mergedMatches.length > splitTarget.length) {
+        break;
+      }
+    }
+
+    return [lastMergeIndex, lastSplitIndex];
+  }
+
+  private static selectInitialCostAt(
+    buffer: SegmentableDistanceCalculation,
+    r: number,
+    c: number
+  ) {
+    const baseCost = buffer.getCostAt(r, c);
+    let mergeCost = Number.MAX_VALUE;
+    let splitCost = Number.MAX_VALUE;
+
+    const [lastMergeIndex, lastSplitIndex] = SegmentableDistanceCalculation.getMergeSplitParent(buffer, r, c);
+    mergeCost = lastMergeIndex == -1 ? mergeCost : (buffer.getCostAt(lastMergeIndex-1, c-1) + 1);
+    splitCost = lastSplitIndex == -1 ? splitCost : (buffer.getCostAt(r-1, lastSplitIndex-1) + 1);
+
+    return Math.min(baseCost, mergeCost, splitCost);
+  }
+
+  addInputChar(token: string): SegmentableDistanceCalculation {
+    const returnBuffer = new SegmentableDistanceCalculation(this);
+    returnBuffer._addInputChar(token);
+
+    // If there isn't a 'match' entry yet, there are no values to compute.  Exit immediately.
+    if(this.matchSequence.length == 0) {
+      return returnBuffer;
+    }
+
+    // Also do new edit types!
+    const r = returnBuffer.inputSequence.length - 1;
+    const row = returnBuffer.resolvedDistances[r];
+    forNewIndices(returnBuffer, true, (r, c, diagIndex) => {
+      row[diagIndex] = SegmentableDistanceCalculation.selectInitialCostAt(returnBuffer, r, c);
+    });
+
+    return returnBuffer;
+  }
+
+  addMatchChar(token: string): SegmentableDistanceCalculation {
+    const returnBuffer = new SegmentableDistanceCalculation(this);
+    returnBuffer._addMatchChar(token);
+
+    // If there isn't an 'input' entry yet, there are no values to compute.  Exit immediately.
+    if(this.matchSequence.length == 0) {
+      return returnBuffer;
+    }
+
+    // Also do new edit types!
+    forNewIndices(returnBuffer, false, (r, c, diagIndex) => {
+      const row = returnBuffer.resolvedDistances[r];
+      // Since diagIndex is from the perspective of the row, it must be inverted to properly index the column.
+      row[diagIndex] = SegmentableDistanceCalculation.selectInitialCostAt(returnBuffer, r, c);
+    });
+
+    return returnBuffer;
+  }
+
+  // TODO:  diagonal expansion
+  // TODO:  edit path with 'split', 'merge' handling  // progress:  infrastructure is likely prepped?
+  // TODO:  visualization
+}
+

web/src/engine/predictive-text/worker-thread/src/main/test-index.ts

@@ -5,6 +5,7 @@ export { ContextTokenization } from './correction/context-tokenization.js';
 export { ContextTracker } from './correction/context-tracker.js';
 export { ContextTransition } from './correction/context-transition.js';
 export * from './correction/alignment-helpers.js';
+export { SegmentableDistanceCalculation } from './correction/segmentable-calculation.js';
 export * as correction from './correction/index.js';
 export * from './model-helpers.js';
 export * as models from './models/index.js';

web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/segmentable-calculation.tests.ts

@@ -0,0 +1,79 @@
+import { assert } from 'chai';
+import { SegmentableDistanceCalculation } from '@keymanapp/lm-worker/test-index';
+
+describe('Split/merge aware edit-distance calculation', () => {
+  it("a,b,c -> a,b,d = 1", () => {
+    let calc = new SegmentableDistanceCalculation();
+    calc.diagonalWidth = 2;
+
+    ['a', 'b', 'c'].forEach(c => calc = calc.addInputChar(c));
+    ['a', 'b', 'd'].forEach(c => calc = calc.addMatchChar(c));
+
+    assert.equal(calc.getFinalCost(), 1);
+  });
+
+  it("a,b,c -> a,bc = 1", () => {
+    let calc = new SegmentableDistanceCalculation();
+    calc.diagonalWidth = 2;
+
+    ['a', 'b', 'c'].forEach(c => calc = calc.addInputChar(c));
+    ['a', 'bc'].forEach(c => calc = calc.addMatchChar(c));
+
+    assert.equal(calc.getFinalCost(), 1);
+  });
+
+  it("ab,c,d -> a,b,cd = 2", () => {
+    let calc = new SegmentableDistanceCalculation();
+    calc.diagonalWidth = 2;
+
+    ['ab', 'c', 'd'].forEach(c => calc = calc.addInputChar(c));
+    ['a', 'b', 'cd'].forEach(c => calc = calc.addMatchChar(c));
+
+    assert.equal(calc.getFinalCost(), 2);
+  });
+
+  it("a,bc,d -> a,b,c,d = 1", () => {
+    let calc = new SegmentableDistanceCalculation();
+    calc.diagonalWidth = 2;
+
+    ['a', 'bc', 'd'].forEach(c => calc = calc.addInputChar(c));
+    ['a', 'b', 'c', 'd'].forEach(c => calc = calc.addMatchChar(c));
+
+    assert.equal(calc.getFinalCost(), 1);
+  });
+
+  it("a,b,c,d -> a,bc,d = 1", () => {
+    let calc = new SegmentableDistanceCalculation();
+    calc.diagonalWidth = 2;
+
+    ['a', 'b', 'c', 'd'].forEach(c => calc = calc.addInputChar(c));
+    ['a', 'bc', 'd'].forEach(c => calc = calc.addMatchChar(c));
+
+    assert.equal(calc.getFinalCost(), 1);
+  });
+
+  it("ab,d,c,e,g,h -> a,b,c,d,f,gh = 4", () => {
+    let calc = new SegmentableDistanceCalculation();
+    calc.diagonalWidth = 4;
+
+    // 'ab' => 'a', 'b'
+    // d,c transposed
+    // e => f substitution
+    // 'g', 'h' => 'gh'
+
+    ['ab', 'd', 'c', 'e', 'g', 'h'].forEach(c => calc = calc.addInputChar(c));
+    ['a', 'b', 'c', 'd', 'f', 'gh'].forEach(c => calc = calc.addMatchChar(c));
+
+    assert.equal(calc.getFinalCost(), 4);
+  });
+
+  it("abc,d,f,g,h -> a,b,c,d,fgh = 2", () => {
+    let calc = new SegmentableDistanceCalculation();
+    calc.diagonalWidth = 4;
+
+    ['abc', 'd', 'f', 'g', 'h'].forEach(c => calc = calc.addInputChar(c));
+    ['a', 'b', 'c', 'd', 'fgh'].forEach(c => calc = calc.addMatchChar(c));
+
+    assert.equal(calc.getFinalCost(), 2);
+  });
+});