change(web): add transposition unit tests, loosen search correction thresholding

web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts

@@ -194,6 +194,7 @@ export class SearchNode {
       // This is unique at each level, though it will reuse a previous ID if no new
       // one is provided (say, for 'insert' edits).
       this.spaceId = spaceId ?? priorNode.spaceId;
+      this.addedEditCost = priorNode.addedEditCost;
     } else {
       this.calculation = new ClassicalDistanceCalculation();
       this.matchedTraversals = [param1];

web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts

@@ -61,27 +61,7 @@ export class LegacyQuotientSpur extends SearchQuotientSpur {
   }
 
   protected buildEdgesFromResults(priorResults: ReadonlyArray<TokenResultMapping>, inputs?: Distribution<Transform>): SearchNode[] {
-    const edgeInputs = inputs ?? this.inputs;
-
-    // With a newly-available input, we can extend new input-dependent paths from
-    // our previously-reached 'extractedResults' nodes.
-    let outboundNodes = priorResults.map((result) => {
-      // Hard restriction:  no further edits will be supported.  This helps keep the search
-      // more narrowly focused.
-      const substitutionsOnly = result.editCount == 2;
-
-      let deletionEdges: SearchNode[] = [];
-      if(!substitutionsOnly) {
-        deletionEdges         = result.buildDeletionEdges(edgeInputs, this.spaceId);
-      }
-      const substitutionEdges = result.buildSubstitutionEdges(edgeInputs, this.spaceId);
-
-      // Skip the queue for the first pass; there will ALWAYS be at least one pass,
-      // and queue-enqueing does come with a cost - avoid unnecessary overhead here.
-      return substitutionEdges.flatMap(e => e.processSubsetEdge()).concat(deletionEdges);
-    }).flat();
-
-    return outboundNodes;
+    return buildEdgesFromResults(priorResults, inputs ?? this.inputs, this.spaceId);
   }
 
   get currentCost() {
@@ -153,11 +133,42 @@ export class LegacyQuotientSpur extends SearchQuotientSpur {
   protected processPendingRoots(): void {
     super.processPendingRoots();
 
-    while(this.incomingTransposeRootNodes.length > 0) {
-      // Build only substitution edges from these.
-      const transpositionFirstHalves = this.incomingTransposeRootNodes.pop().buildSubstitutionEdges(this.inputs, this.spaceId);
-      transpositionFirstHalves.forEach((n) => n.addEdit());
+    if(this.incomingTransposeRootNodes.length > 0) {
+      const transpositionFirstHalves = processTransposeRoots(this.incomingTransposeRootNodes, this.inputs, this.spaceId);
+
+      this.incomingTransposeRootNodes.splice(0, this.incomingTransposeRootNodes.length);
       this.transposeQueue.enqueueAll(transpositionFirstHalves);
     }
   }
+}
+
+export function processTransposeRoots(priorResults: TokenResultMapping[], inputs: Distribution<Transform>, spaceId: number) {
+  // Build only substitution edges from these.
+  const transpositionFirstHalves = priorResults
+    .flatMap((entry) => entry.buildSubstitutionEdges(inputs, spaceId))
+    .flatMap(e => e.processSubsetEdge());
+  transpositionFirstHalves.forEach((n) => n.addEdit());
+  return transpositionFirstHalves;
+}
+
+export function buildEdgesFromResults(priorResults: ReadonlyArray<TokenResultMapping>, inputs: Distribution<Transform>, spaceId: number): SearchNode[] {
+  // With a newly-available input, we can extend new input-dependent paths from
+  // our previously-reached 'extractedResults' nodes.
+  let outboundNodes = priorResults.map((result) => {
+    // Hard restriction:  no further edits will be supported.  This helps keep the search
+    // more narrowly focused.
+    const substitutionsOnly = result.editCount == 2;
+
+    let deletionEdges: SearchNode[] = [];
+    if(!substitutionsOnly) {
+      deletionEdges         = result.buildDeletionEdges(inputs, spaceId);
+    }
+    const substitutionEdges = result.buildSubstitutionEdges(inputs, spaceId);
+
+    // Skip the queue for the first pass; there will ALWAYS be at least one pass,
+    // and queue-enqueing does come with a cost - avoid unnecessary overhead here.
+    return substitutionEdges.flatMap(e => e.processSubsetEdge()).concat(deletionEdges);
+  }).flat();
+
+  return outboundNodes;
 }

web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts

@@ -21,7 +21,7 @@ import { ContextState, determineContextSlideTransform } from './correction/conte
 import { ContextTransition, TransitionReversionView } from './correction/context-transition.js';
 import { ExecutionTimer } from './correction/execution-timer.js';
 import { ModelCompositor } from './model-compositor.js';
-import { getBestTokenMatches } from './correction/distance-modeler.js';
+import { EDIT_DISTANCE_COST_SCALE, getBestTokenMatches } from './correction/distance-modeler.js';
 
 import CasingForm = LexicalModelTypes.CasingForm;
 import Context = LexicalModelTypes.Context;
@@ -78,7 +78,12 @@ export const CORRECTION_SEARCH_THRESHOLDS = {
    * in log-space, the search would stop at a total cost of 1 + this value if
    * a "full" set of suggestions had already been found.
    */
-  REPLACEMENT_SEARCH_THRESHOLD: 4 as const // e^-4 = 0.0183156388.  Allows "80%" of an extra edit.
+
+  // Ensure at least one "edit distance cost unit" so that even heavily
+  // fat-fingered transpositions have a chance.  Note that the level is this
+  // applied, wordlist weightings have no effect and cannot prevent correction
+  // thresholding!
+  REPLACEMENT_SEARCH_THRESHOLD: EDIT_DISTANCE_COST_SCALE * 1.1
 }
 
 /**

web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/early-correction-search-stopping.tests.ts

@@ -38,7 +38,11 @@ describe('correction-search: shouldStopSearchingEarly', () => {
   });
 
   it('stops checking corrections earlier when enough predictions have been found', () => {
-    const predictionProbs = [.010, .009, .008, .008, .0075, .0075, .007, .007, .006, .006, .005, .005];
+    // Thresholding is performed in log-space.
+    const baseCost = 1;
+    const expectedThreshold = CORRECTION_SEARCH_THRESHOLDS.REPLACEMENT_SEARCH_THRESHOLD;
+
+    const predictionProbs = [.010, .009, .008, .008, .0075, .007, .006, .005, .004, .003, .002, Math.exp(- baseCost - expectedThreshold)];
     assert.isAtLeast(predictionProbs.length, ModelCompositor.MAX_SUGGESTIONS, "test setup no longer valid");
 
     // The only part for each entry we actually care about here:  .totalProb.
@@ -49,12 +53,8 @@ describe('correction-search: shouldStopSearchingEarly', () => {
       } as CorrectionPredictionTupleCore
     });
 
-    const baseCost = 1;
-
-    // Thresholding is performed in log-space.
-    const expectedThreshold = CORRECTION_SEARCH_THRESHOLDS.REPLACEMENT_SEARCH_THRESHOLD;
-
+    // The actual assertions.
     assert.isFalse(shouldStopSearchingEarly(baseCost, baseCost + expectedThreshold - 0.01, predictions));
-    assert.isTrue(shouldStopSearchingEarly( baseCost, baseCost + expectedThreshold + 0.01, predictions));
+    assert.isTrue(shouldStopSearchingEarly(baseCost, baseCost + expectedThreshold + 0.01, predictions));
   });
 });

web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/legacy-quotient-spur.tests.ts

@@ -9,17 +9,23 @@
 
 import { assert } from 'chai';
 
+import { LexicalModelTypes } from '@keymanapp/common-types';
 import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs';
 import {
+  buildEdgesFromResults,
   generateSubsetId,
   LegacyQuotientRoot,
   LegacyQuotientSpur,
-  models
+  models,
+  processTransposeRoots,
+  TokenResultMapping
 } from '@keymanapp/lm-worker/test-index';
 
 import { buildCantLinearFixture } from '../../helpers/buildCantLinearFixture.js';
 import { analyzeQuotientNodeResults } from '../../helpers/analyzeQuotientNodeResults.js';
 
+import Distribution = LexicalModelTypes.Distribution;
+import Transform = LexicalModelTypes.Transform;
 import TrieModel = models.TrieModel;
 
 const testModel = new TrieModel(jsonFixture('models/tries/english-1000'));
@@ -322,4 +328,108 @@ describe('LegacyQuotientSpur', () => {
       assert.isEmpty(analysis.foundWithDuplicates);
     });
   });
+
+  describe('transposition handling', () => {
+    const tehDistributions: Distribution<Transform>[] = [
+      [
+        { sample: { insert: 't', deleteLeft: 0, id: 1 }, p: .55},
+        { sample: { insert: 'r', deleteLeft: 0, id: 1 }, p: .45}
+      ], [
+        { sample: { insert: 'e', deleteLeft: 0, id: 1 }, p: .9},
+        { sample: { insert: 's', deleteLeft: 0, id: 1 }, p: .1}
+      ], [
+        { sample: { insert: 'h', deleteLeft: 0, id: 1 }, p: .9},
+        { sample: { insert: 'n', deleteLeft: 0, id: 1 }, p: .1}
+      ]
+    ];
+
+    it('corrects to `the` for a targeted, deep search for a `teh` transposition', () => {
+      const root = new LegacyQuotientRoot(testModel);
+
+      const rootResults: TokenResultMapping[] = [];
+      while(root.currentCost < Number.POSITIVE_INFINITY) {
+        const result = root.handleNextNode();
+        if(result.type == 'complete') {
+          rootResults.push(result.mapping);
+        }
+      }
+
+      const entry_empty = rootResults.find((entry) => entry.matchString == '')
+      assert.isOk(entry_empty);
+
+      const firstSpur = new LegacyQuotientSpur(root, tehDistributions[0], tehDistributions[0][0]);
+      const edgesFromEmpty = buildEdgesFromResults([entry_empty], firstSpur.inputs, firstSpur.spaceId);
+      const edge_t = edgesFromEmpty.find((entry) => entry.resultKey == 't');
+      assert.isOk(edge_t);
+
+      // now, try to do something with entry_t.
+      const secondSpur = new LegacyQuotientSpur(firstSpur,  tehDistributions[1], tehDistributions[1][0]);
+      const thirdSpur  = new LegacyQuotientSpur(secondSpur, tehDistributions[2], tehDistributions[2][0]);
+
+      const entry_t = new TokenResultMapping(firstSpur, edge_t);
+      const transposeFirstHalves = processTransposeRoots([entry_t], thirdSpur.inputs, thirdSpur.spaceId);
+
+      const edge_th = transposeFirstHalves.find((entry) => entry.resultKey == 'th' && entry.editCount == 1);
+      assert.isOk(edge_th);
+
+      const entry_th = new TokenResultMapping(thirdSpur, edge_th);
+      const transposeSecondHalves = buildEdgesFromResults([entry_th], secondSpur.inputs, thirdSpur.spaceId);
+      const edge_the = transposeSecondHalves.find((entry) => entry.resultKey == 'the' && entry.editCount == 1);
+      assert.isOk(edge_the);
+    });
+
+    it('corrects to `the` for an sequential, broad search for a `teh` transposition', () => {
+      const root = new LegacyQuotientRoot(testModel);
+
+      const rootResults: TokenResultMapping[] = [];
+      while(root.currentCost < Number.POSITIVE_INFINITY) {
+        const result = root.handleNextNode();
+        if(result.type == 'complete') {
+          rootResults.push(result.mapping);
+        }
+      }
+
+      const entry_empty = rootResults.find((entry) => entry.matchString == '')
+      assert.isOk(entry_empty);
+
+      const firstSpur = new LegacyQuotientSpur(root, tehDistributions[0], tehDistributions[0][0]);
+      const firstResults: TokenResultMapping[] = [];
+      while(firstSpur.currentCost < Number.POSITIVE_INFINITY) {
+        const result = firstSpur.handleNextNode();
+        if(result.type == 'complete') {
+          firstResults.push(result.mapping);
+        }
+      }
+
+      const entry_t = firstResults.find((entry) => entry.matchString == 't' && entry.editCount == 0);
+      assert.isOk(entry_t);
+
+      // now, try to do something with entry_t.
+      const secondSpur = new LegacyQuotientSpur(firstSpur,  tehDistributions[1], tehDistributions[1][0]);
+      const secondResults: TokenResultMapping[] = [];
+      while(secondSpur.currentCost < Number.POSITIVE_INFINITY) {
+        const result = secondSpur.handleNextNode();
+        if(result.type == 'complete') {
+          secondResults.push(result.mapping);
+        }
+      }
+
+      const thirdSpur  = new LegacyQuotientSpur(secondSpur, tehDistributions[2], tehDistributions[2][0]);
+      const thirdResults: TokenResultMapping[] = [];
+      while(thirdSpur.currentCost < Number.POSITIVE_INFINITY) {
+        const result = thirdSpur.handleNextNode();
+        if(result.type == 'complete') {
+          thirdResults.push(result.mapping);
+        }
+      }
+
+      const entry_the = thirdResults.find((entry) => entry.matchString == 'the' && entry.editCount == 1);
+      assert.isOk(entry_the);
+
+      thirdResults.sort((a, b) => a.totalCost - b.totalCost);
+      const the_index = thirdResults.findIndex((entry) => entry.matchString == 'the' && entry.editCount == 1);
+      // `teh` should appear fairly early as a viable correction .
+      assert.isBelow(the_index, 10);
+    });
+  });
 });