refactor(web): manage true-input data from SearchQuotientSpur

Once we start considering alternate tokenization schemes, we'll want to note how each potential input path aligns with the original input keystrokes.  As there can be multiple paths to land within the same token, it's best to store this data on the SearchQuotientSpur objects instead.  This becomes especially relevant when considering token splits and merges, which will be the next follow-ups.

Build-bot: skip build:web
Test-bot: skip

web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts

@@ -11,7 +11,7 @@ import { applyTransform, buildMergedTransform } from "@keymanapp/models-template
 import { LexicalModelTypes } from '@keymanapp/common-types';
 import { deepCopy, KMWString } from "@keymanapp/web-utils";
 
-import { SearchQuotientNode } from "./search-quotient-node.js";
+import { SearchQuotientNode, TokenInputSource } from "./search-quotient-node.js";
 import { TokenSplitMap } from "./context-tokenization.js";
 import { LegacyQuotientSpur } from "./legacy-quotient-spur.js";
 import { LegacyQuotientRoot } from "./legacy-quotient-root.js";
@@ -20,15 +20,6 @@ import Distribution = LexicalModelTypes.Distribution;
 import LexicalModel = LexicalModelTypes.LexicalModel;
 import Transform = LexicalModelTypes.Transform;
 
-/**
- * Notes critical properties of the inputs comprising each ContextToken.
- */
-export interface TokenInputSource {
-  trueTransform: Transform;
-  inputStartIndex: number;
-  bestProbFromSet: number;
-}
-
 /**
  * Breaks apart a raw text string into individual, single-codepoint
  * transforms, all set with the specified transform ID.
@@ -76,13 +67,6 @@ export class ContextToken {
    */
   appliedTransitionId?: number;
 
-  /**
-   * Represents the original, 'true' input transforms (tokenized, as necessary)
-   * applied to the actual context for the set of keystrokes contributing to
-   * this token.
-   */
-  private _inputRange: TokenInputSource[];
-
   /**
    * Constructs a new, empty instance for use with the specified LexicalModel.
    * @param model
@@ -109,14 +93,12 @@ export class ContextToken {
       // In case we are unable to perfectly track context (say, due to multitaps)
       // we need to ensure that only fully-utilized keystrokes are considered.
       this._searchModule = priorToken.searchModule;
-      this._inputRange = priorToken._inputRange.slice();
     } else {
       const model = param;
 
       // May be altered outside of the constructor.
       this.isWhitespace = false;
       this.isPartial = !!isPartial;
-      this._inputRange = [];
 
       rawText ||= '';
 
@@ -125,12 +107,12 @@ export class ContextToken {
       let searchModule: SearchQuotientNode = new LegacyQuotientRoot(model);
       const BASE_PROBABILITY = 1;
       textToCharTransforms(rawText).forEach((transform) => {
-        this._inputRange.push({
+        let inputMetadata: TokenInputSource = {
           trueTransform: transform,
           inputStartIndex: 0,
           bestProbFromSet: BASE_PROBABILITY
-        });
-        searchModule = new LegacyQuotientSpur(searchModule, [{sample: transform, p: BASE_PROBABILITY}], 1);
+        };
+        searchModule = new LegacyQuotientSpur(searchModule, [{sample: transform, p: BASE_PROBABILITY}], inputMetadata);
       });
 
       this._searchModule = searchModule;
@@ -142,16 +124,11 @@ export class ContextToken {
    * corresponding to this token.
    */
   addInput(inputSource: TokenInputSource, distribution: Distribution<Transform>) {
-    this._inputRange.push(inputSource);
-    this._searchModule = new LegacyQuotientSpur(this._searchModule, distribution, inputSource.bestProbFromSet);
+    this._searchModule = new LegacyQuotientSpur(this._searchModule, distribution, inputSource);
   }
 
-  /**
-   * Denotes the original keystroke Transforms comprising the range corresponding
-   * to this token.
-   */
-  get inputRange(): Readonly<TokenInputSource[]> {
-    return this._inputRange;
+  get inputCount() {
+    return this._searchModule.inputCount;
   }
 
   /**
@@ -161,6 +138,14 @@ export class ContextToken {
     return this.exampleInput == '';
   }
 
+  /**
+   * Denotes the original keystroke Transforms comprising the range corresponding
+   * to this token.
+   */
+  get inputRange() {
+    return this.searchModule.sourceIdentifiers;
+  }
+
   /**
    * Gets the unique identifier that may be used to match this ContextToken with
    * a correction-search result.
@@ -175,8 +160,9 @@ export class ContextToken {
    */
   get sourceRangeKey(): string {
     const components: string[] = [];
+    const sources = this.searchModule.sourceIdentifiers;
 
-    for(const source of this.inputRange) {
+    for(const source of sources) {
       const i = source.inputStartIndex;
       components.push(`T${source.trueTransform.id}${i != 0 ? '@' + i : ''}`);
     }
@@ -207,10 +193,10 @@ export class ContextToken {
     let lastSourceInput: TokenInputSource;
     let lastInputDistrib: Distribution<Transform>;
     for(const token of tokensToMerge) {
-      const inputCount = token.inputRange.length;
+      const inputCount = token.inputCount;
       let startIndex = 0;
 
-      if(token.inputRange.length == 0) {
+      if(inputCount == 0) {
         continue;
       }
 

web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts

@@ -582,7 +582,7 @@ export class ContextTokenization {
       // If we are completely replacing a token via delete left, erase the deleteLeft;
       // that part applied to a _previous_ token that no longer exists.
       // We start at index 0 in the insert string for the "new" token.
-      if(affectedToken.inputRange.length == 0 && distribution[0].sample.deleteLeft != 0) {
+      if(affectedToken.inputCount == 0 && distribution[0].sample.deleteLeft != 0) {
         distribution = distribution.map((mass) => ({sample: { ...mass.sample, deleteLeft: 0 }, p: mass.p }));
       }
       affectedToken.addInput({trueTransform: sourceInput, inputStartIndex: appliedLength, bestProbFromSet}, distribution);

web/src/engine/predictive-text/worker-thread/src/main/correction/legacy-quotient-spur.ts

@@ -11,7 +11,7 @@
 import { LexicalModelTypes } from '@keymanapp/common-types';
 
 import { SearchNode } from './distance-modeler.js';
-import { PathResult, SearchQuotientNode } from './search-quotient-node.js';
+import { PathResult, SearchQuotientNode, TokenInputSource } from './search-quotient-node.js';
 import { SearchQuotientSpur } from './search-quotient-spur.js';
 
 import Distribution = LexicalModelTypes.Distribution;
@@ -27,8 +27,8 @@ export class LegacyQuotientSpur extends SearchQuotientSpur {
    * @param inputs
    * @param bestProbFromSet
    */
-  constructor(space: SearchQuotientNode, inputs: Distribution<Transform>, bestProbFromSet: number) {
-    super(space, inputs, space.lowestPossibleSingleCost - Math.log(bestProbFromSet));
+  constructor(space: SearchQuotientNode, inputs: Distribution<Transform>, inputSource: TokenInputSource) {
+    super(space, inputs, inputSource);
     this.queueNodes(this.buildEdgesForNodes(space.previousResults.map(r => r.node)));
     return;
   }

web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-node.ts

@@ -39,6 +39,15 @@ type CompleteSearchPath = {
 
 export type PathResult = NullPath | IntermediateSearchPath | CompleteSearchPath;
 
+/**
+ * Notes critical properties of the inputs comprising each ContextToken.
+ */
+export interface TokenInputSource {
+  trueTransform: Transform;
+  inputStartIndex: number;
+  bestProbFromSet: number;
+}
+
 /**
  * Represents all or a portion of the dynamically-generated graph used to search
  * for predictive-text corrections.
@@ -121,6 +130,12 @@ export interface SearchQuotientNode {
    * the correction-search graph and its paths.
    */
   readonly bestExample: { text: string, p: number };
+
+  /**
+   * Gets components useful for building a string-based representation of the
+   * keystroke range corrected by this search space.
+   */
+  readonly sourceIdentifiers: TokenInputSource[];
 }
 
 /**

web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-root.ts

@@ -2,7 +2,7 @@
 import { LexicalModelTypes } from '@keymanapp/common-types';
 
 import { SearchNode, SearchResult } from './distance-modeler.js';
-import { generateSpaceSeed, PathResult, SearchQuotientNode } from './search-quotient-node.js';
+import { generateSpaceSeed, PathResult, SearchQuotientNode, TokenInputSource } from './search-quotient-node.js';
 
 import LexicalModel = LexicalModelTypes.LexicalModel;
 
@@ -89,4 +89,8 @@ export class SearchQuotientRoot implements SearchQuotientNode {
       return [this.rootResult];
     }
   }
+
+  get sourceIdentifiers(): TokenInputSource[] {
+    return [];
+  }
 }

web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts

@@ -12,7 +12,7 @@ import { QueueComparator as Comparator, KMWString, PriorityQueue } from '@keyman
 import { LexicalModelTypes } from '@keymanapp/common-types';
 
 import { EDIT_DISTANCE_COST_SCALE, SearchNode, SearchResult } from './distance-modeler.js';
-import { generateSpaceSeed, PathResult, SearchQuotientNode } from './search-quotient-node.js';
+import { generateSpaceSeed, PathResult, SearchQuotientNode, TokenInputSource } from './search-quotient-node.js';
 
 import Distribution = LexicalModelTypes.Distribution;
 import Transform = LexicalModelTypes.Transform;
@@ -25,7 +25,8 @@ export const QUEUE_NODE_COMPARATOR: Comparator<SearchNode> = function(arg1, arg2
 // Whenever a wordbreak boundary is crossed, a new instance should be made.
 export abstract class SearchQuotientSpur implements SearchQuotientNode {
   private selectionQueue: PriorityQueue<SearchNode> = new PriorityQueue(QUEUE_NODE_COMPARATOR);
-  readonly inputs?: Distribution<Readonly<Transform>>;
+  readonly inputs?: Distribution<Transform>;
+  readonly inputSource?: TokenInputSource;
 
   private parentNode: SearchQuotientNode;
   readonly spaceId: number;
@@ -50,13 +51,14 @@ export abstract class SearchQuotientSpur implements SearchQuotientNode {
    *
    * @param parentNode
    * @param inputs
-   * @param costHeuristic
+   * @param inputSource
    */
-  constructor(parentNode: SearchQuotientNode, inputs: Distribution<Transform>, costHeuristic: number) {
+  constructor(parentNode: SearchQuotientNode, inputs: Distribution<Readonly<Transform>>, inputSource: TokenInputSource) {
     this.spaceId = generateSpaceSeed();
 
     this.parentNode = parentNode;
-    this.lowestPossibleSingleCost = (parentNode?.lowestPossibleSingleCost ?? 0) - Math.log(costHeuristic);
+    this.inputSource = inputSource;
+    this.lowestPossibleSingleCost = (parentNode?.lowestPossibleSingleCost ?? 0) - Math.log(inputSource?.bestProbFromSet ?? 1);
     this.inputs = inputs?.length > 0 ? inputs : null;
     this.inputCount = (parentNode?.inputCount ?? 0) + (this.inputs ? 1 : 0);
   }
@@ -221,4 +223,22 @@ export abstract class SearchQuotientSpur implements SearchQuotientNode {
   public get previousResults(): SearchResult[] {
     return Object.values(this.returnedValues ?? {}).map(v => new SearchResult(v));
   }
+
+  public get sourceIdentifiers(): TokenInputSource[] {
+    if(!this.parentNode) {
+      return [];
+    }
+
+    const parentSources = this.parentNode.sourceIdentifiers;
+    if(this.inputSource) {
+      const inputId = this.inputSource.trueTransform.id;
+      if(inputId && parentSources.length > 0 && parentSources[parentSources.length - 1].trueTransform.id == inputId) {
+        return parentSources;
+      }
+
+      parentSources.push(this.inputSource);
+    }
+
+    return parentSources;
+  }
 }

web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/getBestMatches.tests.ts

@@ -146,9 +146,9 @@ describe('getBestMatches', () => {
       {sample: {insert: 'n', deleteLeft: 0}, p: 0.25}
     ];
 
-    const searchPath1 = new LegacyQuotientSpur(searchPath, synthInput1, 1);
-    const searchPath2 = new LegacyQuotientSpur(searchPath1, synthInput2, .75);
-    const searchPath3 = new LegacyQuotientSpur(searchPath2, synthInput3, .75);
+    const searchPath1 = new LegacyQuotientSpur(searchPath, synthInput1, {trueTransform: synthInput1[0].sample, inputStartIndex: 0, bestProbFromSet: 1});
+    const searchPath2 = new LegacyQuotientSpur(searchPath1, synthInput2, {trueTransform: synthInput2[0].sample, inputStartIndex: 0, bestProbFromSet: .75});
+    const searchPath3 = new LegacyQuotientSpur(searchPath2, synthInput3, {trueTransform: synthInput3[0].sample, inputStartIndex: 0, bestProbFromSet: .75});
 
     assert.notEqual(searchPath1.spaceId, searchPath.spaceId);
     assert.notEqual(searchPath2.spaceId, searchPath1.spaceId);
@@ -180,9 +180,9 @@ describe('getBestMatches', () => {
       {sample: {insert: 'n', deleteLeft: 0}, p: 0.25}
     ];
 
-    const searchPath1 = new LegacyQuotientSpur(searchPath, synthInput1, 1);
-    const searchPath2 = new LegacyQuotientSpur(searchPath1, synthInput2, .75);
-    const searchPath3 = new LegacyQuotientSpur(searchPath2, synthInput3, .75);
+    const searchPath1 = new LegacyQuotientSpur(searchPath, synthInput1, {trueTransform: synthInput1[0].sample, inputStartIndex: 0, bestProbFromSet: 1});
+    const searchPath2 = new LegacyQuotientSpur(searchPath1, synthInput2, {trueTransform: synthInput2[0].sample, inputStartIndex: 0, bestProbFromSet: .75});
+    const searchPath3 = new LegacyQuotientSpur(searchPath2, synthInput3, {trueTransform: synthInput3[0].sample, inputStartIndex: 0, bestProbFromSet: .75});
 
     assert.notEqual(searchPath1.spaceId, searchPath.spaceId);
     assert.notEqual(searchPath2.spaceId, searchPath1.spaceId);

web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-spur.tests.ts

@@ -24,24 +24,24 @@ export function buildSimplePathSplitFixture() {
     { sample: {insert: 'r', deleteLeft: 0, id: 11}, p: 0.4 },
     { sample: {insert: 't', deleteLeft: 0, id: 11}, p: 0.1 }
   ];
-  const path1 = new LegacyQuotientSpur(rootPath, distrib1, distrib1[0].p);
+  const path1 = new LegacyQuotientSpur(rootPath, distrib1, {trueTransform: distrib1[0].sample, inputStartIndex: 0, bestProbFromSet: distrib1[0].p});
 
   const distrib2 = [
     { sample: {insert: 'a', deleteLeft: 0, id: 12}, p: 0.7 },
     { sample: {insert: 'e', deleteLeft: 0, id: 12}, p: 0.3 }
   ];
-  const path2 = new LegacyQuotientSpur(path1, distrib2, distrib2[0].p);
+  const path2 = new LegacyQuotientSpur(path1, distrib2, {trueTransform: distrib2[0].sample, inputStartIndex: 0, bestProbFromSet: distrib2[0].p});
 
   const distrib3 = [
     { sample: {insert: 'n', deleteLeft: 0, id: 13}, p: 0.8 },
     { sample: {insert: 'r', deleteLeft: 0, id: 13}, p: 0.2 }
   ];
-  const path3 = new LegacyQuotientSpur(path2, distrib3, distrib3[0].p);
+  const path3 = new LegacyQuotientSpur(path2, distrib3, {trueTransform: distrib3[0].sample, inputStartIndex: 0, bestProbFromSet: distrib3[0].p});
 
   const distrib4 = [
     { sample: {insert: 't', deleteLeft: 0, id: 14}, p: 1 }
   ];
-  const path4 = new LegacyQuotientSpur(path3, distrib4, distrib4[0].p);
+  const path4 = new LegacyQuotientSpur(path3, distrib4, {trueTransform: distrib4[0].sample, inputStartIndex: 0, bestProbFromSet: distrib4[0].p});
 
   return {
     paths: [rootPath, path1, path2, path3, path4],
@@ -68,7 +68,7 @@ describe('SearchQuotientSpur', () => {
         {sample: {insert: 'o', deleteLeft: 0, id: 13 }, p: 0.2}
       ];
 
-      const extendedPath = new LegacyQuotientSpur(rootPath, leadEdgeDistribution, leadEdgeDistribution[0].p);
+      const extendedPath = new LegacyQuotientSpur(rootPath, leadEdgeDistribution, {trueTransform: leadEdgeDistribution[0].sample, inputStartIndex: 0, bestProbFromSet: leadEdgeDistribution[0].p});
 
       assert.equal(extendedPath.inputCount, 1);
       assert.isNumber(extendedPath.spaceId);
@@ -97,7 +97,7 @@ describe('SearchQuotientSpur', () => {
       const length1Path = new LegacyQuotientSpur(
         rootPath,
         leadEdgeDistribution,
-        leadEdgeDistribution[0].p
+        {trueTransform: leadEdgeDistribution[0].sample, inputStartIndex: 0, bestProbFromSet: leadEdgeDistribution[0].p}
       );
 
       const tailEdgeDistribution = [
@@ -109,7 +109,7 @@ describe('SearchQuotientSpur', () => {
       const length2Path = new LegacyQuotientSpur(
         length1Path,
         tailEdgeDistribution,
-        tailEdgeDistribution[0].p
+        {trueTransform: tailEdgeDistribution[0].sample, inputStartIndex: 0, bestProbFromSet: tailEdgeDistribution[0].p}
       );
 
       // Verify that the prior distribution remains fully unaltered.
@@ -143,7 +143,7 @@ describe('SearchQuotientSpur', () => {
       const length1Path = new LegacyQuotientSpur(
         rootPath,
         leadEdgeDistribution,
-        leadEdgeDistribution[0].p
+        {trueTransform: leadEdgeDistribution[0].sample, inputStartIndex: 0, bestProbFromSet: leadEdgeDistribution[0].p}
       );
 
       const tailEdgeDistribution = [
@@ -155,7 +155,7 @@ describe('SearchQuotientSpur', () => {
       const length2Path = new LegacyQuotientSpur(
         length1Path,
         tailEdgeDistribution,
-        tailEdgeDistribution[0].p
+        {trueTransform: tailEdgeDistribution[0].sample, inputStartIndex: 0, bestProbFromSet: tailEdgeDistribution[0].p}
       );
 
       // Verify that the prior distribution remains fully unaltered.