Merge branch 'feat/web/compute-merge-split-effects' into feat/web/tokenized-transform-assembly

web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts

@@ -257,7 +257,7 @@ export class ContextState {
     const tokens = resultTokenization.tokens;
     const lastIndex = tokens.length - 1;
     // Ignore a context-final empty '' token; the interesting one is what comes before.
-    const nonEmptyTail = tokens[lastIndex].sourceText != '' ? tokens[lastIndex] : tokens[lastIndex - 1];
+    const nonEmptyTail = !tokens[lastIndex].isEmptyToken ? tokens[lastIndex] : tokens[lastIndex - 1];
     const appliedSuggestionTransitionId = nonEmptyTail?.appliedTransitionId;
 
     // Used to construct and represent the part of the incoming transform that

web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts

@@ -16,6 +16,14 @@ import Distribution = LexicalModelTypes.Distribution;
 import LexicalModel = LexicalModelTypes.LexicalModel;
 import Transform = LexicalModelTypes.Transform;
 
+/**
+ * Notes critical properties of the inputs comprising each ContextToken.
+ */
+export interface TokenInputSource {
+  trueTransform: Transform;
+  inputStartIndex: number;
+}
+
 /**
  * Breaks apart a raw text string into individual, single-codepoint
  * transforms, all set with the specified transform ID.
@@ -65,7 +73,7 @@ export class ContextToken {
    * applied to the actual context for the set of keystrokes contributing to
    * this token.
    */
-  private _inputRange: Transform[];
+  private _inputRange: TokenInputSource[];
 
   /**
    * Constructs a new, empty instance for use with the specified LexicalModel.
@@ -117,7 +125,10 @@ export class ContextToken {
         return [{sample: transform, p: 1.0}];
       });
       rawTransformDistributions.forEach((entry) => {
-        this._inputRange.push(entry[0].sample);
+        this._inputRange.push({
+          trueTransform: entry[0].sample,
+          inputStartIndex: 0
+        });
         this.searchSpace.addInput(entry);
       });
     }
@@ -127,26 +138,52 @@ export class ContextToken {
    * Call this to record the original keystroke Transforms for the context range
    * corresponding to this token.
    */
-  addSourceInput(transform: Transform) {
-    this._inputRange.push(transform);
+  addInput(inputSource: TokenInputSource, distribution: Distribution<Transform>) {
+    this._inputRange.push(inputSource);
+    this.searchSpace.addInput(distribution);
   }
 
   /**
    * Denotes the original keystroke Transforms comprising the range corresponding
    * to this token.
    */
-  get inputRange(): Readonly<Transform[]> {
+  get inputRange(): Readonly<TokenInputSource[]> {
     return this._inputRange;
   }
 
   /**
-   * Gets a simple, human-readable representation of `inputRange`.
+   * Indicates whether or not this ContextToken likely represents an empty token.
+   */
+  get isEmptyToken(): boolean {
+    return this.exampleInput == '';
+  }
+
+  /**
+   * Gets a compact string-based representation of `inputRange` that
+   * maps compatible token source ranges to each other.
+   */
+  get sourceRangeKey(): string {
+    const components: string[] = [];
+
+    for(const source of this.inputRange) {
+      const i = source.inputStartIndex;
+      components.push(`T${source.trueTransform.id}${i != 0 ? '@' + i : ''}`);
+    }
+
+    return components.join('+');
+  }
+
+  /**
+   * Gets a simple, compact string-based representation of `inputRange`.
    *
-   * Should not actually be used in code - its use is intended only for
-   * debugging.
+   * This should only ever be used for debugging purposes.
    */
   get sourceText(): string {
-    const composite = this._inputRange.reduce((accum, current) => buildMergedTransform(accum, current), { insert: '', deleteLeft: 0 });
+    const composite = this._inputRange.reduce((accum, current) => {
+      const alteredTransform = {...current.trueTransform};
+      alteredTransform.insert = alteredTransform.insert.slice(current.inputStartIndex);
+      return buildMergedTransform(accum, current.trueTransform)
+    }, { insert: '', deleteLeft: 0 });
     const prefix = '\u{2421}'.repeat(composite.deleteLeft);
     return prefix + composite.insert;
   }
@@ -166,7 +203,7 @@ export class ContextToken {
      * most likely keystroke data afterward.
      */
     const transforms = this.searchSpace.inputSequence.map((dist) => dist[0].sample)
-    const composite = transforms.reduce((accum, current) => buildMergedTransform(accum, current), { insert: '', deleteLeft: 0});
+    const composite = transforms.reduce((accum, current) => buildMergedTransform(accum, current), {insert: '', deleteLeft: 0});
     return composite.insert;
   }
 }

web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts

@@ -40,7 +40,7 @@ interface TokenMergeMap {
 
 interface TokenSplitMap {
   input: EditTokenMap,
-  matches: EditTokenMap[]
+  matches: (EditTokenMap & { textOffset: number })[]
 };
 
 /**
@@ -75,11 +75,11 @@ export class ContextTokenization {
   /**
    * Returns plain-text strings representing the most probable representation for all
    * tokens represented by this tokenization instance.
+   *
+   * Intended for debugging use only.
    */
   get sourceText() {
-    return this.tokens
-      .filter(token => token.sourceText !== null)
-      .map(token => token.sourceText);
+    return this.tokens.map(token => token.sourceText);
   }
 
   /**
@@ -87,10 +87,7 @@ export class ContextTokenization {
    * tokens represented by this tokenization instance.
    */
   get exampleInput(): string[] {
-    return this.tokens
-      // Hide any tokens representing invisible wordbreaks.  (Thinking ahead to phrase-level possibilities)
-      .filter(token => token.exampleInput !== null)
-      .map(token => token.exampleInput);
+    return this.tokens.map(token => token.exampleInput);
   }
 
   /**
@@ -104,7 +101,7 @@ export class ContextTokenization {
    * the tokenization modeled by this instance.
    */
   computeAlignment(incomingTokenization: string[], isSliding: boolean, noSubVerify?: boolean): ContextStateAlignment {
-    return computeAlignment(this.sourceText, incomingTokenization, isSliding, noSubVerify);
+    return computeAlignment(this.exampleInput, incomingTokenization, isSliding, noSubVerify);
   }
 
   /**
@@ -368,6 +365,7 @@ export class ContextTokenization {
     // edited, those edits occur to the left as well - and further left of whatever
     // the new tail token is *if* tokens were removed.
     const firstTailEditIndex = Math.min((1 - tailEditLength), 0) + Math.min(tailTokenShift, 0);
+    let primaryInputAppliedLen = 0;
     for(let i = 0; i < tailEditLength; i++) {
       const tailIndex = firstTailEditIndex + i;
 
@@ -393,11 +391,12 @@ export class ContextTokenization {
         // Erase any applied-suggestion transition ID; it is no longer valid.
         token.appliedTransitionId = undefined;
         const emptySample: ProbabilityMass<Transform> = { sample: { insert: '', deleteLeft: 0 }, p: 1 };
-        token.addSourceInput(primaryInput ?? emptySample.sample);
-        token.searchSpace.addInput(tokenDistribution.map((seq) => seq.get(tailIndex) ?? emptySample));
+        const dist = tokenDistribution.map((seq) => seq.get(tailIndex) ?? emptySample);
+        token.addInput({trueTransform: primaryInput ?? emptySample.sample, inputStartIndex: primaryInputAppliedLen}, dist);
       }
 
       tokenization[incomingIndex] = token;
+      primaryInputAppliedLen += KMWString.length(primaryInput?.insert ?? '');
     }
 
     if(tailTokenShift < 0) {
@@ -454,11 +453,10 @@ export class ContextTokenization {
           // If there are no entries in our would-be distribution, there's no
           // reason to pass in what amounts to a no-op.
           if(transformDistribution) {
-            pushedToken.addSourceInput(primaryInput);
             // If we ever stop filtering tokenized transform distributions, it may
             // be worth adding an empty transform here with weight to balance
             // the distribution back to a cumulative prob sum of 1.
-            pushedToken.searchSpace.addInput(transformDistribution);
+            pushedToken.addInput({ trueTransform: primaryInput, inputStartIndex: primaryInputAppliedLen }, transformDistribution);
           }
         } else if(incomingToken.text) {
           // We have no transform data to match against an inserted token with text; abort!
@@ -470,6 +468,7 @@ export class ContextTokenization {
 
         // Auto-replaces the search space to correspond with the new token.
         tokenization.push(pushedToken);
+        primaryInputAppliedLen += KMWString.length(primaryInput.insert);
       }
     }
 
@@ -848,12 +847,12 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo
           text: preTokenization[input]
         }]
       };
-      let currentMerge: string;
+      let currentMerge = preTokenization[input];
       let inputLookahead = 1;
       // Look-ahead 1
-      let nextMerge = preTokenization[input] + preTokenization[input + inputLookahead++];
+      let nextMerge = currentMerge + preTokenization[input + inputLookahead++];
       // Conditional validates if look-ahead 1 passes (which it should)
-      for(/* next line */; mergeTarget.indexOf(nextMerge) == 0; nextMerge = preTokenization[input + inputLookahead++]) {
+      for(/* next line */; mergeTarget.indexOf(nextMerge) == 0; nextMerge = currentMerge + preTokenization[input + inputLookahead++]) {
         merge.inputs.push({
           index: input + inputLookahead - 1,
           text: preTokenization[input + inputLookahead-1]
@@ -876,18 +875,21 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo
         },
         matches: [ {
           index: match,
-          text: resultTokenization[match]
+          text: resultTokenization[match],
+          textOffset: 0
         }],
       };
-      let currentMerge: string;
+      let currentMerge = resultTokenization[match];
       matchOffset = 1;
       // Look-ahead 1
-      let nextMerge = resultTokenization[match] + resultTokenization[match + matchOffset++];
-      for(/* next line */; splitTarget.indexOf(nextMerge) == 0; nextMerge = preTokenization[match + matchOffset++]) {
+      let nextMerge = currentMerge + resultTokenization[match + matchOffset++];
+      for(/* next line */; splitTarget.indexOf(nextMerge) == 0; nextMerge = currentMerge + preTokenization[match + matchOffset++]) {
+        const textOffset = KMWString.length(currentMerge);
         currentMerge = nextMerge;
         split.matches.push({
           index: match + matchOffset - 1,
-          text: resultTokenization[match + matchOffset-1]
+          text: resultTokenization[match + matchOffset-1],
+          textOffset
         });
         // Each time we 'pass' the condition, we've successfully processed an associated edit.
 

web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts

@@ -394,7 +394,7 @@ export function determineSuggestionAlignment(
 
   // Did the wordbreaker (or similar) append a blank token before the caret?  If so,
   // preserve that by preventing corrections from triggering left-deletion.
-  if(transition.final.tokenization.tail.sourceText == '') {
+  if(transition.final.tokenization.tail.isEmptyToken) {
     deleteLeft = 0;
   }
 

web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts

@@ -1062,7 +1062,7 @@ describe('ContextTokenization', function() {
         merges: [],
         splits: [ {
           input: { text: 'can\'', index: 7 },
-          matches: [ { text: 'can', index: 7 }, { text: '\'', index: 8 }]
+          matches: [ { text: 'can', index: 7, textOffset: 0 }, { text: '\'', index: 8, textOffset: 3 }]
         } ],
         mergeOffset: 0,
         splitOffset: -1,