Merge pull request #16165 from keymanapp/change/web/edit-token-mapping-types

change(web): clarify merge, split edit token-mapping types 🚂

web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts

@@ -10,7 +10,7 @@
 import { LexicalModelTypes } from '@keymanapp/common-types';
 
 import { SearchQuotientNode, PathInputProperties } from "./search-quotient-node.js";
-import { TokenSplitMap } from "./context-tokenization.js";
+import { TokenSplitMapping } from "./context-tokenization.js";
 import { LegacyQuotientSpur } from "./legacy-quotient-spur.js";
 import { LegacyQuotientRoot } from "./legacy-quotient-root.js";
 import { generateSubsetId } from './tokenization-subsets.js';
@@ -199,7 +199,7 @@ export class ContextToken {
    * @param lexicalModel
    * @returns
    */
-  split(split: TokenSplitMap): ContextToken[] {
+  split(split: TokenSplitMapping): ContextToken[] {
     // Split from tail to head - leave as much 'head' intact as possible at each
     // step, rather than needing to reconstruct the tail multiple times.
     const splitSpecs = split.matches.slice();

web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts

@@ -31,7 +31,7 @@ const MIN_CHARS_TO_RECONSIDER_FOR_TOKENIZATION = 8;
  * This type is used to indicate properties of tokens affected by merges and
  * splits during a context transition.
  */
-interface EditTokenMap {
+interface EditTokenMappingHalf {
   /**
    * The index of the affected token.
    */
@@ -42,10 +42,22 @@ interface EditTokenMap {
   text: string
 };
 
+/**
+ * This type is used to indicate properties of tokens affected by splits during
+ * a context transition.
+ */
+interface SplitTokenMappingHalf extends EditTokenMappingHalf {
+  /**
+   * The codepoint index within the original token at which the split-off token
+   * begins.
+   */
+  textOffset: number
+};
+
 /**
  * This type represents mappings for tokens affected by merge edit operations.
  */
-interface TokenMergeMap {
+interface TokenMergeMapping {
   /**
    * Entries here represent source-context tokens that are combined as part of
    * the merge edit operation.
@@ -53,7 +65,7 @@ interface TokenMergeMap {
    * Entries will appear in the same ordering as the codepoints they represent
    * appear in the underlying context.
    */
-  inputs: EditTokenMap[],
+  inputs: EditTokenMappingHalf[],
 
   /**
    * This entry represents the post-transition context token produced from the
@@ -62,14 +74,14 @@ interface TokenMergeMap {
    * Note that it is possible for extra codepoints to exist here that were not
    * represented in the original source-context tokens.
    */
-  match: EditTokenMap
+  match: EditTokenMappingHalf
 };
 
 
 /**
  * This type represents mappings for tokens affected by split edit operations.
  */
-export interface TokenSplitMap {
+export interface TokenSplitMapping {
   /**
    * This entry represents the source-context token that was split as part of
    * the split edit operation.
@@ -77,7 +89,7 @@ export interface TokenSplitMap {
    * Note that it is possible for codepoints to exist here but not be
    * represented in the post-transition context tokens resulting from the split.
    */
-  input: EditTokenMap,
+  input: EditTokenMappingHalf,
 
   /**
    * Entries here represent post-transition tokens that represent pieces of the
@@ -86,7 +98,7 @@ export interface TokenSplitMap {
    * Entries will appear in the same ordering as the codepoints they represent
    * appear in the underlying context.
    */
-  matches: (EditTokenMap & { textOffset: number })[]
+  matches: SplitTokenMappingHalf[]
 };
 
 /**
@@ -99,11 +111,11 @@ export interface TransitionEdgeAlignment {
   /**
    * Denotes any token merge edits needed after applying the Transform.
    */
-  merges: TokenMergeMap[];
+  merges: TokenMergeMapping[];
   /**
    * Denotes any token split edits needed after applying the Transform.
    */
-  splits: TokenSplitMap[];
+  splits: TokenSplitMapping[];
   /**
    * Denotes any further token edits needed that cannot be attributed to
    * 'merge's, 'split's, or edits from the input `Transform`.
@@ -1036,13 +1048,13 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo
    * generality, if two separate groups of tokens are merged, two groups will be
    * defined - one for each token resulting from a merge.
    */
-  merges: TokenMergeMap[],
+  merges: TokenMergeMapping[],
   /**
    * Indicates groupings of directly related splits.  Without loss of
    * generality, if two separate tokens are split, two groups will be defined -
    * one for each source token split.
    */
-  splits: TokenSplitMap[]
+  splits: TokenSplitMapping[]
 } {
   // We've found the root token to which changes may apply.
   // We've found the last post-application token to which transform changes contributed.
@@ -1080,8 +1092,8 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo
   const mappedPath: EditTuple<EditOperation>[] = [];
   let mergeOffset = 0;
   let splitOffset = 0;
-  const merges: TokenMergeMap[] = [];
-  const splits: TokenSplitMap[] = [];
+  const merges: TokenMergeMapping[] = [];
+  const splits: TokenSplitMapping[] = [];
   while(queueIndex < editPath.length) {
     const edit = editPath[queueIndex];
     const { input, match } = edit;
@@ -1091,7 +1103,7 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo
     let matchOffset: number = 0;
     if(op == 'merge') {
       const mergeTarget = resultTokenization[match];
-      const merge: TokenMergeMap = {
+      const merge: TokenMergeMapping = {
         match: {
           index: match,
           text: mergeTarget
@@ -1122,7 +1134,7 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo
       merges.push(merge);
     } else if(op == 'split') {
       const splitTarget = preTokenization[input];
-      const split: TokenSplitMap = {
+      const split: TokenSplitMapping = {
         input: {
           index: input,
           text: splitTarget