Merge pull request #13785 from keymanapp/change/web/drop-usvstring-alias

change(web): replace USVString with string in predictive-text engine

common/web/types/src/lexical-model-types.ts

@@ -5,18 +5,6 @@
 
 /****************************** Lexical Models ******************************/
 
-/**
- * A JavaScript string with the restriction that it must only
- * contain Unicode scalar values.
- *
- * This means that any lone high surrogate must be paired with
- * a low surrogate, if it exists. Lone surrogate code units are
- * forbidden.
- *
- * See also: https://developer.mozilla.org/en-US/docs/Web/API/USVString
- */
-export type USVString = string;
-
 export type CasingForm = 'lower' | 'initial' | 'upper';
 
 /**
@@ -67,7 +55,7 @@ export interface LexiconTraversal {
    * - If `char` = 'e', the child represents a prefix of 'the'.
    * - Then `traversal` allows traversing the part of the lexicon prefixed by 'the'.
    */
-  children(): Generator<{char: USVString, traversal: () => LexiconTraversal}>;
+  children(): Generator<{char: string, traversal: () => LexiconTraversal}>;
 
   /**
    * Allows direct access to the traversal state that results when appending one
@@ -81,7 +69,7 @@ export interface LexiconTraversal {
    * That is, if a model "keys" `è` to `e`, there will be no `è` child.
    * @param char
    */
-  child(char: USVString): LexiconTraversal | undefined;
+  child(char: string): LexiconTraversal | undefined;
 
   /**
    * Any entries directly keyed by the currently-represented lookup prefix.  Entries and
@@ -178,7 +166,7 @@ export interface LexicalModel {
    * @param text The original input text.
    * @returns The 'keyed' form of that text.
    */
-  toKey?(text: USVString): USVString;
+  toKey?(text: string): string;
 
   /**
    * Generates predictive suggestions corresponding to the state of context after the proposed
@@ -248,7 +236,7 @@ export interface LexicalModel {
    * @param context
    * @deprecated
    */
-  wordbreak?(context: Context): USVString;
+  wordbreak?(context: Context): string;
 
   /**
    * Lexical models _may_ provide a LexiconTraversal object usable to enhance
@@ -279,7 +267,7 @@ export interface Transform {
    *
    * Corresponds to `s` in com.keyman.KeyboardInterface.output.
    */
-  insert: USVString;
+  insert: string;
 
   /**
    * The number of code units to delete to the left of the cursor.
@@ -380,7 +368,7 @@ export interface Context {
    * buffer. If there is nothing to the left of the buffer, this is
    * an empty string.
    */
-  readonly left: USVString;
+  readonly left: string;
 
   /**
    * Up to maxRightContextCodeUnits code units of Unicode scalar value
@@ -390,7 +378,7 @@ export interface Context {
    *
    * This property may be missing entirely.
    */
-  readonly right?: USVString;
+  readonly right?: string;
 
   /**
    * Whether the insertion point is at the start of the buffer.

common/web/types/tests/lexical-model-types.tests.ts

@@ -7,7 +7,7 @@
 
 import { KMXPlus, LdmlKeyboardTypes, LexicalModelTypes } from "@keymanapp/common-types";
 
-export let u: LexicalModelTypes.USVString;
+export let u: string;
 export let l: LexicalModelTypes.Transform;
 export let s: LexicalModelTypes.Suggestion;
 export let st: LexicalModelTypes.SuggestionTag;

web/src/engine/predictive-text/templates/src/tokenization.ts

@@ -214,6 +214,6 @@ export function getLastPreCaretToken(wordBreaker: LexicalModelTypes.WordBreaking
 // While it is currently identical to getLastWord, this may change in the future.
 // It's best not to write ourselves into a corner on this one, as disambiguating later
 // would likely be pretty painful.
-export function wordbreak(wordBreaker: LexicalModelTypes.WordBreakingFunction, context: LexicalModelTypes.Context): LexicalModelTypes.USVString {
+export function wordbreak(wordBreaker: LexicalModelTypes.WordBreakingFunction, context: LexicalModelTypes.Context): string {
   return getLastPreCaretToken(wordBreaker, context);
 }

web/src/engine/predictive-text/templates/src/trie-model.ts

@@ -42,7 +42,6 @@ import LexiconTraversal = LexicalModelTypes.LexiconTraversal;
 import Suggestion = LexicalModelTypes.Suggestion;
 import TextWithProbability = LexicalModelTypes.TextWithProbability;
 import Transform = LexicalModelTypes.Transform;
-import USVString = LexicalModelTypes.USVString;
 import WithOutcome = LexicalModelTypes.WithOutcome;
 import WordBreakingFunction = LexicalModelTypes.WordBreakingFunction;
 
@@ -109,7 +108,7 @@ class Traversal implements LexiconTraversal {
     this.totalWeight = totalWeight;
   }
 
-  child(char: USVString): LexiconTraversal | undefined {
+  child(char: string): LexiconTraversal | undefined {
     // May result for blank tokens resulting immediately after whitespace.
     if(char == '') {
       return this;
@@ -128,7 +127,7 @@ class Traversal implements LexiconTraversal {
   }
 
   // Handles one code unit at a time.
-  private _child(char: USVString): Traversal | undefined {
+  private _child(char: string): Traversal | undefined {
     const root = this.root;
     const totalWeight = this.totalWeight;
     const nextPrefix = this.prefix + char;
@@ -154,7 +153,7 @@ class Traversal implements LexiconTraversal {
     }
   }
 
-  *children(): Generator<{char: USVString, traversal: () => LexiconTraversal}> {
+  *children(): Generator<{char: string, traversal: () => LexiconTraversal}> {
     let root = this.root;
 
     // We refer to the field multiple times in this method, and it doesn't change.
@@ -299,7 +298,7 @@ export default class TrieModel implements LexicalModel {
     };
   }
 
-  toKey(text: USVString): USVString {
+  toKey(text: string): string {
     return this._trie.toKey(text);
   }
 

web/src/engine/predictive-text/types/message.d.ts

@@ -24,7 +24,6 @@ import { LexicalModelTypes } from '@keymanapp/common-types';
 import Configuration = LexicalModelTypes.Configuration;
 import Reversion = LexicalModelTypes.Reversion;
 import Suggestion = LexicalModelTypes.Suggestion;
-import USVString = LexicalModelTypes.USVString;
 
 /**
  * Tokens are signed 31-bit integers!
@@ -87,7 +86,7 @@ interface CurrentWordMessage {
    * Contains the 'current word' left of the caret given the Context
    * of its source message - the 'wordbreak' message with matching Token value.
    */
-  word: USVString;
+  word: string;
 }
 
 /**

web/src/engine/predictive-text/worker-main/src/lmlayer.ts

@@ -28,7 +28,6 @@ import Distribution = LexicalModelTypes.Distribution;
 import Reversion = LexicalModelTypes.Reversion;
 import Suggestion = LexicalModelTypes.Suggestion;
 import Transform = LexicalModelTypes.Transform;
-import USVString = LexicalModelTypes.USVString;
 import PromiseStore from "./promise-store.js";
 import { OutgoingMessage } from '@keymanapp/lm-message-types';
 
@@ -62,7 +61,7 @@ export default class LMLayer {
   /** Call this when the LMLayer has sent us the 'ready' message! */
   private _declareLMLayerReady: (conf: Configuration) => void;
   private _predictPromises: PromiseStore<Suggestion[]>;
-  private _wordbreakPromises: PromiseStore<USVString>;
+  private _wordbreakPromises: PromiseStore<string>;
   private _acceptPromises: PromiseStore<Reversion>;
   private _revertPromises: PromiseStore<Suggestion[]>;
   private _nextToken: number;
@@ -152,7 +151,7 @@ export default class LMLayer {
     });
   }
 
-  wordbreak(context: Context): Promise<USVString> {
+  wordbreak(context: Context): Promise<string> {
     let token = this._nextToken++;
     return new Promise((resolve, reject) => {
       this._wordbreakPromises.make(token, resolve, reject);

web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts

@@ -12,7 +12,6 @@ import Distribution = LexicalModelTypes.Distribution;
 import LexicalModel = LexicalModelTypes.LexicalModel;
 import Suggestion = LexicalModelTypes.Suggestion;
 import Transform = LexicalModelTypes.Transform;
-import USVString = LexicalModelTypes.USVString;
 
 function textToCharTransforms(text: string, transformId?: number) {
   let perCharTransforms: Transform[] = [];
@@ -95,7 +94,7 @@ export class TrackedContextToken {
    * @param tokenText
    * @param transformId
    */
-  updateWithBackspace(tokenText: USVString, transformId: number) {
+  updateWithBackspace(tokenText: string, transformId: number) {
     // It's a backspace transform; time for special handling!
     //
     // For now, with 14.0, we simply compress all remaining Transforms for the token into
@@ -114,7 +113,7 @@ export class TrackedContextToken {
     this.clearReplacements();
   }
 
-  update(transformDistribution: Distribution<Transform>, tokenText?: USVString) {
+  update(transformDistribution: Distribution<Transform>, tokenText?: string) {
     // Preserve existing text if new text isn't specified.
     tokenText = tokenText || (tokenText === '' ? '' : this.raw);
 
@@ -211,7 +210,7 @@ export class TrackedContextState {
   }
 
   toRawTokenization() {
-    let sequence: USVString[] = [];
+    let sequence: string[] = [];
 
     for(let token of this.tokens) {
       // Hide any tokens representing wordbreaks.  (Thinking ahead to phrase-level possibilities)
@@ -353,7 +352,7 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
     transformSequenceDistribution?: Distribution<Transform[]>
   ): ContextMatchResult {
     // Map the previous tokenized state to an edit-distance friendly version.
-    let matchContext: USVString[] = matchState.toRawTokenization();
+    let matchContext: string[] = matchState.toRawTokenization();
 
     // Inverted order, since 'match' existed before our new context.
     let mapping = ClassicalDistanceCalculation.computeDistance(

web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts

@@ -9,7 +9,6 @@ import LexicalModel = LexicalModelTypes.LexicalModel;
 import LexiconTraversal = LexicalModelTypes.LexiconTraversal;
 import ProbabilityMass = LexicalModelTypes.ProbabilityMass;
 import Transform = LexicalModelTypes.Transform;
-import USVString = LexicalModelTypes.USVString;
 
 type RealizedInput = ProbabilityMass<Transform>[];  // NOT Distribution - they're masses from separate distributions.
 
@@ -48,15 +47,15 @@ export class SearchNode {
   calculation: ClassicalDistanceCalculation<string, EditToken<string>, TraversableToken<string>>;
 
   currentTraversal: LexiconTraversal;
-  toKey: (wordform: USVString) => USVString = str => str;
+  toKey: (wordform: string) => string = str => str;
   priorInput: RealizedInput;
 
   // Internal lazy-cache for .inputSamplingCost, as it's a bit expensive to re-compute.
   private _inputCost?: number;
 
-  constructor(rootTraversal: LexiconTraversal, toKey?: (arg0: USVString) => USVString);
+  constructor(rootTraversal: LexiconTraversal, toKey?: (arg0: string) => string);
   constructor(node: SearchNode);
-  constructor(rootTraversal: LexiconTraversal | SearchNode, toKey?: (arg0: USVString) => USVString) {
+  constructor(rootTraversal: LexiconTraversal | SearchNode, toKey?: (arg0: string) => string) {
     toKey = toKey || (x => x);
 
     if(rootTraversal instanceof SearchNode) {
@@ -294,11 +293,11 @@ export class SearchResult {
     return this.resultNode.priorInput;
   }
 
-  get matchSequence(): TraversableToken<USVString>[] {
+  get matchSequence(): TraversableToken<string>[] {
     return this.resultNode.calculation.matchSequence;
   };
 
-  get matchString(): USVString {
+  get matchString(): string {
     return this.resultNode.resultKey;
   }
 

web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts

@@ -15,7 +15,6 @@ import LexicalModelPunctuation = LexicalModelTypes.LexicalModelPunctuation;
 import Reversion = LexicalModelTypes.Reversion;
 import Suggestion = LexicalModelTypes.Suggestion;
 import Transform = LexicalModelTypes.Transform;
-import USVString = LexicalModelTypes.USVString;
 
 export class ModelCompositor {
   private lexicalModel: LexicalModel;
@@ -194,7 +193,7 @@ export class ModelCompositor {
   }
 
   // Responsible for applying casing rules to suggestions.
-  private applySuggestionCasing(suggestion: Suggestion, baseWord: USVString, casingForm: CasingForm) {
+  private applySuggestionCasing(suggestion: Suggestion, baseWord: string, casingForm: CasingForm) {
     // Step 1:  does the suggestion replace the whole word?  If not, we should extend the suggestion to do so.
     let unchangedLength  = KMWString.length(baseWord) - suggestion.transform.deleteLeft;