diff --git a/common/web/types/src/lexical-model-types.ts b/common/web/types/src/lexical-model-types.ts index 7c5cfdf643..e2670d2e21 100644 --- a/common/web/types/src/lexical-model-types.ts +++ b/common/web/types/src/lexical-model-types.ts @@ -5,18 +5,6 @@ /****************************** Lexical Models ******************************/ -/** - * A JavaScript string with the restriction that it must only - * contain Unicode scalar values. - * - * This means that any lone high surrogate must be paired with - * a low surrogate, if it exists. Lone surrogate code units are - * forbidden. - * - * See also: https://developer.mozilla.org/en-US/docs/Web/API/USVString - */ -export type USVString = string; - export type CasingForm = 'lower' | 'initial' | 'upper'; /** @@ -67,7 +55,7 @@ export interface LexiconTraversal { * - If `char` = 'e', the child represents a prefix of 'the'. * - Then `traversal` allows traversing the part of the lexicon prefixed by 'the'. */ - children(): Generator<{char: USVString, traversal: () => LexiconTraversal}>; + children(): Generator<{char: string, traversal: () => LexiconTraversal}>; /** * Allows direct access to the traversal state that results when appending one @@ -81,7 +69,7 @@ export interface LexiconTraversal { * That is, if a model "keys" `è` to `e`, there will be no `è` child. * @param char */ - child(char: USVString): LexiconTraversal | undefined; + child(char: string): LexiconTraversal | undefined; /** * Any entries directly keyed by the currently-represented lookup prefix. Entries and @@ -178,7 +166,7 @@ export interface LexicalModel { * @param text The original input text. * @returns The 'keyed' form of that text. */ - toKey?(text: USVString): USVString; + toKey?(text: string): string; /** * Generates predictive suggestions corresponding to the state of context after the proposed @@ -248,7 +236,7 @@ export interface LexicalModel { * @param context * @deprecated */ - wordbreak?(context: Context): USVString; + wordbreak?(context: Context): string; /** * Lexical models _may_ provide a LexiconTraversal object usable to enhance @@ -279,7 +267,7 @@ export interface Transform { * * Corresponds to `s` in com.keyman.KeyboardInterface.output. */ - insert: USVString; + insert: string; /** * The number of code units to delete to the left of the cursor. @@ -380,7 +368,7 @@ export interface Context { * buffer. If there is nothing to the left of the buffer, this is * an empty string. */ - readonly left: USVString; + readonly left: string; /** * Up to maxRightContextCodeUnits code units of Unicode scalar value @@ -390,7 +378,7 @@ export interface Context { * * This property may be missing entirely. */ - readonly right?: USVString; + readonly right?: string; /** * Whether the insertion point is at the start of the buffer. diff --git a/common/web/types/tests/lexical-model-types.tests.ts b/common/web/types/tests/lexical-model-types.tests.ts index ef8bed28a5..3075582a39 100644 --- a/common/web/types/tests/lexical-model-types.tests.ts +++ b/common/web/types/tests/lexical-model-types.tests.ts @@ -7,7 +7,7 @@ import { KMXPlus, LdmlKeyboardTypes, LexicalModelTypes } from "@keymanapp/common-types"; -export let u: LexicalModelTypes.USVString; +export let u: string; export let l: LexicalModelTypes.Transform; export let s: LexicalModelTypes.Suggestion; export let st: LexicalModelTypes.SuggestionTag; diff --git a/web/src/engine/predictive-text/templates/src/tokenization.ts b/web/src/engine/predictive-text/templates/src/tokenization.ts index a9bd03c5d3..fd8ed28d5c 100644 --- a/web/src/engine/predictive-text/templates/src/tokenization.ts +++ b/web/src/engine/predictive-text/templates/src/tokenization.ts @@ -214,6 +214,6 @@ export function getLastPreCaretToken(wordBreaker: LexicalModelTypes.WordBreaking // While it is currently identical to getLastWord, this may change in the future. // It's best not to write ourselves into a corner on this one, as disambiguating later // would likely be pretty painful. -export function wordbreak(wordBreaker: LexicalModelTypes.WordBreakingFunction, context: LexicalModelTypes.Context): LexicalModelTypes.USVString { +export function wordbreak(wordBreaker: LexicalModelTypes.WordBreakingFunction, context: LexicalModelTypes.Context): string { return getLastPreCaretToken(wordBreaker, context); } diff --git a/web/src/engine/predictive-text/templates/src/trie-model.ts b/web/src/engine/predictive-text/templates/src/trie-model.ts index 32825f8930..e33c47ce2b 100644 --- a/web/src/engine/predictive-text/templates/src/trie-model.ts +++ b/web/src/engine/predictive-text/templates/src/trie-model.ts @@ -42,7 +42,6 @@ import LexiconTraversal = LexicalModelTypes.LexiconTraversal; import Suggestion = LexicalModelTypes.Suggestion; import TextWithProbability = LexicalModelTypes.TextWithProbability; import Transform = LexicalModelTypes.Transform; -import USVString = LexicalModelTypes.USVString; import WithOutcome = LexicalModelTypes.WithOutcome; import WordBreakingFunction = LexicalModelTypes.WordBreakingFunction; @@ -109,7 +108,7 @@ class Traversal implements LexiconTraversal { this.totalWeight = totalWeight; } - child(char: USVString): LexiconTraversal | undefined { + child(char: string): LexiconTraversal | undefined { // May result for blank tokens resulting immediately after whitespace. if(char == '') { return this; @@ -128,7 +127,7 @@ class Traversal implements LexiconTraversal { } // Handles one code unit at a time. - private _child(char: USVString): Traversal | undefined { + private _child(char: string): Traversal | undefined { const root = this.root; const totalWeight = this.totalWeight; const nextPrefix = this.prefix + char; @@ -154,7 +153,7 @@ class Traversal implements LexiconTraversal { } } - *children(): Generator<{char: USVString, traversal: () => LexiconTraversal}> { + *children(): Generator<{char: string, traversal: () => LexiconTraversal}> { let root = this.root; // We refer to the field multiple times in this method, and it doesn't change. @@ -299,7 +298,7 @@ export default class TrieModel implements LexicalModel { }; } - toKey(text: USVString): USVString { + toKey(text: string): string { return this._trie.toKey(text); } diff --git a/web/src/engine/predictive-text/types/message.d.ts b/web/src/engine/predictive-text/types/message.d.ts index a0ca7c9cb7..39ff92d6ae 100644 --- a/web/src/engine/predictive-text/types/message.d.ts +++ b/web/src/engine/predictive-text/types/message.d.ts @@ -24,7 +24,6 @@ import { LexicalModelTypes } from '@keymanapp/common-types'; import Configuration = LexicalModelTypes.Configuration; import Reversion = LexicalModelTypes.Reversion; import Suggestion = LexicalModelTypes.Suggestion; -import USVString = LexicalModelTypes.USVString; /** * Tokens are signed 31-bit integers! @@ -87,7 +86,7 @@ interface CurrentWordMessage { * Contains the 'current word' left of the caret given the Context * of its source message - the 'wordbreak' message with matching Token value. */ - word: USVString; + word: string; } /** diff --git a/web/src/engine/predictive-text/worker-main/src/lmlayer.ts b/web/src/engine/predictive-text/worker-main/src/lmlayer.ts index 2f59917d0e..5727dabd94 100644 --- a/web/src/engine/predictive-text/worker-main/src/lmlayer.ts +++ b/web/src/engine/predictive-text/worker-main/src/lmlayer.ts @@ -28,7 +28,6 @@ import Distribution = LexicalModelTypes.Distribution; import Reversion = LexicalModelTypes.Reversion; import Suggestion = LexicalModelTypes.Suggestion; import Transform = LexicalModelTypes.Transform; -import USVString = LexicalModelTypes.USVString; import PromiseStore from "./promise-store.js"; import { OutgoingMessage } from '@keymanapp/lm-message-types'; @@ -62,7 +61,7 @@ export default class LMLayer { /** Call this when the LMLayer has sent us the 'ready' message! */ private _declareLMLayerReady: (conf: Configuration) => void; private _predictPromises: PromiseStore; - private _wordbreakPromises: PromiseStore; + private _wordbreakPromises: PromiseStore; private _acceptPromises: PromiseStore; private _revertPromises: PromiseStore; private _nextToken: number; @@ -152,7 +151,7 @@ export default class LMLayer { }); } - wordbreak(context: Context): Promise { + wordbreak(context: Context): Promise { let token = this._nextToken++; return new Promise((resolve, reject) => { this._wordbreakPromises.make(token, resolve, reject); diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts index 5dc4f7d347..516ac779fb 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts @@ -12,7 +12,6 @@ import Distribution = LexicalModelTypes.Distribution; import LexicalModel = LexicalModelTypes.LexicalModel; import Suggestion = LexicalModelTypes.Suggestion; import Transform = LexicalModelTypes.Transform; -import USVString = LexicalModelTypes.USVString; function textToCharTransforms(text: string, transformId?: number) { let perCharTransforms: Transform[] = []; @@ -95,7 +94,7 @@ export class TrackedContextToken { * @param tokenText * @param transformId */ - updateWithBackspace(tokenText: USVString, transformId: number) { + updateWithBackspace(tokenText: string, transformId: number) { // It's a backspace transform; time for special handling! // // For now, with 14.0, we simply compress all remaining Transforms for the token into @@ -114,7 +113,7 @@ export class TrackedContextToken { this.clearReplacements(); } - update(transformDistribution: Distribution, tokenText?: USVString) { + update(transformDistribution: Distribution, tokenText?: string) { // Preserve existing text if new text isn't specified. tokenText = tokenText || (tokenText === '' ? '' : this.raw); @@ -211,7 +210,7 @@ export class TrackedContextState { } toRawTokenization() { - let sequence: USVString[] = []; + let sequence: string[] = []; for(let token of this.tokens) { // Hide any tokens representing wordbreaks. (Thinking ahead to phrase-level possibilities) @@ -353,7 +352,7 @@ export class ContextTracker extends CircularArray { transformSequenceDistribution?: Distribution ): ContextMatchResult { // Map the previous tokenized state to an edit-distance friendly version. - let matchContext: USVString[] = matchState.toRawTokenization(); + let matchContext: string[] = matchState.toRawTokenization(); // Inverted order, since 'match' existed before our new context. let mapping = ClassicalDistanceCalculation.computeDistance( diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts index 555e896c39..316d3bc3e4 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts @@ -9,7 +9,6 @@ import LexicalModel = LexicalModelTypes.LexicalModel; import LexiconTraversal = LexicalModelTypes.LexiconTraversal; import ProbabilityMass = LexicalModelTypes.ProbabilityMass; import Transform = LexicalModelTypes.Transform; -import USVString = LexicalModelTypes.USVString; type RealizedInput = ProbabilityMass[]; // NOT Distribution - they're masses from separate distributions. @@ -48,15 +47,15 @@ export class SearchNode { calculation: ClassicalDistanceCalculation, TraversableToken>; currentTraversal: LexiconTraversal; - toKey: (wordform: USVString) => USVString = str => str; + toKey: (wordform: string) => string = str => str; priorInput: RealizedInput; // Internal lazy-cache for .inputSamplingCost, as it's a bit expensive to re-compute. private _inputCost?: number; - constructor(rootTraversal: LexiconTraversal, toKey?: (arg0: USVString) => USVString); + constructor(rootTraversal: LexiconTraversal, toKey?: (arg0: string) => string); constructor(node: SearchNode); - constructor(rootTraversal: LexiconTraversal | SearchNode, toKey?: (arg0: USVString) => USVString) { + constructor(rootTraversal: LexiconTraversal | SearchNode, toKey?: (arg0: string) => string) { toKey = toKey || (x => x); if(rootTraversal instanceof SearchNode) { @@ -294,11 +293,11 @@ export class SearchResult { return this.resultNode.priorInput; } - get matchSequence(): TraversableToken[] { + get matchSequence(): TraversableToken[] { return this.resultNode.calculation.matchSequence; }; - get matchString(): USVString { + get matchString(): string { return this.resultNode.resultKey; } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts index 2a526a0454..05da67ec98 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts @@ -15,7 +15,6 @@ import LexicalModelPunctuation = LexicalModelTypes.LexicalModelPunctuation; import Reversion = LexicalModelTypes.Reversion; import Suggestion = LexicalModelTypes.Suggestion; import Transform = LexicalModelTypes.Transform; -import USVString = LexicalModelTypes.USVString; export class ModelCompositor { private lexicalModel: LexicalModel; @@ -194,7 +193,7 @@ export class ModelCompositor { } // Responsible for applying casing rules to suggestions. - private applySuggestionCasing(suggestion: Suggestion, baseWord: USVString, casingForm: CasingForm) { + private applySuggestionCasing(suggestion: Suggestion, baseWord: string, casingForm: CasingForm) { // Step 1: does the suggestion replace the whole word? If not, we should extend the suggestion to do so. let unchangedLength = KMWString.length(baseWord) - suggestion.transform.deleteLeft;