diff --git a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/LanguageSettingsActivity.java b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/LanguageSettingsActivity.java index 8db40bb3de..9ec6919bb1 100644 --- a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/LanguageSettingsActivity.java +++ b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/LanguageSettingsActivity.java @@ -110,11 +110,11 @@ public final class LanguageSettingsActivity extends BaseActivity { RadioGroup radioGroup = (RadioGroup) findViewById(R.id.suggestion_radio_group); radioGroup.clearCheck(); - // Auto-correct disabled for Keyman 18.0 #12767 int[] RadioButtonArray = { R.id.suggestion_radio_0, R.id.suggestion_radio_1, - R.id.suggestion_radio_2}; + R.id.suggestion_radio_2, + R.id.suggestion_radio_3}; RadioButton radioButton = (RadioButton)radioGroup.findViewById(RadioButtonArray[maySuggest]); radioButton.setChecked(true); diff --git a/android/KMEA/app/src/main/res/layout/language_settings_list_layout.xml b/android/KMEA/app/src/main/res/layout/language_settings_list_layout.xml index 84f6f189bb..0bddb5c2bd 100644 --- a/android/KMEA/app/src/main/res/layout/language_settings_list_layout.xml +++ b/android/KMEA/app/src/main/res/layout/language_settings_list_layout.xml @@ -83,13 +83,12 @@ android:layout_gravity="center_vertical" android:text="@string/suggestions_radio_2" /> - - + android:text="@string/suggestions_radio_3" /> diff --git a/common/test/resources/json/models/future_suggestions/i_got_distracted_by_hazel.json b/common/test/resources/json/models/future_suggestions/i_got_distracted_by_hazel.json index d01c7f5c5a..1c4dea5050 100644 --- a/common/test/resources/json/models/future_suggestions/i_got_distracted_by_hazel.json +++ b/common/test/resources/json/models/future_suggestions/i_got_distracted_by_hazel.json @@ -2,21 +2,33 @@ [ { "transform": { - "insert": "I ", + "insert": "I", + "deleteLeft": 0 + }, + "appendedTransform": { + "insert": " ", "deleteLeft": 0 }, "displayAs": "I" }, { "transform": { - "insert": "I'm ", + "insert": "I'm", + "deleteLeft": 0 + }, + "appendedTransform": { + "insert": " ", "deleteLeft": 0 }, "displayAs": "I'm" }, { "transform": { - "insert": "Oh ", + "insert": "Oh", + "deleteLeft": 0 + }, + "appendedTransform": { + "insert": " ", "deleteLeft": 0 }, "displayAs": "Oh" @@ -25,21 +37,33 @@ [ { "transform": { - "insert": "love ", + "insert": "love", + "deleteLeft": 0 + }, + "appendedTransform": { + "insert": " ", "deleteLeft": 0 }, "displayAs": "love" }, { "transform": { - "insert": "am ", + "insert": "am", + "deleteLeft": 0 + }, + "appendedTransform": { + "insert": " ", "deleteLeft": 0 }, "displayAs": "am" }, { "transform": { - "insert": "got ", + "insert": "got", + "deleteLeft": 0 + }, + "appendedTransform": { + "insert": " ", "deleteLeft": 0 }, "displayAs": "got" @@ -48,21 +72,33 @@ [ { "transform": { - "insert": "distracted by ", + "insert": "distracted by", + "deleteLeft": 0 + }, + "appendedTransform": { + "insert": " ", "deleteLeft": 0 }, "displayAs": "distracted by" }, { "transform": { - "insert": "distracted ", + "insert": "distracted", + "deleteLeft": 0 + }, + "appendedTransform": { + "insert": " ", "deleteLeft": 0 }, "displayAs": "distracted" }, { "transform": { - "insert": "a ", + "insert": "a", + "deleteLeft": 0 + }, + "appendedTransform": { + "insert": " ", "deleteLeft": 0 }, "displayAs": "a" @@ -71,21 +107,33 @@ [ { "transform": { - "insert": "Hazel ", + "insert": "Hazel", + "deleteLeft": 0 + }, + "appendedTransform": { + "insert": " ", "deleteLeft": 0 }, "displayAs": "Hazel" }, { "transform": { - "insert": "the ", + "insert": "the", + "deleteLeft": 0 + }, + "appendedTransform": { + "insert": " ", "deleteLeft": 0 }, "displayAs": "the" }, { "transform": { - "insert": "a ", + "insert": "a", + "deleteLeft": 0 + }, + "appendedTransform": { + "insert": " ", "deleteLeft": 0 }, "displayAs": "a" diff --git a/common/web/types/src/lexical-model-types.ts b/common/web/types/src/lexical-model-types.ts index 82619b4ce4..9938abec5a 100644 --- a/common/web/types/src/lexical-model-types.ts +++ b/common/web/types/src/lexical-model-types.ts @@ -303,11 +303,18 @@ export interface Suggestion { id?: number; /** - * The suggested update to the buffer. Note that this transform should - * be applied AFTER the instigating transform, if any. + * Specifies the edits needed to correct and extend the currently-edited word + * (within the text buffer) to match the suggested word from the lexicon. + * Note that this transform should be applied BEFORE the instigating transform, if any. */ readonly transform: Transform; + /** + * Applies extra language-appropriate whitespace and/or punctuation after the main + * Suggestion body as specified by the source LexicalModel. + */ + appendedTransform?: Transform; + /** * A string to display the suggestion to the typist. * This should aid the typist understand what the transform diff --git a/web/README.md b/web/README.md index 7f7415c698..05f9432c20 100644 --- a/web/README.md +++ b/web/README.md @@ -1,5 +1,5 @@ # Keyman Engine for Web -The Original Code is (C) SIL International +The Original Code is (C) SIL Global ## Prerequisites See [build configuration](../docs/build/index.md) for details on how to diff --git a/web/src/engine/main/src/headless/languageProcessor.ts b/web/src/engine/main/src/headless/languageProcessor.ts index b9731c9783..2921396b5e 100644 --- a/web/src/engine/main/src/headless/languageProcessor.ts +++ b/web/src/engine/main/src/headless/languageProcessor.ts @@ -21,7 +21,7 @@ export class LanguageProcessor extends EventEmitter { private _mayPredict: boolean = true; private _mayCorrect: boolean = true; - private _mayAutoCorrect: boolean = false; // initialized to false - #12767 + private _mayAutoCorrect: boolean = true; private _state: StateChangeEnum = 'inactive'; @@ -216,6 +216,9 @@ export class LanguageProcessor extends EventEmitter { // Step 1: determine the final output text const final = Mock.from(original.preInput, false); final.apply(suggestion.transform); + if(suggestion.appendedTransform) { + final.apply(suggestion.appendedTransform); + } // Step 2: build a final, master Transform that will produce the desired results from the CURRENT state. // In embedded mode, both Android and iOS are best served by calculating this transform and applying its @@ -289,6 +292,9 @@ export class LanguageProcessor extends EventEmitter { // Step 1: determine the final output text const final = Mock.from(original.preInput, false); final.apply(reversion.transform); // Should match original.transform, actually. (See applySuggestion) + if(reversion.appendedTransform) { + final.apply(reversion.appendedTransform); + } // Step 2: build a final, master Transform that will produce the desired results from the CURRENT state. // In embedded mode, both Android and iOS are best served by calculating this transform and applying its diff --git a/web/src/engine/osk/src/banner/suggestionBanner.ts b/web/src/engine/osk/src/banner/suggestionBanner.ts index a4d852cb49..25143cb316 100644 --- a/web/src/engine/osk/src/banner/suggestionBanner.ts +++ b/web/src/engine/osk/src/banner/suggestionBanner.ts @@ -773,11 +773,10 @@ export class SuggestionBanner extends Banner { if(suggestions.length > i) { const suggestion = suggestions[i]; d.update(suggestion, optionFormat); - if(this.predictionContext.selected == suggestion) { - d.highlight(true); - } + d.highlight(this.predictionContext.selected == suggestion) } else { d.update(null, optionFormat); + d.highlight(false); } } diff --git a/web/src/engine/predictive-text/templates/src/common.ts b/web/src/engine/predictive-text/templates/src/common.ts index f23ad6095b..b59bbccb48 100644 --- a/web/src/engine/predictive-text/templates/src/common.ts +++ b/web/src/engine/predictive-text/templates/src/common.ts @@ -55,13 +55,18 @@ export function buildMergedTransform(first: Transform, second: Transform): Trans } } - return { + const returnedObj: Transform = { insert: mergedFirstInsert + second.insert, - deleteLeft: first.deleteLeft + mergedSecondDelete, + deleteLeft: first.deleteLeft + mergedSecondDelete + } + + if(first.deleteRight != undefined || second.deleteRight != undefined) { // As `first` would affect the context before `second` could take effect, // this is the correct way to merge `deleteRight`. - deleteRight: (first.deleteRight || 0) + (second.deleteRight || 0) + returnedObj.deleteRight = (first.deleteRight || 0) + (second.deleteRight || 0) } + + return returnedObj; } /** diff --git a/web/src/engine/predictive-text/wordbreakers/src/main/default/index.ts b/web/src/engine/predictive-text/wordbreakers/src/main/default/index.ts index c807c4ab2a..26b5dde564 100644 --- a/web/src/engine/predictive-text/wordbreakers/src/main/default/index.ts +++ b/web/src/engine/predictive-text/wordbreakers/src/main/default/index.ts @@ -38,7 +38,7 @@ export interface DefaultWordBreakerOptions { * @see http://unicode.org/reports/tr29/#Word_Boundaries * @see https://github.com/eddieantonio/unicode-default-word-boundary/tree/v12.0.0 */ -export default function default_(text: string, options?: DefaultWordBreakerOptions): LexicalModelTypes.Span[] { + function default_(text: string, options?: DefaultWordBreakerOptions): LexicalModelTypes.Span[] { let boundaries = findBoundaries(text, options); if (boundaries.length == 0) { return []; @@ -64,6 +64,16 @@ export default function default_(text: string, options?: DefaultWordBreakerOptio return spans; } +// Exposes `searchForProperty` for external use while associating it with this wordbreaker. +const def = Object.assign(default_, { + /** + * This method returns enum values corresponding to the character type as perceived by the wordbreaking algorithm. + */ + searchForProperty: searchForProperty +}); + +export default def; + /** * A span that does not cut out the substring until it absolutely has to! */ diff --git a/web/src/engine/predictive-text/wordbreakers/src/main/index.ts b/web/src/engine/predictive-text/wordbreakers/src/main/index.ts index 79cba5d83a..5248908119 100644 --- a/web/src/engine/predictive-text/wordbreakers/src/main/index.ts +++ b/web/src/engine/predictive-text/wordbreakers/src/main/index.ts @@ -1,6 +1,7 @@ import placeholder from "./placeholder.js"; import ascii from "./ascii.js"; import default_ from "./default/index.js"; +import { WordBreakProperty } from "./default/data.inc.js"; -export { placeholder, ascii, default_ as default, default_ as defaultWordbreaker }; -export { type BreakerContext } from "./default/index.js"; \ No newline at end of file +export { placeholder, ascii, default_ as default, default_ as defaultWordbreaker, WordBreakProperty }; +export { type BreakerContext } from "./default/index.js"; diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/alignment-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/alignment-helpers.ts new file mode 100644 index 0000000000..b2b2e73675 --- /dev/null +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/alignment-helpers.ts @@ -0,0 +1,124 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2025-07-30 + * + * This file defines methods used as helpers when aligning cached context state + * information with incoming contexts and when validating partial substitution + * edits for aligned context tokens. + */ + +import { ClassicalDistanceCalculation, EditOperation } from "./classical-calculation.js"; + +/** + * Determines the proper 'last match' index for a tokenized sequence based on its edit path. + * + * In particular, this method is designed to handle the following case: + * ['to', 'apple', ' ', ''] => ['to', 'apply', ' ', 'n'] + * + * Edit path for this example case: + * ['match', 'substitute', 'match', 'substitute'] + * + * In cases such as these, the whitespace match should be considered 'edited'. While the ' ' + * is unedited, it follows the edited 'apple' => 'apply', so it must have been deleted and + * then re-inserted. As a result, 'to' is the true "last matched" token. + * @param editPath + * @returns + */ +export function getEditPathLastMatch(editPath: EditOperation[]) { + const editLength = editPath.length; + // Special handling: appending whitespace to whitespace with the default wordbreaker. + // The default wordbreaker currently adds an empty token after whitespace; this would + // show up with 'substitute', 'match' at the end of the edit path. (This should remain.) + if(editLength >= 2 && editPath[editLength - 2] == 'substitute' && editPath[editLength - 1] == 'match') { + return editPath.lastIndexOf('match', editLength - 2); + } else { + return editPath.lastIndexOf('match'); + } +} + +/** + * Aligns two tokens on a character-by-character basis as needed for higher, token-level alignment + * operations. + * @param incomingToken The incoming token value + * @param matchingToken The pre-existing token value to use for comparison and alignment + * @param forNearCaret If `false`, disallows any substitutions and activates a leading-edge alignment + * validation mode. + * @returns + */ +export function isSubstitutionAlignable( + incomingToken: string, + matchingToken: string, + forNearCaret?: boolean +): boolean { + // 1 - Determine the edit path for the word. + let subEditPath = ClassicalDistanceCalculation.computeDistance( + [...matchingToken].map(value => ({key: value})), + [...incomingToken].map(value => ({key: value})), + // Diagonal width to consider must be at least 2, as adding a single + // whitespace after a token tends to add two tokens: one for whitespace, + // one for the empty token to follow it. + 3 + ).editPath(); + + const firstInsert = subEditPath.indexOf('insert'); + const firstDelete = subEditPath.indexOf('delete'); + + // 2 - deletions and insertions should be mutually exclusive. + // A fixed, unedited word can't slide across both 'left' and 'right' boundaries at the same time. + if(firstInsert != -1 && firstDelete != -1) { + return false; + }; + + // 3 - checks exclusive to leading-edge conditions + if(!forNearCaret) { + const firstSubstitute = subEditPath.indexOf('substitute'); + const firstMatch = subEditPath.indexOf('match'); + if(firstSubstitute > -1) { + return false; + } else if(firstMatch > -1) { + // Should not have inserts on both sides of matched text! + if(firstInsert > -1 && firstInsert < firstMatch && subEditPath.lastIndexOf('insert') > firstMatch) { + return false; + } else if(firstDelete > -1 && firstDelete < firstMatch && subEditPath.lastIndexOf('delete') > firstMatch) { + return false; + } + } + + // Further checks below are oriented for text/tokens at the caret. + return true; + } + + // 4 - check the stats for total edits of each type and validate that edits don't overly exceed + // original characters. + const editCount = { + matchMove: 0, + rawEdit: 0 + }; + + subEditPath.forEach((entry) => { + switch(entry) { + case 'transpose-end': + case 'transpose-start': + case 'match': + editCount.matchMove++; + break; + case 'insert': + case 'transpose-insert': + case 'delete': + case 'transpose-delete': + case 'substitute': + editCount.rawEdit++; + } + }); + + // We shouldn't have more raw substitutions, inserts, and deletes than matches + transposes, + // though allowing +1 as a fudge factor. + // The 'a' => 'à' pattern can be a reasonably common Keyman keyboard rule and + // is one substitution, zero matches in NFC. + if(editCount.matchMove + 1 < editCount.rawEdit) { + return false; + } + + return true; +} \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts new file mode 100644 index 0000000000..70244011ea --- /dev/null +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts @@ -0,0 +1,146 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2025-07-30 + * + * Represents cached data about the state of the sliding context window either + * before or after a context transition event and related functionality. + */ + +import { LexicalModelTypes } from '@keymanapp/common-types'; + +import { ContextTokenization } from './context-tokenization.js'; + +import Context = LexicalModelTypes.Context; +import Distribution = LexicalModelTypes.Distribution; +import LexicalModel = LexicalModelTypes.LexicalModel; +import Suggestion = LexicalModelTypes.Suggestion; +import Transform = LexicalModelTypes.Transform; +import { ContextToken } from './context-token.js'; +import { determineModelTokenizer } from '#./model-helpers.js'; + +/** + * Represents a state of the active context at some point in time along with the + * results of all related, reusable predictive-text operations. + */ +export class ContextState { + /** + * The context window in view for the represented Context state, + * as passed between the predictive-text worker and its host. + */ + readonly context: Context; + + /** + * The active lexical model operating upon the Context. + */ + readonly model: LexicalModel; + + /** + * Denotes the most likely tokenization for the represented Context. + */ + tokenization: ContextTokenization; + + /** + * Denotes the keystroke-sourced Transform that was last applied to a + * prior ContextState. + * + * Note: if this specific ContextState resulted from applying a + * Suggestion, this may not match text seen in the current Context! + */ + appliedInput?: Transform; + + /** + * Denotes all keystroke data contributing to ContextTokens seen in + * .tokenization. For each contributing context transition, its ID + * may be used to retrieve the original fat-finger distribution for + * potential keystroke effects. + */ + inputTransforms: Map>; + + /** + * The full set of Suggestions produced for the transition to this context state. + */ + suggestions: Suggestion[]; + + /** + * If set, denotes the suggestion ID for the suggestion (from .suggestions) that + * was applied for the final transition to this context state. + */ + appliedSuggestionId?: number; + + /** + * Indicates whether or not the applied suggestion (if it exists) was applied + * directly by the user. + * + * - `true` if directly applied via banner interaction or other explicitly-intended + * behaviors + * - `false` if indirectly applied (say, by triggering whitespace/punctuation input) + * - `undefined` if no suggestion has been applied. + */ + isManuallyApplied?: boolean; + + /** + * Deep-copies a prior instance. + * @param stateToClone + */ + constructor(stateToClone: ContextState); + /** + * Initializes a new ContextState instance based on the active model and context. + * + * If a precomputed tokenization of the context (with prior correction-search + * calculation data) is not available, it will be spun up from scratch. + * + * @param context The context available within the current sliding context-window + * @param model The active lexical model. + * @param tokenization Precomputed tokenization for the context, leveraging previous + * correction-search progress and results + */ + constructor(context: Context, model: LexicalModel, tokenization?: ContextTokenization); + constructor(param1: Context | ContextState, model?: LexicalModel, tokenization?: ContextTokenization) { + if(!(param1 instanceof ContextState)) { + this.context = param1; + this.model = model; + if(tokenization) { + this.tokenization = tokenization; + } else { + this.initFromReset(); + } + } else { + const stateToClone = param1; + + Object.assign(this, stateToClone); + this.inputTransforms = new Map(stateToClone.inputTransforms); + this.tokenization = new ContextTokenization(stateToClone.tokenization); + + // A shallow copy of the array is fine, but we'd be best off + // not aliasing the array itself. + if(stateToClone.suggestions?.length ?? 0 > 0) { + this.suggestions = [].concat(stateToClone.suggestions); + } + } + } + + /** + * Initializes the ContextState instance for use when no valid prior + * information is available - typically, immediately after engine + * initialization or a context reset. + */ + private initFromReset() { + const tokenizedContext = determineModelTokenizer(this.model)(this.context).left; + const baseTokens = tokenizedContext.map((entry) => { + const token = new ContextToken(this.model, entry.text); + + if(entry.isWhitespace) { + token.isWhitespace = true; + } + + return token; + }); + + // And now build the final context state object, which includes whitespace 'tokens'.); + if(baseTokens.length == 0) { + baseTokens.push(new ContextToken(this.model)); + } + this.tokenization = new ContextTokenization(baseTokens); + } +} \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts new file mode 100644 index 0000000000..2168d31300 --- /dev/null +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts @@ -0,0 +1,129 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2025-07-30 + * + * Represents cached data about one token (either a word or a unit of whitespace) + * in the context and associated correction-search progress and results. + */ + +import { buildMergedTransform } from "@keymanapp/models-templates"; +import { LexicalModelTypes } from '@keymanapp/common-types'; + +import { SearchSpace } from "./distance-modeler.js"; + +import Distribution = LexicalModelTypes.Distribution; +import LexicalModel = LexicalModelTypes.LexicalModel; +import Suggestion = LexicalModelTypes.Suggestion; +import Transform = LexicalModelTypes.Transform; + +/** + * Breaks apart a raw text string into individual, single-codepoint + * transforms, all set with the specified transform ID. + * + * This is designed for use when initializing a new ContextToken without + * any prior cached data or for rewriting its probabilities after + * receiving backspace input. + * @param text + * @param transformId + * @returns + */ +function textToCharTransforms(text: string, transformId?: number): Transform[] { + return transformId ? + [...text].map(insert => ({insert, deleteLeft: 0, id: transformId})) : + [...text].map(insert => ({insert, deleteLeft: 0})); +} + +/** + * Represents cached data about one token (either a word or a unit of whitespace) + * in the context and associated correction-search progress and results. + */ +export class ContextToken { + /** + * Indicates whether or not the token is considered whitespace. + */ + isWhitespace: boolean; + + /** + * Contains all relevant correction-search data for use in generating + * corrections for this ContextToken instance. + */ + readonly searchSpace: SearchSpace; + + /* The next two fields will **not land here** in the final version for + epic/autocorrect / 19.0-beta! That said, their future location has + not yet been reworked, so we'll keep them here for now. */ + + /** + * The set of suggestions generated for the current token + */ + suggestions: Suggestion[]; + + /** + * The ID of the suggestion applied to the current token, if any. + * + * Should be set to undefined when no such suggestion exists. + */ + appliedSuggestionId?: number; + + /** + * Constructs a new, empty instance for use with the specified LexicalModel. + * @param model + */ + constructor(model: LexicalModel); + /** + * Constructs a new instance with pre-existing text for use with the specified LexicalModel. + * @param model + * @param rawText + */ + constructor(model: LexicalModel, rawText: string); + /** + * This constructor deep-copies the specified instance. + * @param baseToken + */ + constructor(baseToken: ContextToken); + constructor(param: ContextToken | LexicalModel, rawText?: string) { + if(param instanceof ContextToken) { + const priorToken = param; + this.isWhitespace = priorToken.isWhitespace; + + // We need to construct a separate search space from other token copies. + // + // In case we are unable to perfectly track context (say, due to multitaps) + // we need to ensure that only fully-utilized keystrokes are considered. + this.searchSpace = new SearchSpace(priorToken.searchSpace); + this.suggestions = priorToken.suggestions.slice(); + + // because of unit tests. + if(priorToken.appliedSuggestionId !== undefined) { + this.appliedSuggestionId = priorToken.appliedSuggestionId; + } + } else { + const model = param; + + // May be altered outside of the constructor. + this.isWhitespace = false; + this.searchSpace = new SearchSpace(model); + + rawText ||= ''; + + // Supports the old pathway for: updateWithBackspace(tokenText: string, transformId: number) + const rawTransformDistributions: Distribution[] = textToCharTransforms(rawText).map(function(transform) { + return [{sample: transform, p: 1.0}]; + }); + rawTransformDistributions.forEach((entry) => this.searchSpace.addInput(entry)); + + this.suggestions = []; + } + } + + /** + * Displays text corresponding to the net effects of the most likely inputs received + * that can correspond to the current instance. + */ + get exampleInput(): string { + const transforms = this.searchSpace.inputSequence.map((dist) => dist[0].sample) + const composite = transforms.reduce((accum, current) => buildMergedTransform(accum, current), { insert: '', deleteLeft: 0}); + return composite.insert; + } +} \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts new file mode 100644 index 0000000000..03e2bb0d8b --- /dev/null +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -0,0 +1,302 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2025-07-30 + * + * Represents cached data about one potential tokenization of contents of + * the sliding context window for one specific instance of context state. + */ + +import { ContextToken } from './context-token.js'; +import { ClassicalDistanceCalculation } from './classical-calculation.js'; +import { getEditPathLastMatch, isSubstitutionAlignable } from './alignment-helpers.js'; + +/** + * Represents token-count values resulting from an alignment attempt between two + * different modeled context states. + */ +export type ContextStateAlignment = { + /** + * Denotes whether or not alignment is possible between two contexts. + */ + canAlign: false +} | { + /** + * Denotes whether or not alignment is possible between two contexts. + */ + canAlign: true, + /** + * Notes the number of tokens added to the head of the 'incoming'/'new' context + * of the contexts being aligned. If negative, the incoming context deleted + * a token found in the 'original' / base context. + * + * For the alignment, [base context index] + leadTokenShift = [incoming context index]. + */ + leadTokenShift: number, + /** + * The count of tokens perfectly aligned, with no need for edits, for two successfully- + * alignable contexts. + */ + matchLength: number, + /** + * The count of tokens at the tail perfectly aligned (existing in both contexts) but + * edited for two successfully-alignable contexts. These tokens directly follow those + * that need no edits. + */ + tailEditLength: number, + /** + * The count of new tokens added at the end of the incoming context for two aligned contexts. + * If negative, the incoming context deleted a previously-existing token from the original. + */ + tailTokenShift: number +}; + +/** + * This class represents the sequence of tokens (words and whitespace blocks) + * held within the active sliding context-window at a single point in time. + */ +export class ContextTokenization { + readonly tokens: ContextToken[]; + readonly alignment?: ContextStateAlignment; + + constructor(priorToClone: ContextTokenization); + constructor(tokens: ContextToken[], alignment?: ContextStateAlignment); + constructor(param1: ContextToken[] | ContextTokenization, alignment?: ContextStateAlignment) { + if(!(param1 instanceof ContextTokenization)) { + const tokens = param1; + this.tokens = [].concat(tokens); + this.alignment = alignment; + } else { + const priorToClone = param1; + this.tokens = priorToClone.tokens.map((entry) => new ContextToken(entry)); + this.alignment = {...priorToClone.alignment}; + } + } + + /** + * Returns the token adjacent to the text insertion point. + */ + get tail(): ContextToken { + return this.tokens[this.tokens.length - 1]; + } + + /** + * Returns a plain-text string representing the most probable representation for all + * tokens represented by this tokenization instance. + */ + get exampleInput(): string[] { + return this.tokens + // Hide any tokens representing invisible wordbreaks. (Thinking ahead to phrase-level possibilities) + .filter(token => token.exampleInput !== null) + .map(token => token.exampleInput); + } + + /** + * Determines the alignment between a new, incoming tokenization source and the + * tokenization modeled by the current instance. + * @param incomingTokenization Raw strings corresponding to the tokenization of the incoming context + * @returns Alignment data that details if and how the incoming tokenization aligns with + * the tokenization modeled by this instance. + */ + computeAlignment(incomingTokenization: string[]): ContextStateAlignment { + // Map the tokenized state to an edit-distance friendly version. + const tokenizationToMatch = this.exampleInput; + + // Inverted order, since 'match' existed before our new context. + let mapping = ClassicalDistanceCalculation.computeDistance( + tokenizationToMatch.map(value => ({key: value})), + incomingTokenization.map(value => ({key: value})), + // Diagonal width to consider must be at least 2, as adding a single + // whitespace after a token tends to add two tokens: one for whitespace, + // one for the empty token to follow it. + 3 + ); + + let editPath = mapping.editPath(); + // Special case: new context bootstrapping - first token often substitutes. + // The text length is small enough that no words should be able to rotate out the start of the context. + // Special handling needed in case of no 'match'; the rest of the method assumes at least one 'match'. + if(editPath.length <= 3 && (editPath[0] == 'substitute' || editPath[0] == 'match')) { + let matchCount = 0; + let subCount = 0; + for(let i = 0; i < editPath.length; i++) { + if(editPath[i] == 'substitute') { + subCount++; + if(!isSubstitutionAlignable(incomingTokenization[i], tokenizationToMatch[i], true)) { + return { + canAlign: false + }; + } + } else if(editPath[i] == 'match') { + // If a substitution is already recorded, treat the 'match' as a substitution. + if(subCount > 0) { + subCount++; + } else { + matchCount++; + } + } + } + + const insertCount = editPath.filter((entry) => entry == 'insert').length; + const deleteCount = editPath.filter((entry) => entry == 'delete').length; + + return { + canAlign: true, + matchLength: matchCount, + leadTokenShift: 0, + tailEditLength: subCount, + tailTokenShift: insertCount - deleteCount + } + } + + // From here on assumes that at least one 'match' exists on the path. + // It all works great... once the context is long enough for at least one stable token. + const firstMatch = editPath.indexOf('match'); + const lastMatch = getEditPathLastMatch(editPath); + if(firstMatch == -1) { + // If there are no matches, there's no alignment. + return { + canAlign: false + }; + } + + // Transpositions are not allowed at the token level during context alignment. + if(editPath.find((entry) => entry.indexOf('transpose') > -1)) { + return { + canAlign: false + }; + } + + let matchLength = lastMatch - firstMatch + 1; + let tailInsertLength = 0; + let tailDeleteLength = 0; + for(let i = lastMatch; i < editPath.length; i++) { + if(editPath[i] == 'insert') { + tailInsertLength++; + } else if(editPath[i] == 'delete') { + tailDeleteLength++; + } + } + if(tailInsertLength > 0 && tailDeleteLength > 0) { + // Something's gone weird if this happens; that should appear as a substitution instead. + // Otherwise, we have a VERY niche edit scenario. + return { + canAlign: false + }; + } + const tailSubstituteLength = (editPath.length - 1 - lastMatch) - tailInsertLength - tailDeleteLength; + + // Assertion: for a long context, the bulk of the edit path should be a + // continuous block of 'match' entries. If there's anything else in + // the middle, we have a context mismatch. + if(firstMatch > -1) { + for(let i = firstMatch+1; i < lastMatch; i++) { + if(editPath[i] != 'match') { + return { + canAlign: false + }; + } + } + } + + // If we have a perfect match with a pre-existing context, no mutations have + // happened; we have a 100% perfect match. + if(firstMatch == 0 && lastMatch == editPath.length - 1) { + return { + canAlign: true, + leadTokenShift: 0, + matchLength, + tailEditLength: tailSubstituteLength, + tailTokenShift: tailInsertLength - tailDeleteLength + }; + } + + // The edit path calc tries to put substitutes first, before inserts. + // We don't want that on the leading edge. + const lastEarlyInsert = editPath.lastIndexOf('insert', firstMatch); + const firstSubstitute = editPath.indexOf('substitute'); + if(firstSubstitute > -1 && firstSubstitute < firstMatch && firstSubstitute < lastEarlyInsert) { + editPath[firstSubstitute] = 'insert'; + editPath[lastEarlyInsert] = 'substitute'; + } + + // If mutations HAVE happened, we need to double-check the context-state alignment. + let priorEdit: typeof editPath[0]; + let leadTokensRemoved = 0; + let leadSubstitutions = 0; + + // The `i` index below aligns based upon the index within the `tokenizationToMatch` sequence + // and how it would have to be edited to align to the `incomingTokenization` sequence. + for(let i = 0; i < firstMatch; i++) { + switch(editPath[i]) { + case 'delete': + // All deletions should appear at the sliding window edge; if a deletion appears + // after the edge, but before the first match, something's wrong. + if(priorEdit && priorEdit != 'delete') { + return { + canAlign: false + }; + } + leadTokensRemoved++; + break; + case 'substitute': + // We only allow for one leading token to be substituted. + // + // Any extras in the front would be pure inserts, not substitutions, due to + // the sliding context window and its implications. + if(leadSubstitutions++ > 0) { + return { + canAlign: false + }; + } + + // Find the word before and after substitution. + const incomingSub = incomingTokenization[i - (leadTokensRemoved > 0 ? leadTokensRemoved : 0)]; + const matchingSub = tokenizationToMatch[i + (leadTokensRemoved < 0 ? leadTokensRemoved : 0)]; + + // Double-check the word - does the 'substituted' word itself align? + if(!isSubstitutionAlignable(incomingSub, matchingSub)) { + return { + canAlign: false + }; + } + + // There's no major need to drop parts of a token being 'slid' out of the context window. + // We'll leave it intact and treat it as a 'match' + matchLength++; + break; + case 'insert': + // Only allow an insert at the leading edge, as with 'delete's. + if(priorEdit && priorEdit != 'insert') { + return { + canAlign: false + }; + } + // In case of backspaces, it's also possible to 'insert' a 'new' + // token - an old one that's slid back into view. + leadTokensRemoved--; + break; + default: + // No 'match' can exist before the first found index for a 'match'. + // No 'transpose-' edits should exist within this section, either. + return { + canAlign: false + }; + } + priorEdit = editPath[i]; + } + + // If we need some form of tail-token substitution verification, add that here. + + return { + canAlign: true, + // leadTokensRemoved represents the number of tokens that must be removed from the base context + // when aligning the contexts. Externally, it's more helpful to think in terms of the count added + // to the incoming context. + leadTokenShift: -leadTokensRemoved + 0, // add 0 in case of a 'negative zero', which affects unit tests. + matchLength, + tailEditLength: tailSubstituteLength, + tailTokenShift: tailInsertLength - tailDeleteLength + }; + } +} \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts index 375e414b1b..f6c629e1d4 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts @@ -1,227 +1,17 @@ -import { applyTransform, buildMergedTransform, Token } from '@keymanapp/models-templates'; -import { KMWString } from '@keymanapp/web-utils'; +import { applyTransform, buildMergedTransform } from '@keymanapp/models-templates'; -import { ClassicalDistanceCalculation, EditOperation } from './classical-calculation.js'; -import { SearchSpace } from './distance-modeler.js'; import TransformUtils from '../transformUtils.js'; import { determineModelTokenizer } from '../model-helpers.js'; -import { tokenizeTransform, tokenizeTransformDistribution } from './transform-tokenization.js'; +import { tokenizeAndFilterDistribution } from './transform-tokenization.js'; import { LexicalModelTypes } from '@keymanapp/common-types'; import Context = LexicalModelTypes.Context; import Distribution = LexicalModelTypes.Distribution; import LexicalModel = LexicalModelTypes.LexicalModel; -import Suggestion = LexicalModelTypes.Suggestion; import Transform = LexicalModelTypes.Transform; - -function textToCharTransforms(text: string, transformId?: number) { - let perCharTransforms: Transform[] = []; - - for(let i=0; i < KMWString.length(text); i++) { - let char = KMWString.charAt(text, i); // is SMP-aware - - let transform: Transform = { - insert: char, - deleteLeft: 0, - id: transformId - }; - - perCharTransforms.push(transform); - } - - return perCharTransforms; -} - -export function getEditPathLastMatch(editPath: EditOperation[]) { - const editLength = editPath.length; - // Special handling: appending whitespace to whitespace with the default wordbreaker. - // The default wordbreaker currently adds an empty token after whitespace; this would - // show up with 'substitute', 'match' at the end of the edit path. (This should remain.) - if(editLength >= 2 && editPath[editLength - 2] == 'substitute' && editPath[editLength - 1] == 'match') { - return editPath.lastIndexOf('match', editLength - 2); - } else { - return editPath.lastIndexOf('match'); - } -} - -export class TrackedContextSuggestion { - suggestion: Suggestion; - tokenWidth: number; -} - -export class TrackedContextToken { - raw: string; - replacementText: string; - isWhitespace?: boolean; - - transformDistributions: Distribution[] = []; - replacements: TrackedContextSuggestion[] = []; - activeReplacementId: number = -1; - - constructor(); - constructor(instance: TrackedContextToken); - constructor(instance?: TrackedContextToken) { - if(instance) { - Object.assign(this, instance); - // We don't alter the values in replacements, but we do wish to prevent aliasing - // of the array containing them. - this.replacements = instance.replacements.slice(); - } - } - - get currentText(): string { - if(this.replacementText === undefined || this.replacementText === null) { - return this.raw; - } else { - return this.replacementText; - } - } - - get replacement(): TrackedContextSuggestion { - let replacementId = this.activeReplacementId; - return this.replacements.find(function(replacement) { - return replacement.suggestion.id == replacementId; - }); - } - - clearReplacements() { - this.activeReplacementId = -1; - this.replacements = [] - } - - /** - * Used for 14.0's backspace workaround, which flattens all previous Distribution - * entries because of limitations with direct use of backspace transforms. - * @param tokenText - * @param transformId - */ - updateWithBackspace(tokenText: string, transformId: number) { - // It's a backspace transform; time for special handling! - // - // For now, with 14.0, we simply compress all remaining Transforms for the token into - // multiple single-char transforms. Probabalistically modeling BKSP is quite complex, - // so we simplify by assuming everything remaining after a BKSP is 'true' and 'intended' text. - // - // Note that we cannot just use a single, monolithic transform at this point b/c - // of our current edit-distance optimization strategy; diagonalization is currently... - // not very compatible with that. - let backspacedTokenContext: Distribution[] = textToCharTransforms(tokenText, transformId).map(function(transform) { - return [{sample: transform, p: 1.0}]; - }); - - this.raw = tokenText; - this.transformDistributions = backspacedTokenContext; - this.clearReplacements(); - } - - update(transformDistribution: Distribution, tokenText?: string) { - // Preserve existing text if new text isn't specified. - tokenText = tokenText || (tokenText === '' ? '' : this.raw); - - if(transformDistribution?.length > 0) { - this.transformDistributions.push(transformDistribution); - } - - // Replace old token's raw-text with new token's raw-text. - this.raw = tokenText; - this.clearReplacements(); - } -} - -export class TrackedContextState { - // Stores the post-transform Context. Useful as a debugging reference, but also used to - // pre-validate context state matches in case of discarded changes from multitaps. - taggedContext: Context; - model: LexicalModel; - - tokens: TrackedContextToken[]; - /** - * How many tokens were removed from the start of the best-matching ancestor. - * Useful for restoring older states, e.g., when the user moves the caret backwards, we can recover the context at that position. - */ - indexOffset: number; - - // Tracks all search spaces starting at the current token. - // In the lm-layer's current form, this should only ever have one entry. - // Leaves 'design space' for if/when we add support for phrase-level corrections/predictions. - searchSpace: SearchSpace[] = []; - - constructor(source: TrackedContextState); - constructor(model: LexicalModel); - constructor(obj: TrackedContextState | LexicalModel) { - if(obj instanceof TrackedContextState) { - let source = obj; - // Be sure to deep-copy the tokens! Pointer-aliasing is bad here. - this.tokens = source.tokens.map(function(token) { - let copy = new TrackedContextToken(); - Object.assign(copy, token); - copy.replacements = copy.replacements.slice(); - copy.transformDistributions = copy.transformDistributions.slice(); - return copy; - }); - - this.indexOffset = 0; - const lexicalModel = this.model = obj.model; - this.taggedContext = obj.taggedContext; - - if(lexicalModel?.traverseFromRoot) { - // We need to construct a separate search space from other ContextStates. - // - // In case we are unable to perfectly track context (say, due to multitaps) - // we need to ensure that only fully-utilized keystrokes are considered. - this.searchSpace = obj.searchSpace.map((space) => new SearchSpace(space)); - } - } else { - let lexicalModel = obj; - this.tokens = []; - this.indexOffset = Number.MIN_SAFE_INTEGER; - this.model = lexicalModel; - - if(lexicalModel && lexicalModel.traverseFromRoot) { - this.searchSpace = [new SearchSpace(lexicalModel)]; - } - } - } - - get head(): TrackedContextToken { - return this.tokens[0]; - } - - get tail(): TrackedContextToken { - return this.tokens[this.tokens.length - 1]; - } - - popHead() { - this.tokens.splice(0, 1); - this.indexOffset -= 1; - } - - pushTail(token: TrackedContextToken) { - if(this.model && this.model.traverseFromRoot) { - this.searchSpace = [new SearchSpace(this.model)]; // yeah, need to update SearchSpace for compatibility - } else { - this.searchSpace = []; - } - this.tokens.push(token); - - let state = this; - if(state.searchSpace.length > 0) { - token.transformDistributions.forEach(distrib => state.searchSpace[0].addInput(distrib)); - } - } - - toRawTokenization() { - let sequence: string[] = []; - - for(let token of this.tokens) { - // Hide any tokens representing wordbreaks. (Thinking ahead to phrase-level possibilities) - if(token.currentText !== null) { - sequence.push(token.currentText); - } - } - - return sequence; - } -} +import { ContextToken } from './context-token.js'; +import { ContextTokenization } from './context-tokenization.js'; +import { ContextState } from './context-state.js'; +import { ContextTransition } from './context-transition.js'; class CircularArray { static readonly DEFAULT_ARRAY_SIZE = 5; @@ -314,365 +104,227 @@ class CircularArray { } } -interface ContextMatchResult { - /** - * Represents the current state of the context after applying incoming keystroke data. - */ - state: TrackedContextState; - - /** - * Represents the previously-cached context state that best matches `state` if available. - * May be `null` if no such state could be found within the context-state cache. - */ - baseState: TrackedContextState; - - /** - * Indicates the portion of the incoming keystroke data, if any, that applies to - * tokens before the last pre-caret token and thus should not be replaced by predictions - * based upon `state`. If the provided context state + the incoming transform do not - * adequately match the current context, the match attempt will fail with a `null` result. - * - * Should generally be non-null if the token before the caret did not previously exist. - * - * The result may be null if it does not match the prior context state or if bookkeeping - * based upon it is problematic - say, if wordbreaking effects shift due to new input, - * causing a mismatch with the prior state's tokenization. - * (Refer to #12494 for an example case.) - */ - preservationTransform?: Transform; - - headTokensRemoved: number; - tailTokensAdded: number; -} - -export class ContextTracker extends CircularArray { +export class ContextTracker extends CircularArray { + // Aim: relocate to ContextTransition in some form? + // Or can we split it up in some manner across the different types? static attemptMatchContext( - tokenizedContext: Token[], - matchState: TrackedContextState, - transformSequenceDistribution?: Distribution - ): ContextMatchResult { - // Map the previous tokenized state to an edit-distance friendly version. - let matchContext: string[] = matchState.toRawTokenization(); + context: Context, + lexicalModel: LexicalModel, + matchState: ContextState, + // the distribution should be tokenized already. + transformDistribution?: Distribution // transform distribution is needed here. + ): ContextTransition { + const baseTransition = new ContextTransition(matchState, matchState.appliedInput?.id); + const transformSequenceDistribution = tokenizeAndFilterDistribution(context, lexicalModel, transformDistribution); - // Inverted order, since 'match' existed before our new context. - let mapping = ClassicalDistanceCalculation.computeDistance( - matchContext.map(value => ({key: value})), - tokenizedContext.map(value => ({key: value.text})), - // Must be at least 2, as adding a single whitespace after a token tends - // to add two tokens: one for whitespace, one for the empty token to - // follow it. - 3 - ); + if(transformDistribution?.[0]) { + context = applyTransform(transformDistribution[0].sample, context); + } + const tokenizedContext = determineModelTokenizer(lexicalModel)(context).left; + const alignmentResults = matchState.tokenization.computeAlignment(tokenizedContext.map((token) => token.text)); - let editPath = mapping.editPath(); - const firstMatch = editPath.indexOf('match'); - const lastMatch = getEditPathLastMatch(editPath); - - // Assertion: for a long context, the bulk of the edit path should be a - // continuous block of 'match' entries. If there's anything else in - // the middle, we have a context mismatch. - if(firstMatch) { - for(let i = firstMatch+1; i < lastMatch; i++) { - if(editPath[i] != 'match') { - return null; - } - } + if(!alignmentResults.canAlign) { + return null; } - // If we have a perfect match with a pre-existing context, no mutations have - // happened; just re-use the old context state. - if(firstMatch == 0 && lastMatch == editPath.length - 1) { - return { state: matchState, baseState: matchState, headTokensRemoved: 0, tailTokensAdded: 0 }; - } - - // If mutations HAVE happened, we have work to do. - let state = matchState; - - let priorEdit: typeof editPath[0]; - let poppedTokenCount = 0; - for(let i = 0; i < firstMatch; i++) { - switch(editPath[i]) { - case 'delete': - if(priorEdit && priorEdit != 'delete') { - return null; - } - if(state == matchState) { - state = new TrackedContextState(state); - } - state.popHead(); - poppedTokenCount++; - break; - case 'substitute': - // There's no major need to drop parts of a token being 'slid' out of the context window. - // We'll leave it intact. - break; - default: - // No 'insert' should exist on the leading edge of context when the - // context window slides. - // - // No 'transform' edits should exist within this section, either. - return null; - } - } + const { + leadTokenShift, + matchLength, + tailEditLength, + tailTokenShift + } = alignmentResults; const hasDistribution = transformSequenceDistribution && Array.isArray(transformSequenceDistribution); - // Reset priorEdit for the end-of-context updating loop. - priorEdit = undefined; - - // Used to construct and represent the part of the incoming transform that - // does not land as part of the final token in the resulting context. This - // component should be preserved by any suggestions that get applied. - let preservationTransform: Transform; - let pushedTokenCount = 0; - - // Now to update the end of the context window. - for(let i = lastMatch+1; i < editPath.length; i++) { - const isLastToken = i == editPath.length - 1; - + // If we have a perfect match with a pre-existing context, no mutations have + // happened; just re-use the old context state. + if(tailEditLength == 0 && leadTokenShift == 0 && tailTokenShift == 0) { + baseTransition.finalize(matchState, transformDistribution); + return baseTransition; + } else { // If we didn't get any input, we really should perfectly match // a previous context state. If such a state is out of our cache, // it should simply be rebuilt. if(!hasDistribution) { return null; } - const transformDistIndex = i - (lastMatch + 1); - const tokenDistribution = transformSequenceDistribution.map((entry) => { - const sample = entry.sample[transformDistIndex]; - if(!sample) { - return null; - } - return { - sample, - p: entry.p - }; - }); - - const incomingToken = tokenizedContext[i - poppedTokenCount]; - - // If the tokenized part of the input is a completely empty transform, - // replace it with null. This can happen with our default wordbreaker - // immediately after a whitespace. We don't want to include this - // transform as part of the input when doing correction-search. - let primaryInput = hasDistribution ? tokenDistribution[0]?.sample : null; - - // If the incoming token has text but we have no transform (or 'insert') to match - // it with, abort the matching attempt. We can't match this case well yet. - if(editPath[i] != 'delete') { - if(!incomingToken) { - return null; - } else if(!(primaryInput || editPath[i] == 'insert' ) && incomingToken?.text != '') { - return null; - } - } - - if(primaryInput && primaryInput.insert == "" && primaryInput.deleteLeft == 0 && !primaryInput.deleteRight) { - primaryInput = null; - } - - // If this token's transform component is not part of the final token, - // it's something we'll want to preserve even when applying suggestions - // for the final token. - // - // Note: will need a either a different approach or more specialized - // handling if/when supporting phrase-level (multi-token) suggestions. - if(!isLastToken) { - preservationTransform = preservationTransform && primaryInput ? buildMergedTransform(preservationTransform, primaryInput) : (primaryInput ?? preservationTransform); - } - const isBackspace = primaryInput && TransformUtils.isBackspace(primaryInput); - - switch(editPath[i]) { - case 'substitute': - if(isLastToken) { - state = new TrackedContextState(state); - } - - const sourceToken = matchState.tokens[i]; - state.tokens[i - poppedTokenCount] = sourceToken; - const token = state.tokens[i - poppedTokenCount]; - - // TODO: I'm beginning to believe that searchSpace should (eventually) be tracked - // on the tokens, rather than on the overall 'state'. - // - Reason: phrase-level corrections / predictions would likely need a search-state - // across per potentially-affected token. - // - Shifting the paradigm should be a separate work unit than the - // context-tracker rework currently being done, though. - if(isBackspace) { - token.updateWithBackspace(incomingToken.text, primaryInput.id); - if(isLastToken) { - state.tokens.pop(); // pops `token` - // puts it back in, rebuilding a fresh search-space that uses the rebuilt - // keystroke distribution from updateWithBackspace. - state.pushTail(token); - } - } else { - token.update( - tokenDistribution, - incomingToken.text - ); - - if(isLastToken) { - // Search spaces may not exist during some unit tests; the state - // may not have an associated model during some. - state.searchSpace[0]?.addInput(tokenDistribution); - } - } - - // For this case, we were _likely_ called by - // ModelCompositor.acceptSuggestion(), which would have marked the - // accepted suggestion. - // - // Upon inspection, this doesn't seem entirely ideal. It works for - // the common case, but not for specially crafted keystroke - // transforms. That said, it's also very low impact. Best as I can - // see, this is only really used for debugging info? - if(state != matchState && !isLastToken) { - token.replacementText = incomingToken.text; - } - - break; - case 'insert': - if(priorEdit && priorEdit != 'substitute' && priorEdit != 'match' && priorEdit != 'insert') { - return null; - } - - if(!preservationTransform) { - // Allows for consistent handling of "insert" cases; even if there's no edit - // from a prior token, having a defined transform here indicates that - // a new token has been produced. This serves as a useful conditional flag - // for prediction logic. - preservationTransform = { insert: '', deleteLeft: 0 }; - } - - if(state == matchState) { - state = new TrackedContextState(state); - } - - let pushedToken = new TrackedContextToken(); - pushedToken.raw = incomingToken.text; - - // TODO: assumes that there was no shift in wordbreaking from the - // prior context to the current one. This may actually be a major - // issue for dictionary-based wordbreaking! - // - // If there was such a shift, then we may have extra transforms - // originally on a 'previous' token that got moved into this one! - // - // Suppose we're using a dictionary-based wordbreaker and have - // `butterfl` for our context, which could become butterfly. If the - // next keystroke results in `butterfli`, this would likely be - // tokenized `butter` `fli`. (e.g: `fli` leads to `flight`.) How do - // we know to properly relocate the `f` and `l` transforms? - if(primaryInput) { - pushedToken.transformDistributions = tokenDistribution ? [tokenDistribution] : []; - } else if(incomingToken.text) { - // We have no transform data to match against an inserted token with text; abort! - // Refer to #12494 for an example case; we currently can't map previously-committed - // input transforms to a newly split-off token. - return null; - } - pushedToken.isWhitespace = incomingToken.isWhitespace; - - // Auto-replaces the search space to correspond with the new token. - state.pushTail(pushedToken); - pushedTokenCount++; - break; - case 'match': - // The default (Unicode) wordbreaker returns an empty token after whitespace blocks. - // Adding new whitespace extends the whitespace block but preserves the empty token - // following it. - if(priorEdit == 'substitute' && tokenizedContext[tokenizedContext.length-1].text == '') { - // Keep the blank token as-is; no edit needed! - continue; - } - // else 'fallthrough' / return null - case 'delete': - // While we do keep a cache of recent contexts, logic constraints for handling - // multitaps makes it tricky to reliably use in all situations. - // It's best to handle `delete` cases directly for this reason. - for(let j = i + 1; j < editPath.length; j++) { - // If something _other_ than delete follows a 'delete' on the edit path, - // we probably have a context mismatch. - // - // It's possible to construct cases where this isn't true, but it's likely not - // worth trying to handle such rare cases. - if(editPath[j] != 'delete') { - return null; - } - } - - // If ALL that remains are deletes, we're good to go. - // - // This may not be the token at the index, but since all that remains are deletes, - // we'll have deleted the correct total number from the end once all iterations - // are done. - if(state == matchState) { - state = new TrackedContextState(state); - } - - state.tokens.pop(); - break; - default: - // No 'transform' edits should exist within this section. - return null; - } - - priorEdit = editPath[i]; } - return { - state, - baseState: matchState, - preservationTransform, - headTokensRemoved: poppedTokenCount, - tailTokensAdded: pushedTokenCount - }; - } + // If mutations HAVE happened, we have work to do. + const tokenization = matchState.tokenization.tokens.map((token) => new ContextToken(token)); - static modelContextState( - tokenizedContext: Token[], - lexicalModel: LexicalModel - ): TrackedContextState { - let baseTokens = tokenizedContext.map(function(entry) { - let token = new TrackedContextToken(); - token.raw = entry.text; - if(entry.isWhitespace) { - token.isWhitespace = true; - } + if(leadTokenShift < 0) { + tokenization.splice(0, -leadTokenShift); + } else if(leadTokenShift > 0) { + // TODO: insert token(s) at the start to match the text that's back within the + // sliding context window. + // + // (was not part of original `attemptContextMatch`) + return null; + } - if(token.raw) { - token.transformDistributions = textToCharTransforms(token.raw).map(function(transform) { - return [{sample: transform, p: 1.0}]; - }); - } else { - // Helps model context-final wordbreaks. - token.transformDistributions = []; - } - return token; + // If no TAIL mutations have happened, we're safe to return now. + if(tailEditLength == 0 && tailTokenShift == 0) { + const state = new ContextState(context, lexicalModel); + state.tokenization = new ContextTokenization(tokenization, alignmentResults); + baseTransition.finalize(state, transformDistribution); + return baseTransition; + } + + // *** + + // first non-matched tail index within the incoming context + const incomingTailUpdateIndex = matchLength + (leadTokenShift > 0 ? leadTokenShift : 0); + // first non-matched tail index in `matchState`, the base context state. + const matchingTailUpdateIndex = matchLength - (leadTokenShift < 0 ? leadTokenShift : 0); + + // The assumed input from the input distribution is always at index 0. + const tokenizedPrimaryInput = hasDistribution ? transformSequenceDistribution[0].sample : null; + // first index: original sample's tokenization + // second index: token index within original sample + const tokenDistribution = transformSequenceDistribution.map((entry) => { + return entry.sample.map((sample) => { + return { + sample: sample, + p: entry.p + } + }); }); - // And now build the final context state object, which includes whitespace 'tokens'. - let state = new TrackedContextState(lexicalModel); + // // Gets distribution of token index 1s as excerpted from the sequences' distribution. + // let a = tokenDistribution.map((sequence) => sequence[1]); - while(baseTokens.length > 0) { - // We don't have a pre-existing distribution for this token, so we'll build one as - // if we'd just produced the token from a backspace. - if(baseTokens.length == 1) { - baseTokens[0].updateWithBackspace(baseTokens[0].raw, null); + // Using these as base indices... + + let tailIndex = 0; + // let lastTailIndex = tailEditLength + (tailTokenShift > 0 ? tailTokenShift : 0); + + // Used to construct and represent the part of the incoming transform that + // does not land as part of the final token in the resulting context. This + // component should be preserved by any suggestions that get applied. + let preservationTransform: Transform; + + for(let i = 0; i < tailEditLength; i++) { + // do tail edits + const incomingIndex = i + incomingTailUpdateIndex; + const matchingIndex = i + matchingTailUpdateIndex; + + const incomingToken = tokenizedContext[incomingIndex]; + const matchedToken = matchState.tokenization.tokens[matchingIndex]; + + let primaryInput = hasDistribution ? tokenizedPrimaryInput[i] : null; + const isBackspace = primaryInput && TransformUtils.isBackspace(primaryInput); + + const isLastToken = incomingIndex == tokenizedContext.length - 1; + + if(isLastToken) { + // If this token's transform component is not part of the final token, + // it's something we'll want to preserve even when applying suggestions + // for the final token. + // + // Note: will need a either a different approach or more specialized + // handling if/when supporting phrase-level (multi-token) suggestions. + } else { + preservationTransform = preservationTransform && primaryInput ? buildMergedTransform(preservationTransform, primaryInput) : (primaryInput ?? preservationTransform); } - state.pushTail(baseTokens.splice(0, 1)[0]); + let token: ContextToken; + + if(isBackspace) { + token = new ContextToken(lexicalModel, incomingToken.text); + token.searchSpace.inputSequence.forEach((entry) => entry[0].sample.id = primaryInput.id); + } else { + // Assumption: there have been no intervening keystrokes since the last well-aligned context. + // (May not be valid with epic/dict-breaker or with complex, word-boundary crossing transforms) + token = new ContextToken(matchedToken); + token.searchSpace.addInput(tokenDistribution.map((seq) => seq[tailIndex])); + } + + tokenization[incomingIndex] = token; + tailIndex++; } - if(state.tokens.length == 0) { - let token = new TrackedContextToken(); - token.raw = ''; + if(tailTokenShift < 0) { + // delete tail tokens + for(let i = 0; i > tailTokenShift; i--) { + // If ALL that remains are deletes, we're good to go. + // + // This may not be the token at the index, but since all that remains are deletes, + // we'll have deleted the correct total number from the end once all iterations + // are done. + tokenization.pop(); + } + } else { + for(let i = tailEditLength; i < tailEditLength + tailTokenShift; i++) { + // create tail tokens + const incomingIndex = i + incomingTailUpdateIndex; + const incomingToken = tokenizedContext[incomingIndex]; + // // Assertion: there should be no matching token; this should be a newly-appended token. + // const matchingIndex = i + tailEditLength + matchingTailUpdateIndex; - state.pushTail(token); + const primaryInput = hasDistribution ? tokenizedPrimaryInput[i] : null; + + if(!preservationTransform) { + // Allows for consistent handling of "insert" cases; even if there's no edit + // from a prior token, having a defined transform here indicates that + // a new token has been produced. This serves as a useful conditional flag + // for prediction logic. + preservationTransform = { insert: '', deleteLeft: 0 }; + } + + const isLastToken = incomingIndex == tokenizedContext.length - 1; + if(!isLastToken) { + preservationTransform = preservationTransform && primaryInput ? buildMergedTransform(preservationTransform, primaryInput) : (primaryInput ?? preservationTransform); + } + + let pushedToken = new ContextToken(lexicalModel); + + // TODO: assumes that there was no shift in wordbreaking from the + // prior context to the current one. This may actually be a major + // issue for dictionary-based wordbreaking! + // + // If there was such a shift, then we may have extra transforms + // originally on a 'previous' token that got moved into this one! + // + // Suppose we're using a dictionary-based wordbreaker and have + // `butterfl` for our context, which could become butterfly. If the + // next keystroke results in `butterfli`, this would likely be + // tokenized `butter` `fli`. (e.g: `fli` leads to `flight`.) How do + // we know to properly relocate the `f` and `l` transforms? + let tokenDistribComponent = tokenDistribution.map((seq) => { + const entry = seq[tailIndex]; + if(!entry || TransformUtils.isEmpty(entry.sample)) { + return null; + } else { + return entry; + } + }).filter((entry) => !!entry); + if(primaryInput) { + let transformDistribution = tokenDistribComponent.length > 0 ? tokenDistribComponent : null; + if(transformDistribution) { + pushedToken.searchSpace.addInput(transformDistribution); + } + } else if(incomingToken.text) { + // We have no transform data to match against an inserted token with text; abort! + // Refer to #12494 for an example case; we currently can't map previously-committed + // input transforms to a newly split-off token. + return null; + } + pushedToken.isWhitespace = incomingToken.isWhitespace; + + // Auto-replaces the search space to correspond with the new token. + tokenization.push(pushedToken); + + tailIndex++; + } } - return state; + const state = new ContextState(context, lexicalModel); + state.tokenization = new ContextTokenization(tokenization, alignmentResults); + baseTransition.finalize(state, transformDistribution, preservationTransform); + return baseTransition; } + // Aim: relocate to ContextState in some form... or ContextTransition? /** * Compares the current, post-input context against the most recently-seen contexts from previous prediction calls, returning * the most information-rich `TrackedContextState` possible. If a match is found, the state will be annotated with the @@ -688,42 +340,31 @@ export class ContextTracker extends CircularArray { context: Context, transformDistribution?: Distribution, preserveMatchState?: boolean - ): ContextMatchResult { + ): ContextTransition { if(!model.traverseFromRoot) { // Assumption: LexicalModel provides a valid traverseFromRoot function. (Is technically optional) // Without it, no 'corrections' may be made; the model can only be used to predict, not correct. throw "This lexical model does not provide adequate data for correction algorithms and context reuse"; } - let tokenize = determineModelTokenizer(model); - - const inputTransform = transformDistribution?.[0]; - let transformTokenLength = 0; - let tokenizedDistribution: Distribution = null; - if(inputTransform) { - // These two methods apply transforms internally; do not mutate context here. - // This particularly matters for the 'distribution' variant. - transformTokenLength = tokenizeTransform(tokenize, context, inputTransform.sample).length; - tokenizedDistribution = tokenizeTransformDistribution(tokenize, context, transformDistribution); - - // Now we update the context used for context-state management based upon our input. - context = applyTransform(inputTransform.sample, context); - - // While we lack phrase-based / phrase-oriented prediction support, we'll just extract the - // set that matches the token length that results from our input. - tokenizedDistribution = tokenizedDistribution.filter((entry) => entry.sample.length == transformTokenLength); + if(transformDistribution?.length == 0) { + transformDistribution = null; } - const tokenizedContext = tokenize(context); + const inputTransform = transformDistribution?.[0]; + const postContext = inputTransform ? applyTransform(inputTransform.sample, context) : context; - if(tokenizedContext.left.length > 0) { + const tokenize = determineModelTokenizer(model); + const tokenizedPostContext = tokenize(postContext) + + if(tokenizedPostContext.left.length > 0) { for(let i = this.count - 1; i >= 0; i--) { const priorMatchState = this.item(i); // Skip intermediate multitap-produced contexts. // When multitapping, we skip all contexts from prior taps within the same interaction, // but not any contexts from before the multitap started. - const priorTaggedContext = priorMatchState.taggedContext; + const priorTaggedContext = priorMatchState.context; if(priorTaggedContext && transformDistribution && transformDistribution.length > 0) { // Using the potential `matchState` + the incoming transform, do the results line up for // our observed context? If not, skip it. @@ -733,27 +374,26 @@ export class ContextTracker extends CircularArray { // // `priorTaggedContext` must not be `null`! const doublecheckContext = applyTransform(transformDistribution[0].sample, priorTaggedContext); - if(doublecheckContext.left != context.left) { + if(doublecheckContext.left != postContext.left) { continue; } - } else if(priorTaggedContext?.left != context.left) { + } else if(priorTaggedContext?.left != postContext.left) { continue; } - let result = ContextTracker.attemptMatchContext(tokenizedContext.left, this.item(i), tokenizedDistribution); + let result = ContextTracker.attemptMatchContext(context, model, this.item(i), transformDistribution); - if(result?.state) { + if(result?.final) { // Keep it reasonably current! And it's probably fine to have it more than once // in the history. However, if it's the most current already, there's no need // to refresh it. - if(this.newest != result.state && this.newest != priorMatchState) { + if(this.newest != result.final && this.newest != priorMatchState) { // Already has a taggedContext. this.enqueue(priorMatchState); } - result.state.taggedContext = context; - if(result.state != this.item(i)) { - this.enqueue(result.state); + if(result.final != this.item(i)) { + this.enqueue(result.final); } return result; } @@ -765,10 +405,13 @@ export class ContextTracker extends CircularArray { // // Assumption: as a caret needs to move to context before any actual transform distributions occur, // this state is only reached on caret moves; thus, transformDistribution is actually just a single null transform. - let state = ContextTracker.modelContextState(tokenizedContext.left, model); - state.taggedContext = context; + let state = new ContextState(context, model); this.enqueue(state); - return { state, baseState: null, headTokensRemoved: 0, tailTokensAdded: 0 }; + const transition = new ContextTransition(state, /* TODO: we need a clear value here in the future! */ null); + // Hacky, but holds the course for now. This should only really happen from context resets, which can + // then use a different path. + transition.finalize(state, []); + return transition; } clearCache() { diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts new file mode 100644 index 0000000000..01f6288e0a --- /dev/null +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts @@ -0,0 +1,115 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2025-07-30 + * + * Represents cached data about a single context transition event, as well + * as the state of the context both before and after the transition. + */ + +import { LexicalModelTypes } from '@keymanapp/common-types'; + +import { ContextState } from './context-state.js'; + +import Distribution = LexicalModelTypes.Distribution; +import Transform = LexicalModelTypes.Transform; + +/** + * Represents the transition between two context states as triggered + * by input keystrokes or applied suggestions. + */ +export class ContextTransition { + /** + * Represents the state of the context before the transition event occurred. + */ + readonly base: ContextState; + private _final: ContextState; + + /** + * Indicates the fat-finger distribution for the incoming keystroke related to + * the context transition event. + */ + inputDistribution?: Distribution; + + // The transform ID in play. + private _transitionId?: number; + + /** + * Indicates the portion of the incoming keystroke data, if any, that applies to + * tokens before the last pre-caret token and thus should not be replaced by predictions + * based upon `state`. If the provided context state + the incoming transform do not + * adequately match the current context, the match attempt will fail with a `null` result. + * + * Should generally be non-null if the token before the caret did not previously exist. + * + * The result may be null if it does not match the prior context state or if bookkeeping + * based upon it is problematic - say, if wordbreaking effects shift due to new input, + * causing a mismatch with the prior state's tokenization. + * (Refer to #12494 for an example case.) + */ + preservationTransform?: Transform; + + /** + * Constructs a partial context transition object for use during the process + * of analyzing context transitions or for representing the base state of a + * reset context. + * @param context The base state for the represented context transition + * @param transitionId The unique ID corresponding to the transition event + * or context state. + */ + constructor(context: ContextState, transitionId: number); + /** + * Deep-copies a ContextTransition instance. + * @param baseTransition + */ + constructor(baseTransition: ContextTransition); + constructor(param: ContextState | ContextTransition, transitionId?: number) { + if(!(param instanceof ContextTransition)) { + const contextState = param; + // We're initializing a ContextTransition from a blank or reset context. + this.base = contextState; + this._final = null; + this._transitionId = transitionId; + } else { + const baseTransition = param; + Object.assign(this, baseTransition); + + // These need to be deep-copied. + this.base = new ContextState(baseTransition.base); + this._final = new ContextState(baseTransition._final); + } + } + + /** + * Gets the context state resulting from the context transition event, + * including any generated suggestions and data regarding potential + * application thereof. + */ + get final(): ContextState { + return this._final; + } + + /** + * The unique ID corresponding to the transition event or context state. + */ + get transitionId(): number { + return this._transitionId; + } + + /** + * Records the context state resulting from the context transition generated + * by a keystroke. + * @param state The context state to record as the result of the transition + * @param inputDistribution Fat-finger data corresponding to the triggering keystroke + * @param preservationTransform Portions of the most likely input that do not contribute to the final token + * in the final context's tokenization. + */ + finalize(state: ContextState, inputDistribution: Distribution, preservationTransform?: Transform) { + this._final = state; + this.inputDistribution = inputDistribution; + // Long-term, this should never be null... but we need to allow it at this point + // in the refactoring process. + this._transitionId = inputDistribution?.find((entry) => entry.sample.id !== undefined)?.sample.id; + this.preservationTransform = preservationTransform; + } +} \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts index 6404d2eb8f..f28f05b754 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts @@ -411,7 +411,7 @@ export class SearchSpace { private tierOrdering: SearchSpaceTier[] = []; private selectionQueue: PriorityQueue; - private inputSequence: Distribution[] = []; + private _inputSequence: Distribution[] = []; private minInputCost: number[] = []; private rootNode: SearchNode; @@ -455,7 +455,7 @@ export class SearchSpace { this.buildQueueSpaceComparator(); if(arg1 instanceof SearchSpace) { - this.inputSequence = [].concat(arg1.inputSequence); + this._inputSequence = [].concat(arg1._inputSequence); this.minInputCost = [].concat(arg1.minInputCost); this.rootNode = arg1.rootNode; // Re-use already-checked Nodes. @@ -470,9 +470,9 @@ export class SearchSpace { const model = arg1; if(!model) { - throw "The LexicalModel parameter must not be null / undefined."; + throw new Error("The LexicalModel parameter must not be null / undefined."); } else if(!model.traverseFromRoot) { - throw "The provided model does not implement the `traverseFromRoot` function, which is needed to support robust correction searching."; + throw new Error("The provided model does not implement the `traverseFromRoot` function, which is needed to support robust correction searching."); } this.selectionQueue = new PriorityQueue(this.QUEUE_SPACE_COMPARATOR); @@ -534,6 +534,13 @@ export class SearchSpace { } } + /** + * Retrieves the sequence of inputs + */ + public get inputSequence() { + return [...this._inputSequence]; + } + increaseMaxEditDistance() { this.tierOrdering.forEach(function(tier) { tier.increaseMaxEditDistance() }); } @@ -541,7 +548,7 @@ export class SearchSpace { get correctionsEnabled() { // When corrections are disabled, the Web engine will only provide individual Transforms // for an input, not a distribution. No distributions means we shouldn't do corrections. - return !!this.inputSequence.find((distribution) => distribution.length > 1); + return !!this._inputSequence.find((distribution) => distribution.length > 1); } /** @@ -551,7 +558,7 @@ export class SearchSpace { * just the raw keystroke if corrections are disabled) */ addInput(inputDistribution: Distribution) { - this.inputSequence.push(inputDistribution); + this._inputSequence.push(inputDistribution); // Assumes that `inputDistribution` is already sorted. this.minInputCost.push(-Math.log(inputDistribution[0].p)); @@ -683,9 +690,9 @@ export class SearchSpace { let deletionEdges: SearchNode[] = []; if(!substitutionsOnly) { - deletionEdges = currentNode.buildDeletionEdges(this.inputSequence[inputIndex-1]); + deletionEdges = currentNode.buildDeletionEdges(this._inputSequence[inputIndex-1]); } - let substitutionEdges = currentNode.buildSubstitutionEdges(this.inputSequence[inputIndex-1]); + let substitutionEdges = currentNode.buildSubstitutionEdges(this._inputSequence[inputIndex-1]); // Note: we're live-modifying the tier's cost here! The priority queue loses its guarantees as a result. nextTier.correctionQueue.enqueueAll(deletionEdges.concat(substitutionEdges)); diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/transform-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/transform-tokenization.ts index 291245014c..62c157ee24 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/transform-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/transform-tokenization.ts @@ -1,8 +1,10 @@ import { LexicalModelTypes } from '@keymanapp/common-types'; import Context = LexicalModelTypes.Context; import Distribution = LexicalModelTypes.Distribution; +import LexicalModel = LexicalModelTypes.LexicalModel; import Transform = LexicalModelTypes.Transform; import { applyTransform, type Tokenization } from "@keymanapp/models-templates"; +import { determineModelTokenizer } from '#./model-helpers.js'; /** * Determines a tokenization-aware sequence of (`Transform`) edits, one per @@ -85,4 +87,54 @@ export function tokenizeTransformDistribution( p: transform.p }; }); +} + +/** + * Given an incoming distribution of Transforms, this method applies + * `tokenizeTransform` for each, mapping each transform to its tokenized form in + * the returned distribution. + * + * It then filters out all incoming Transforms that do not result in the same + * number of tokens as the "primary input" when applied, as the context-tracker + * and predictive-text engine cannot handle word-breaking divergence well at + * this time. + * @param context + * @param model + * @param transformDistribution + * @returns + */ +export function tokenizeAndFilterDistribution( + context: Context, + model: LexicalModel, + transformDistribution?: Distribution +) { + let tokenize = determineModelTokenizer(model); + const inputTransform = transformDistribution?.[0]; + + let transformTokenLength = 0; + let tokenizedDistribution: Distribution = null; + if(inputTransform) { + // These two methods apply transforms internally; do not mutate context here. + // This particularly matters for the 'distribution' variant. + + // What if a pre-whitespace token has a final substitution as PART of an edit? + // Say, ['apple', ' ', ''] => ['apply', ' ', 'n'] + // For now... we can't really handle that case well - modeling the 'e' => 'y' part. + // Will likely require improvements to tokenizeTransform(), which doesn't yet handle + // deleteLeft tokenization for transforms spanning tokens & whitespace. + // + // See: #14361. + // There's a good shot attemptTokenizedAlignment would be useful for it. + transformTokenLength = tokenizeTransform(tokenize, context, inputTransform.sample).length; + tokenizedDistribution = tokenizeTransformDistribution(tokenize, context, transformDistribution); + + // Now we update the context used for context-state management based upon our input. + context = applyTransform(inputTransform.sample, context); + + // While we lack phrase-based / phrase-oriented prediction support, we'll just extract the + // set that matches the token length that results from our input. + tokenizedDistribution = tokenizedDistribution.filter((entry) => entry.sample.length == transformTokenLength); + } + + return tokenizedDistribution; } \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts index 2a5a4ef60b..8c6fcc9a3a 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts @@ -191,12 +191,7 @@ export class ModelCompositor { // Store the suggestions on the final token of the current context state (if it exists). // Or, once phrase-level suggestions are possible, on whichever token serves as each prediction's root. if(postContextState) { - postContextState.tail.replacements = suggestions.map(function(suggestion) { - return { - suggestion: suggestion, - tokenWidth: 1 - } - }); + postContextState.tokenization.tail.suggestions = suggestions; } return suggestions; @@ -204,7 +199,7 @@ export class ModelCompositor { acceptSuggestion(suggestion: Suggestion, context: Context, postTransform?: Transform): Reversion { // Step 1: generate and save the reversion's Transform. - let sourceTransform = suggestion.transform; + let sourceTransform = models.buildMergedTransform(suggestion.transform, suggestion.appendedTransform ?? { insert: '', deleteLeft: 0}); let deletedLeftChars = KMWString.substr(context.left, -sourceTransform.deleteLeft, sourceTransform.deleteLeft); let insertedLength = KMWString.length(sourceTransform.insert); @@ -260,11 +255,14 @@ export class ModelCompositor { if(this.contextTracker) { let contextState = this.contextTracker.newest; if(!contextState) { - contextState = this.contextTracker.analyzeState(this.lexicalModel, context).state; + contextState = this.contextTracker.analyzeState(this.lexicalModel, context).final; } - contextState.tail.activeReplacementId = suggestion.id; + contextState.tokenization.tail.appliedSuggestionId = suggestion.id; let acceptedContext = models.applyTransform(suggestion.transform, context); + if(suggestion.appendedTransform) { + acceptedContext = models.applyTransform(suggestion.appendedTransform, acceptedContext); + } this.contextTracker.analyzeState(this.lexicalModel, acceptedContext); } @@ -299,7 +297,7 @@ export class ModelCompositor { for(let c = this.contextTracker.count - 1; c >= 0; c--) { let contextState = this.contextTracker.item(c); - if(contextState.tail.activeReplacementId == -reversion.id) { + if(contextState.tokenization.tail.appliedSuggestionId == -reversion.id) { contextMatchFound = true; break; } @@ -310,18 +308,16 @@ export class ModelCompositor { } // Remove all contexts more recent than the one we're reverting to. - while(this.contextTracker.newest.tail.activeReplacementId != -reversion.id) { + while(this.contextTracker.newest.tokenization.tail.appliedSuggestionId != -reversion.id) { this.contextTracker.popNewest(); } - this.contextTracker.newest.tail.activeReplacementId = -1; + this.contextTracker.newest.tokenization.tail.appliedSuggestionId = undefined; // Will need to be modified a bit if/when phrase-level suggestions are implemented. // Those will be tracked on the first token of the phrase, which won't be the tail // if they cover multiple tokens. - let suggestions = this.contextTracker.newest.tail.replacements.map(function(trackedSuggestion) { - return trackedSuggestion.suggestion; - }); + let suggestions = this.contextTracker.newest.tokenization.tail.suggestions; suggestions.forEach(function(suggestion) { // A reversion's transform ID is the additive inverse of its original suggestion; diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 8bd8dda359..55576c24dd 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -1,12 +1,17 @@ import * as models from '@keymanapp/models-templates'; import { KMWString } from '@keymanapp/web-utils'; import { LexicalModelTypes } from '@keymanapp/common-types'; +import { defaultWordbreaker, WordBreakProperty } from '@keymanapp/models-wordbreakers'; import TransformUtils from './transformUtils.js'; import { determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js'; -import { ContextTracker, TrackedContextState } from './correction/context-tracker.js'; +import { ContextTracker } from './correction/context-tracker.js'; +import { ContextState } from './correction/context-state.js'; import { ExecutionTimer } from './correction/execution-timer.js'; import ModelCompositor from './model-compositor.js'; + +const searchForProperty = defaultWordbreaker.searchForProperty; + import CasingForm = LexicalModelTypes.CasingForm; import Context = LexicalModelTypes.Context; import Distribution = LexicalModelTypes.Distribution; @@ -165,7 +170,7 @@ export async function correctAndEnumerate( * * Otherwise, is `null`. */ - postContextState?: TrackedContextState; + postContextState?: ContextState; /** * The suggestions generated based on the user's input state. @@ -236,11 +241,11 @@ export async function correctAndEnumerate( // facilitates a more thorough correction-search pattern. // Token replacement benefits greatly from knowledge of the prior context state. - let { state: contextState } = contextTracker.analyzeState( + let contextState = contextTracker.analyzeState( lexicalModel, context, null - ); + ).final; // Corrections and predictions are based upon the post-context state, though. const contextChangeAnalysis = contextTracker.analyzeState( @@ -250,7 +255,7 @@ export async function correctAndEnumerate( ? transformDistribution : null ); - const postContextState = contextChangeAnalysis.state; + const postContextState = contextChangeAnalysis.final; // TODO: Should we filter backspaces & whitespaces out of the transform distribution? // Ideally, the answer (in the future) will be no, but leaving it in right now may pose an issue. @@ -259,7 +264,7 @@ export async function correctAndEnumerate( // let's just note that right now, there will only ever be one. // // The 'eventual' logic will be significantly more complex, though still manageable. - let searchSpace = postContextState.searchSpace[0]; + const searchSpace = postContextState.tokenization.tail.searchSpace; // No matter the prediction, once we know the root of the prediction, we'll always 'replace' the // same amount of text. We can handle this before the big 'prediction root' loop. @@ -267,9 +272,9 @@ export async function correctAndEnumerate( // The amount of text to 'replace' depends upon whatever sort of context change occurs // from the received input. - const postContextTokens = postContextState.tokens; + const postContextTokens = postContextState.tokenization.tokens; // Only use of `contextState`. - let contextLengthDelta = postContextTokens.length - contextState.tokens.length; + let contextLengthDelta = postContextTokens.length - contextState.tokenization.tokens.length; // If the context now has more tokens, the token we'll be 'predicting' didn't originally exist. if(contextChangeAnalysis.preservationTransform) { // As the word/token being corrected/predicted didn't originally exist, there's no @@ -307,11 +312,11 @@ export async function correctAndEnumerate( // Did the wordbreaker (or similar) append a blank token before the caret? If so, // preserve that by preventing corrections from triggering left-deletion. - if(tailToken.raw == '') { + if(tailToken.exampleInput == '') { deleteLeft = 0; } - const isTokenStart = tailToken.transformDistributions.length <= 1; + const isTokenStart = tailToken.searchSpace.inputSequence.length <= 1; // TODO: whitespace, backspace filtering. Do it here. // Whitespace is probably fine, actually. Less sure about backspace. @@ -700,6 +705,35 @@ export function processSimilarity( }); } +/** + * This function may be used to prevent auto-selection/auto-correct from applying in + * unexpected ways. For example, when typing numbers in English, we don't expect + * '5' to auto-correct to '5th' just because there are no pure-number entries in + * the lexicon rooted on '5'. + * @param correction + * @returns + */ +export function correctionValidForAutoSelect(correction: string) { + let chars = [...correction]; + + // If the _correction_ - the actual, existing text - does not include any letters, + // then predictions built upon it should not be considered valid for auto-correction. + for(let c of chars) { + // Found even one letter? We'll consider it valid. + switch(searchForProperty(c.codePointAt(0))) { + case WordBreakProperty.ALetter: + case WordBreakProperty.Hebrew_Letter: + case WordBreakProperty.Katakana: + return true; + default: + } + } + + // Only reached when the correction has nothing that passes as a letter in-context. + // (MidLet and MidNumLet only count when there are adjacent letters.) + return false; +} + export function predictionAutoSelect(suggestionDistribution: CorrectionPredictionTuple[]) { if(suggestionDistribution.length == 0) { return; @@ -718,6 +752,11 @@ export function predictionAutoSelect(suggestionDistribution: CorrectionPredictio suggestionDistribution = suggestionDistribution.slice(1); if(suggestionDistribution.length == 1) { + // Prevent auto-acceptance when the root doesn't meet validation criteria. + if(!correctionValidForAutoSelect(suggestionDistribution[0].correction.sample)) { + return; + } + // Mark for auto-acceptance; there are no alternatives. suggestionDistribution[0].prediction.sample.autoAccept = true; return; @@ -761,6 +800,10 @@ export function predictionAutoSelect(suggestionDistribution: CorrectionPredictio return; } + if(!correctionValidForAutoSelect(bestSuggestion.correction.sample)) { + return; + } + // compare correction-cost aspects? We disable if the base correction is lower than best, // but should we do other comparisons too? @@ -829,32 +872,42 @@ export function finalizeSuggestions( } }); - // Apply 'after word' punctuation and other post-processing, setting suggestion IDs. - // We delay until now so that utility functions relying on the unmodified Transform may execute properly. - suggestions.forEach((suggestion) => { - // Valid 'keep' suggestions may have zero length; we still need to evaluate the following code - // for such cases. + if(punctuation.insertAfterWord !== "") { + // Apply 'after word' punctuation and other post-processing, setting suggestion IDs. + // We delay until now so that utility functions relying on the unmodified Transform may execute properly. + suggestions.forEach((suggestion) => { + // Valid 'keep' suggestions may have zero length; we still need to evaluate the following code + // for such cases. - // If we're mid-word, delete its original post-caret text. - const tokenization = tokenize(context); - if(tokenization && tokenization.caretSplitsToken) { - // While we wait on the ability to provide a more 'ideal' solution, let's at least - // go with a more stable, if slightly less ideal, solution for now. - // - // A predictive text default (on iOS, at least) - immediately wordbreak - // on suggestions accepted mid-word. - suggestion.transform.insert += punctuation.insertAfterWord; + // If we're mid-word, delete its original post-caret text. + const tokenization = tokenize(context); + if(tokenization && tokenization.caretSplitsToken) { + // While we wait on the ability to provide a more 'ideal' solution, let's at least + // go with a more stable, if slightly less ideal, solution for now. + // + // A predictive text default (on iOS, at least) - immediately wordbreak + // on suggestions accepted mid-word. + suggestion.appendedTransform = { + insert: punctuation.insertAfterWord, + deleteLeft: 0 + }; - // Do we need to manipulate the suggestion's transform based on the current state of the context? - } else if(!context.right) { - suggestion.transform.insert += punctuation.insertAfterWord; - } else if(punctuation.insertAfterWord != '') { - if(context.right.indexOf(punctuation.insertAfterWord) != 0) { - suggestion.transform.insert += punctuation.insertAfterWord; + // Do we need to manipulate the suggestion's transform based on the current state of the context? + } else if(!context.right) { + suggestion.appendedTransform = { + insert: punctuation.insertAfterWord, + deleteLeft: 0 + }; + } else if(punctuation.insertAfterWord != '') { + if(context.right.indexOf(punctuation.insertAfterWord) != 0) { + suggestion.appendedTransform = { + insert: punctuation.insertAfterWord, + deleteLeft: 0 + }; + } } - } - - }); + }); + }; return suggestions; } @@ -898,6 +951,10 @@ export function toAnnotatedSuggestion( tag: annotationType, }; + if(suggestion.appendedTransform) { + result.appendedTransform = suggestion.appendedTransform; + } + if(suggestion.transformId !== undefined) { result.transformId = suggestion.transformId; } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts b/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts index 51001ef69d..9448191650 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts @@ -1,5 +1,10 @@ export { ClassicalDistanceCalculation } from './correction/classical-calculation.js'; +export { ContextState } from './correction/context-state.js'; +export { ContextToken } from './correction/context-token.js'; +export { ContextTokenization } from './correction/context-tokenization.js'; export { ContextTracker } from './correction/context-tracker.js'; +export { EditOperation } from './correction/classical-calculation.js'; +export * from './correction/alignment-helpers.js'; export * as correction from './correction/index.js'; export * from './model-helpers.js'; export * as models from './models/index.js'; diff --git a/web/src/test/auto/headless/engine/interfaces/prediction/predictionContext.tests.js b/web/src/test/auto/headless/engine/interfaces/prediction/predictionContext.tests.js index 6ec7a0bbd9..cf8473c65c 100644 --- a/web/src/test/auto/headless/engine/interfaces/prediction/predictionContext.tests.js +++ b/web/src/test/auto/headless/engine/interfaces/prediction/predictionContext.tests.js @@ -159,21 +159,14 @@ describe("PredictionContext", () => { assert.equal(updateFake.callCount, 3); suggestions = updateFake.thirdCall.args[0]; - // Note: this unit test was originally written with auto-correct on! - // #11941 was written 2024-07-25 (added unit test for auto-correction method) - // #12169 was written 2024-08-14, which is what added THIS unit test. - // This does re-use the apply-revert oriented mocking. // Should skip the (second) "apple", "apply", "apps" round, as it became outdated // by its following request before its response could be received. - assert.deepEqual(suggestions.map((obj) => obj.displayAs), ['applied']); // '“apple”' included with auto-correct enabled. - // Is not displayed; we only display it if auto-correct is on, as 'applied' would be automatic then. - assert.equal(predictiveContext.keepSuggestion.displayAs, '“apple”'); - // assert.equal(suggestions.find((obj) => obj.tag == 'keep').displayAs, '“apple”'); // with auto-correct enabled. - assert.equal(suggestions.find((obj) => obj.transform.deleteLeft != 0).displayAs, 'applied'); + assert.deepEqual(suggestions.map((obj) => obj.displayAs), ['“apple”', 'applied']); + assert.equal(suggestions.find((obj) => obj.tag == 'keep').displayAs, '“apple”'); // Our reused mocking doesn't directly provide the 'keep' suggestion; we // need to remove it before testing for set equality. - assert.deepEqual(suggestions /*.splice(1)*/, expected); + assert.deepEqual(suggestions.splice(1), expected); }); it('sendUpdateState retrieves the most recent suggestion set', async function() { diff --git a/web/src/test/auto/headless/engine/main/headless/languageProcessor.tests.js b/web/src/test/auto/headless/engine/main/headless/languageProcessor.tests.js index 6d866ef8ed..a02f1b7474 100644 --- a/web/src/test/auto/headless/engine/main/headless/languageProcessor.tests.js +++ b/web/src/test/auto/headless/engine/main/headless/languageProcessor.tests.js @@ -117,9 +117,11 @@ describe('LanguageProcessor', function() { languageProcessor.predict(transcription).then(function(suggestions) { assert.isOk(suggestions); assert.equal(suggestions[0].displayAs, '«li»'); - assert.equal(suggestions[0].transform.insert, 'li '); + assert.equal(suggestions[0].transform.insert, 'li'); + assert.equal(suggestions[0].appendedTransform.insert, ' '); assert.equal(suggestions[1].displayAs, 'like'); - assert.equal(suggestions[1].transform.insert, 'like '); + assert.equal(suggestions[1].transform.insert, 'like'); + assert.equal(suggestions[1].appendedTransform.insert, ' '); done(); }).catch(done); }).catch(function() { @@ -154,7 +156,8 @@ describe('LanguageProcessor', function() { languageProcessor.predict(transcription).then(function(suggestions) { assert.isOk(suggestions); assert.equal(suggestions[1].displayAs, 'like'); - assert.equal(suggestions[1].transform.insert, 'like '); + assert.equal(suggestions[1].transform.insert, 'like'); + assert.equal(suggestions[1].appendedTransform.insert, ' '); done(); }).catch(done); }).catch(function() { @@ -172,7 +175,8 @@ describe('LanguageProcessor', function() { // The source suggestion is simply 'like'. assert.isOk(suggestions); assert.equal(suggestions[1].displayAs, 'like'); - assert.equal(suggestions[1].transform.insert, 'like '); + assert.equal(suggestions[1].transform.insert, 'like'); + assert.equal(suggestions[1].appendedTransform.insert, ' '); done(); }).catch(done); }).catch(function() { @@ -189,7 +193,8 @@ describe('LanguageProcessor', function() { languageProcessor.predict(transcription).then(function(suggestions) { assert.isOk(suggestions); assert.equal(suggestions[1].displayAs, 'I'); - assert.equal(suggestions[1].transform.insert, 'I '); + assert.equal(suggestions[1].transform.insert, 'I'); + assert.equal(suggestions[1].appendedTransform.insert, ' '); done(); }).catch(done); }).catch(function() { @@ -209,7 +214,8 @@ describe('LanguageProcessor', function() { // The source suggestion is simply 'like'. assert.isOk(suggestions); assert.equal(suggestions[1].displayAs, 'LIKE'); - assert.equal(suggestions[1].transform.insert, 'LIKE '); + assert.equal(suggestions[1].transform.insert, 'LIKE'); + assert.equal(suggestions[1].appendedTransform.insert, ' '); done(); }).catch(done); }).catch(function() { @@ -226,7 +232,8 @@ describe('LanguageProcessor', function() { languageProcessor.predict(transcription).then(function(suggestions) { assert.isOk(suggestions); assert.equal(suggestions[0].displayAs, 'I'); - assert.equal(suggestions[0].transform.insert, 'I '); + assert.equal(suggestions[0].transform.insert, 'I'); + assert.equal(suggestions[0].appendedTransform.insert, ' '); done(); }).catch(done); }).catch(function() { @@ -247,7 +254,8 @@ describe('LanguageProcessor', function() { // The source suggestion is simply 'like'. assert.isOk(suggestions); assert.equal(suggestions[1].displayAs, 'Like'); - assert.equal(suggestions[1].transform.insert, 'Like '); + assert.equal(suggestions[1].transform.insert, 'Like'); + assert.equal(suggestions[1].appendedTransform.insert, ' '); done(); }).catch(done); }).catch(function() { @@ -267,7 +275,8 @@ describe('LanguageProcessor', function() { // The source suggestion is simply 'like'. assert.isOk(suggestions); assert.equal(suggestions[1].displayAs, 'Like'); - assert.equal(suggestions[1].transform.insert, 'Like '); + assert.equal(suggestions[1].transform.insert, 'Like'); + assert.equal(suggestions[1].appendedTransform.insert, ' '); done(); }).catch(done); }).catch(function() { diff --git a/web/src/test/auto/headless/engine/predictive-text/templates/common.tests.ts b/web/src/test/auto/headless/engine/predictive-text/templates/common.tests.ts index dfd5c875cb..5abd27cd75 100644 --- a/web/src/test/auto/headless/engine/predictive-text/templates/common.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/templates/common.tests.ts @@ -22,8 +22,7 @@ describe('Common utility functions', function() { let final = { insert: 'applebanana', - deleteLeft: 0, - deleteRight: 0 + deleteLeft: 0 }; let mergedTransform = models.buildMergedTransform(apple, banana); @@ -43,8 +42,7 @@ describe('Common utility functions', function() { let final = { insert: 'applebanana', - deleteLeft: 2, - deleteRight: 0 + deleteLeft: 2 }; let mergedTransform = models.buildMergedTransform(apple, banana); @@ -64,8 +62,7 @@ describe('Common utility functions', function() { let final = { insert: 'bananapple', // the 'apple' transform removes the final 'a' from 'banana'. - deleteLeft: 0, - deleteRight: 0 + deleteLeft: 0 }; let mergedTransform = models.buildMergedTransform(banana, apple); @@ -85,8 +82,7 @@ describe('Common utility functions', function() { let final = { insert: 'bananapple', // the 'apple' transform removes the final 'a' from 'banana'. - deleteLeft: 2, - deleteRight: 0 + deleteLeft: 2 }; let mergedTransform = models.buildMergedTransform(banana, apple); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/alignment-helpers.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/alignment-helpers.tests.ts new file mode 100644 index 0000000000..2c9d7d1992 --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/alignment-helpers.tests.ts @@ -0,0 +1,105 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2025-07-30 + * + * This file contains low-level tests designed to validate helper functions + * used when aligning cached context states to incoming contexts and when + * validating potential substitution edit operations. + */ + +import { assert } from 'chai'; +import { EditOperation, getEditPathLastMatch, isSubstitutionAlignable } from '@keymanapp/lm-worker/test-index'; + +describe('getEditPathLastMatch', () => { + it('returns the last match when no substitutions exist', () => { + const path: EditOperation[] = ['delete', 'delete', 'match', 'match', 'match', 'match', 'insert']; + assert.equal(getEditPathLastMatch(path), path.lastIndexOf('match')); + }); + + it('returns the last match when no substitutions exist left of a "match"', () => { + const path: EditOperation[] = ['delete', 'delete', 'match', 'match', 'match', 'match', 'substitute', 'insert']; + assert.equal(getEditPathLastMatch(path), path.lastIndexOf('match')); + }); + + // is intended to handle application of suggestions. + it('returns the second-to-last match when a substitution exists before final "match"', () => { + // limitation: if there is _anything_ after that last match, the first assertion will fail. + const path: EditOperation[] = ['delete', 'delete', 'match', 'match', 'match', 'substitute', 'match']; + assert.notEqual(getEditPathLastMatch(path), path.lastIndexOf('match')); + assert.equal(getEditPathLastMatch(path), path.lastIndexOf('match', path.lastIndexOf('match')-1)); + }); +}); + +describe('isSubstitutionAlignable', () => { + it(`returns true: 'ca' => 'can'`, () => { + assert.isTrue(isSubstitutionAlignable('can', 'ca')); + }); + + // Leading word in context window starts sliding out of said window. + it(`returns true: 'can' => 'an'`, () => { + assert.isTrue(isSubstitutionAlignable('an', 'can')); + }); + + // Same edits on both sides: not valid. + it(`returns false: 'apple' => 'grapples'`, () => { + assert.isFalse(isSubstitutionAlignable('grapples', 'apple')); + }); + + // Edits on one side: valid. + it(`returns true: 'apple' => 'grapple'`, () => { + assert.isTrue(isSubstitutionAlignable('grapple', 'apple')); + }); + + // Edits on one side: valid. + it(`returns true: 'apple' => 'grapple'`, () => { + assert.isTrue(isSubstitutionAlignable('apples', 'apple')); + }); + + // Same edits on both sides: not valid. + it(`returns false: 'grapples' => 'apple'`, () => { + assert.isFalse(isSubstitutionAlignable('apple', 'grapples')); + }); + + // Substitution: not valid when not permitted via parameter. + it(`returns false: 'apple' => 'banana'`, () => { + // edit path: 'insert' ('b' of banana), 'match' (on leading a), rest are 'substitute'. + assert.isFalse(isSubstitutionAlignable('banana', 'apple')); + }); + + // Substitution: not valid if too much is substituted, even if allowed via parameter. + it(`returns false: 'apple' => 'banana' (subs allowed)`, () => { + // edit path: 'insert' ('b' of banana), 'match' (on leading a), rest are 'substitute'. + // 1 match vs 4 substitute = no bueno. It'd require too niche of a keyboard rule. + assert.isFalse(isSubstitutionAlignable('banana', 'apple', true)); + }); + + it(`returns true: 'a' => 'à' (subs allowed)`, () => { + assert.isTrue(isSubstitutionAlignable('à', 'a', true)); + }); + + // Leading substitution: valid if enough of the remaining word matches. + // Could totally happen from a legit Keyman keyboard rule. + it(`returns true: 'can' => 'van' (subs allowed)`, () => { + assert.isTrue(isSubstitutionAlignable('van', 'can', true)); + }); + + // Trailing substitution: invalid if not allowed. + it(`returns false: 'can' => 'cap' (subs not allowed)`, () => { + assert.isFalse(isSubstitutionAlignable('cap', 'can')); + }); + + // Trailing substitution: valid. + it(`returns false: 'can' => 'cap' (subs allowed)`, () => { + assert.isTrue(isSubstitutionAlignable('cap', 'can', true)); + }); + + it(`returns true: 'clasts' => 'clasps' (subs allowed)`, () => { + assert.isTrue(isSubstitutionAlignable('clasps', 'clasts', true)); + }); + + // random deletion at the start + later substitution = still permitted + it(`returns false: 'clasts' => 'lasps' (subs allowed)`, () => { + assert.isTrue(isSubstitutionAlignable('lasps', 'clasts', true)); + }); +}); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts new file mode 100644 index 0000000000..b229118783 --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts @@ -0,0 +1,60 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2025-08-01 + * + * This file tests designed to validate the behavior of ContextState class and + * its integration with the lower-level classes that it utilizes. + */ + +import { assert } from 'chai'; + +import { default as defaultBreaker } from '@keymanapp/models-wordbreakers'; +import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; + +import { ContextState, models } from '@keymanapp/lm-worker/test-index'; + +import TrieModel = models.TrieModel; + +var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'), + {wordBreaker: defaultBreaker}); + +describe('ContextState', () => { + it('', () => { + let context = { left: '', right: '', startOfBuffer: true, endOfBuffer: true }; + let state = new ContextState(context, plainModel); + + assert.equal(state.context, context); + assert.equal(state.model, plainModel); + assert.isOk(state.tokenization); + assert.isUndefined(state.isManuallyApplied); + assert.isNotOk(state.suggestions); + assert.isNotOk(state.appliedSuggestionId); + }); + + describe('initializing without prior tokenization', () => { + it('creates one empty token for an empty context', () => { + let context = { left: '', right: '', startOfBuffer: true, endOfBuffer: true }; + let state = new ContextState(context, plainModel); + assert.isOk(state.tokenization); + assert.equal(state.tokenization.tokens.length, 1); + assert.equal(state.tokenization.tail.exampleInput, ''); + }); + + it('creates tokens for initial text (without ending whitespace)', () => { + let context = { left: 'the quick brown fox', right: '', startOfBuffer: true, endOfBuffer: true }; + let state = new ContextState(context, plainModel); + assert.isOk(state.tokenization); + assert.equal(state.tokenization.tokens.length, 7); + assert.deepEqual(state.tokenization.exampleInput, ['the', ' ', 'quick', ' ', 'brown', ' ', 'fox']); + }); + + it('creates tokens for initial text (with extra empty token for ending whitespace)', () => { + let context = { left: 'the quick brown fox ', right: '', startOfBuffer: true, endOfBuffer: true }; + let state = new ContextState(context, plainModel); + assert.isOk(state.tokenization); + assert.equal(state.tokenization.tokens.length, 9); + assert.deepEqual(state.tokenization.exampleInput, ['the', ' ', 'quick', ' ', 'brown', ' ', 'fox', ' ', '']); + }); + }); +}); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts new file mode 100644 index 0000000000..ae21e14432 --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts @@ -0,0 +1,109 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2025-07-30 + * + * This file contains low-level unit tests designed to validate the behavior + * of the ContextToken class. + */ + +import { assert } from 'chai'; + +// Aliased due to JS keyword. +import { default as defaultBreaker } from '@keymanapp/models-wordbreakers'; +import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; + +import { ContextToken, correction, models } from '@keymanapp/lm-worker/test-index'; + +import ExecutionTimer = correction.ExecutionTimer; +import TrieModel = models.TrieModel; + +var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'), + {wordBreaker: defaultBreaker}); + +describe('ContextToken', function() { + describe("", () => { + it("(model: LexicalModel)", async () => { + let token = new ContextToken(plainModel); + + assert.isEmpty(token.searchSpace.inputSequence); + assert.isEmpty(token.exampleInput); + assert.isFalse(token.isWhitespace); + assert.isEmpty(token.suggestions); + assert.isUndefined(token.appliedSuggestionId); + + // While searchSpace has no inputs, it _can_ match lexicon entries (via insertions). + let searchIterator = token.searchSpace.getBestMatches(new ExecutionTimer(Number.POSITIVE_INFINITY, Number.POSITIVE_INFINITY)); + let firstEntry = await searchIterator.next(); + assert.isFalse(firstEntry.done); + }); + + it("(model: LexicalModel, text: string)", () => { + let token = new ContextToken(plainModel, "and"); + + assert.isNotEmpty(token.searchSpace.inputSequence); + + assert.equal(token.searchSpace.inputSequence.map((entry) => entry[0].sample.insert).join(''), 'and'); + token.searchSpace.inputSequence.forEach((entry) => assert.equal(entry[0].sample.deleteLeft, 0)); + assert.deepEqual(token.searchSpace.inputSequence, [..."and"].map((char) => { + return [{ + sample: { + insert: char, + deleteLeft: 0 + }, + p: 1.0 + }]; + })); + assert.equal(token.exampleInput, 'and'); + + assert.isFalse(token.isWhitespace); + + // Is only set with a different value later, outside the constructor. + assert.isEmpty(token.suggestions); + assert.isUndefined(token.appliedSuggestionId); + }); + + it("(token: ContextToken", () => { + // Same as in a test above, since we verified that it works correctly. + let baseToken = new ContextToken(plainModel, "and"); + baseToken.suggestions = [ + { + transform: { + insert: 'd ', + deleteLeft: 0 + }, + id: 37, + transformId: 1, + displayAs: '"and"', + tag: 'keep', + autoAccept: true + }, + { + transform: { + insert: 'Andes ', + deleteLeft: 2 + }, + id: 38, + transformId: 1, + displayAs: 'Andes', + } + ] + baseToken.appliedSuggestionId = 37; + + let clonedToken = new ContextToken(baseToken); + + assert.notEqual(clonedToken.suggestions, baseToken.suggestions); + assert.deepEqual(clonedToken.suggestions, baseToken.suggestions); + + assert.notEqual(clonedToken.searchSpace, baseToken.searchSpace); + // Deep equality on .searchSpace can't be directly checked due to the internal complexities involved. + // We CAN check for the most important members, though. + assert.notEqual(clonedToken.searchSpace.inputSequence, baseToken.searchSpace.inputSequence); + assert.deepEqual(clonedToken.searchSpace.inputSequence, baseToken.searchSpace.inputSequence); + + assert.notEqual(clonedToken, baseToken); + // Perfectly deep-equal when we ignore .searchSpace. + assert.deepEqual({...clonedToken, searchSpace: null}, {...baseToken, searchSpace: null}); + }); + }); +}); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts new file mode 100644 index 0000000000..67146d9f2a --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -0,0 +1,456 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2025-07-30 + * + * This file contains low-level tests designed to validate the behavior of the + * of the ContextTokenization class and its integration with the lower-level + * classes that it utilizes. + */ + +import { assert } from 'chai'; + +import { default as defaultBreaker } from '@keymanapp/models-wordbreakers'; +import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; + +import { ContextToken, ContextTokenization, models } from '@keymanapp/lm-worker/test-index'; + +import TrieModel = models.TrieModel; + +var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'), + {wordBreaker: defaultBreaker}); + +function buildBaseTokenization(textTokens: string[]) { + const tokens = textTokens.map((entry) => new ContextToken(plainModel, entry)); + return new ContextTokenization(tokens); +} + +function toToken(text: string) { + let isWhitespace = text == ' '; + let token = new ContextToken(plainModel, text); + token.isWhitespace = isWhitespace; + return token; +} + +describe('ContextTokenization', function() { + describe("", () => { + it("constructs from just a token array", () => { + const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; + let tokenization = new ContextTokenization(rawTextTokens.map((text => toToken(text)))); + assert.deepEqual(tokenization.tokens.map((entry) => entry.exampleInput), rawTextTokens); + assert.deepEqual(tokenization.tokens.map((entry) => entry.isWhitespace), rawTextTokens.map((entry) => entry == ' ')); + assert.isNotOk(tokenization.alignment); + assert.equal(tokenization.tail.exampleInput, 'day'); + assert.isFalse(tokenization.tail.isWhitespace); + assert.isUndefined(tokenization.tail.appliedSuggestionId); + }); + + it("constructs from a token array + alignment data", () => { + const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; + let alignment = { + canAlign: true, + leadTokenShift: 0, + matchLength: 6, + tailEditLength: 1, + tailTokenShift: 0 + }; + + let tokenization = new ContextTokenization(rawTextTokens.map((text => toToken(text))), alignment); + + assert.deepEqual(tokenization.tokens.map((entry) => entry.exampleInput), rawTextTokens); + assert.deepEqual(tokenization.tokens.map((entry) => entry.isWhitespace), rawTextTokens.map((entry) => entry == ' ')); + assert.isOk(tokenization.alignment); + assert.deepEqual(tokenization.alignment, alignment); + assert.equal(tokenization.tail.exampleInput, 'day'); + assert.isFalse(tokenization.tail.isWhitespace); + assert.isUndefined(tokenization.tail.appliedSuggestionId); + }); + + it('clones', () => { + const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; + + let baseTokenization = new ContextTokenization(rawTextTokens.map((text => toToken(text))), { + canAlign: true, + leadTokenShift: 0, + matchLength: 6, + tailEditLength: 1, + tailTokenShift: 0 + }); + + let cloned = new ContextTokenization(baseTokenization); + + assert.notDeepEqual(cloned, baseTokenization); + assert.deepEqual(cloned.tokens.map((token) => token.searchSpace.inputSequence), + baseTokenization.tokens.map((token) => token.searchSpace.inputSequence)); + + // The `.searchSpace` instances will not be deep-equal; there are class properties + // that hold functions with closures, configured at runtime. + + // @ts-ignore - TS2704 b/c deleting a readonly property. + baseTokenization.tokens.forEach((token) => delete token.searchSpace); + // @ts-ignore - TS2704 b/c deleting a readonly property. + cloned.tokens.forEach((token) => delete token.searchSpace); + + assert.deepEqual(cloned, baseTokenization); + }); + }); + + it('exampleInput', () => { + const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; + let tokenization = new ContextTokenization(rawTextTokens.map((text => toToken(text)))); + + assert.deepEqual(tokenization.exampleInput, rawTextTokens); + }); + + describe('computeAlignment', () => { + it("properly matches and aligns when contexts match", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + const newContext = [...baseContext]; + + const baseTokenization = buildBaseTokenization(baseContext); + const computedAlignment = baseTokenization.computeAlignment(newContext); + + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: 0, + matchLength: 5, + tailEditLength: 0, + tailTokenShift: 0 + }); + }); + + it("detects unalignable contexts - no matching tokens", () => { + const baseContext = [ + 'swift', 'tan', 'wolf', 'leaped', 'across' + ]; + const newContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + + const baseTokenization = buildBaseTokenization(baseContext); + const computedAlignment = baseTokenization.computeAlignment(newContext); + + assert.deepEqual(computedAlignment, {canAlign: false}); + }); + + it("detects unalignable contexts - too many mismatching tokens", () => { + const baseContext = [ + 'swift', 'tan', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + + const baseTokenization = buildBaseTokenization(baseContext); + const computedAlignment = baseTokenization.computeAlignment(newContext); + + assert.deepEqual(computedAlignment, {canAlign: false}); + }); + + it("fails alignment for leading-edge word substitutions", () => { + const baseContext = [ + 'swift', 'brown', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + + const baseTokenization = buildBaseTokenization(baseContext); + const computedAlignment = baseTokenization.computeAlignment(newContext); + + assert.deepEqual(computedAlignment, {canAlign: false}); + }); + + it("fails alignment for small leading-edge word substitutions", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'sick', 'brown', 'fox', 'jumped', 'over' + ]; + + const baseTokenization = buildBaseTokenization(baseContext); + const computedAlignment = baseTokenization.computeAlignment(newContext); + + assert.deepEqual(computedAlignment, {canAlign: false}); + }); + + it("properly matches and aligns when lead token is modified", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'uick', 'brown', 'fox', 'jumped', 'over' + ]; + + const baseTokenization = buildBaseTokenization(baseContext); + const computedAlignment = baseTokenization.computeAlignment(newContext); + + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: 0, + matchLength: 5, + tailEditLength: 0, + tailTokenShift: 0 + }); + }); + + it("properly matches and aligns when lead token is removed", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'brown', 'fox', 'jumped', 'over' + ]; + + const baseTokenization = buildBaseTokenization(baseContext); + const computedAlignment = baseTokenization.computeAlignment(newContext); + + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: -1, + matchLength: 4, + tailEditLength: 0, + tailTokenShift: 0 + }); + }); + + it("properly matches and aligns when lead token is added", () => { + const baseContext = [ + 'brown', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + + const baseTokenization = buildBaseTokenization(baseContext); + const computedAlignment = baseTokenization.computeAlignment(newContext); + + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: 1, + matchLength: 4, + tailEditLength: 0, + tailTokenShift: 0 + }); + }); + + it("properly matches and aligns when lead tokens are removed and modified", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'ox', 'jumped', 'over' + ]; + + const baseTokenization = buildBaseTokenization(baseContext); + const computedAlignment = baseTokenization.computeAlignment(newContext); + + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: -2, + matchLength: 3, + tailEditLength: 0, + tailTokenShift: 0 + }); + }); + + it("properly matches and aligns when lead tokens are added and modified", () => { + const baseContext = [ + 'rown', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + + const baseTokenization = buildBaseTokenization(baseContext); + const computedAlignment = baseTokenization.computeAlignment(newContext); + + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: 1, + matchLength: 4, + tailEditLength: 0, + tailTokenShift: 0 + }); + }); + + it("properly matches and aligns when lead token is removed and tail token is added", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'brown', 'fox', 'jumped', 'over', 'the' + ]; + + const baseTokenization = buildBaseTokenization(baseContext); + const computedAlignment = baseTokenization.computeAlignment(newContext); + + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: -1, + matchLength: 4, + tailEditLength: 0, + tailTokenShift: 1 + }); + }); + + it("properly matches and aligns when lead token and tail token are modified", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'ove' + ]; + const newContext = [ + 'uick', 'brown', 'fox', 'jumped', 'over' + ]; + + const baseTokenization = buildBaseTokenization(baseContext); + const computedAlignment = baseTokenization.computeAlignment(newContext); + + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: 0, + matchLength: 4, // we treat 'quick' and 'uick' as the same + tailEditLength: 1, + tailTokenShift: 0 + }); + }); + + it("properly matches and aligns when lead token and tail token are modified + new token appended", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'ove' + ]; + const newContext = [ + 'uick', 'brown', 'fox', 'jumped', 'over', 't' + ]; + + const baseTokenization = buildBaseTokenization(baseContext); + const computedAlignment = baseTokenization.computeAlignment(newContext); + + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: 0, + matchLength: 4, // we treat 'quick' and 'uick' as the same + tailEditLength: 1, + tailTokenShift: 1 + }); + }); + + it("properly handles context window sliding backward", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'e', 'quick', 'brown', 'fox', 'jumped', 'ove' + ]; + + const baseTokenization = buildBaseTokenization(baseContext); + const computedAlignment = baseTokenization.computeAlignment(newContext); + + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: 1, + matchLength: 4, // we treat 'quick' and 'uick' as the same + tailEditLength: 1, + tailTokenShift: 0 + }); + }); + + it("properly handles context window sliding far backward", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'the', 'quick', 'brown', 'fox', 'jumped' + ]; + + const baseTokenization = buildBaseTokenization(baseContext); + const computedAlignment = baseTokenization.computeAlignment(newContext); + + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: 1, + matchLength: 4, // we treat 'quick' and 'uick' as the same + tailEditLength: 0, + tailTokenShift: -1 + }); + }); + + it("properly handles context window sliding farther backward", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'the', 'quick', 'brown', 'fox', 'jumpe' + ]; + + const baseTokenization = buildBaseTokenization(baseContext); + const computedAlignment = baseTokenization.computeAlignment(newContext); + + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: 1, + matchLength: 3, // we treat 'quick' and 'uick' as the same + tailEditLength: 1, + tailTokenShift: -1 + }); + }); + + it("fails alignment for mid-head deletion", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'quick', 'fox', 'jumped', 'over' + ]; + + const baseTokenization = buildBaseTokenization(baseContext); + const computedAlignment = baseTokenization.computeAlignment(newContext); + + assert.deepEqual(computedAlignment, {canAlign: false}); + }); + + it("fails alignment for mid-head insertion", () => { + const baseContext = [ + 'quick', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + + const baseTokenization = buildBaseTokenization(baseContext); + const computedAlignment = baseTokenization.computeAlignment(newContext); + + assert.deepEqual(computedAlignment, {canAlign: false}); + }); + + it("fails alignment for mid-tail deletion", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'quick', 'brown', 'fox', 'over' + ]; + + const baseTokenization = buildBaseTokenization(baseContext); + const computedAlignment = baseTokenization.computeAlignment(newContext); + + assert.deepEqual(computedAlignment, {canAlign: false}); + }); + + it("fails alignment for mid-tail insertion", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'quick', 'brown', 'fox', 'jumped', 'far', 'over' + ]; + + const baseTokenization = buildBaseTokenization(baseContext); + const computedAlignment = baseTokenization.computeAlignment(newContext); + + assert.deepEqual(computedAlignment, {canAlign: false}); + }); + }); +}); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts index 22171c92a1..f9d649153f 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts @@ -1,12 +1,21 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2025-07-30 + * + * This file contains tests designed to validate the context-caching + * and context-tracking components for the Keyman predictive-text worker. + */ + import { assert } from 'chai'; import { default as defaultBreaker } from '@keymanapp/models-wordbreakers'; -import { deepCopy } from '@keymanapp/web-utils'; import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; import { LexicalModelTypes } from '@keymanapp/common-types'; -import { ContextTracker, determineModelTokenizer, ModelCompositor, models, tokenizeTransformDistribution } from '@keymanapp/lm-worker/test-index'; +import { ContextState, ContextTracker, ModelCompositor, models } from '@keymanapp/lm-worker/test-index'; +import Suggestion = LexicalModelTypes.Suggestion; import Transform = LexicalModelTypes.Transform; import TrieModel = models.TrieModel; @@ -14,265 +23,316 @@ const plainModel = new TrieModel(jsonFixture('models/tries/english-1000'), {wordBreaker: defaultBreaker} ); -const tokenizer = determineModelTokenizer(new models.DummyModel({wordbreaker: defaultBreaker})); - describe('ContextTracker', function() { - function toWrapperDistribution(transforms: Transform | Transform[]) { - transforms = Array.isArray(transforms) ? transforms : [transforms]; + function toWrapperDistribution(transform: Transform) { return [{ - sample: transforms, + sample: transform, p: 1.0 }]; } describe('attemptMatchContext', function() { it("properly matches and aligns when lead token is removed", function() { - let existingContext = models.tokenize(defaultBreaker, { - left: "an apple a day keeps the doctor" - }); + let existingContext = { + left: "an apple a day keeps the doctor", + startOfBuffer: true, + endOfBuffer: true + }; let transform: Transform = { insert: '', deleteLeft: 0 - } - let newContext = deepCopy(existingContext); - newContext.left.splice(0, 1); + }; + let newContext = { + left: " apple a day keeps the doctor", + startOfBuffer: true, + endOfBuffer: true + }; let rawTokens = [" ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"]; - let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel); - let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform)); - assert.isNotNull(newContextMatch?.state); - assert.deepEqual(newContextMatch.state.tokens.map(token => token.raw), rawTokens); - assert.equal(newContextMatch.headTokensRemoved, 1); - assert.equal(newContextMatch.tailTokensAdded, 0); + let baseState = new ContextState(existingContext, plainModel); + let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform)); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + + if(!newContextMatch.final.tokenization.alignment.canAlign) { + // Done this way b/c TS can infer types correctly afterward. + assert.fail("context alignment failed"); + } + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, -1); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0); }); it("properly matches and aligns when lead token + following whitespace are removed", function() { - let existingContext = models.tokenize(defaultBreaker, { - left: "an apple a day keeps the doctor" - }); + let existingContext = { + left: "an apple a day keeps the doctor", + startOfBuffer: true, + endOfBuffer: true + }; let transform = { insert: '', deleteLeft: 0 - } - let newContext = deepCopy(existingContext); - newContext.left.splice(0, 2); + }; + let newContext = { + left: "apple a day keeps the doctor", + startOfBuffer: true, + endOfBuffer: true + }; let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"]; - let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel); - let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform)); - assert.isNotNull(newContextMatch?.state); - assert.deepEqual(newContextMatch.state.tokens.map(token => token.raw), rawTokens); - assert.equal(newContextMatch.headTokensRemoved, 2); - assert.equal(newContextMatch.tailTokensAdded, 0); + let baseState = new ContextState(existingContext, plainModel); + let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform)); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + + if(!newContextMatch.final.tokenization.alignment.canAlign) { + // Done this way b/c TS can infer types correctly afterward. + assert.fail("context alignment failed"); + } + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, -2); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0); }); it("properly matches and aligns when final token is edited", function() { - let existingContext = models.tokenize(defaultBreaker, { - left: "an apple a day keeps the docto" - }); + let existingContext = { + left: "an apple a day keeps the docto", + startOfBuffer: true, + endOfBuffer: true + }; let transform: Transform = { insert: 'r', deleteLeft: 0 } - let newContext = models.tokenize(defaultBreaker, { - left: "an apple a day keeps the doctor" - }); let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"]; - let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel); - let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform)); - assert.isNotNull(newContextMatch?.state); - assert.deepEqual(newContextMatch.state.tokens.map(token => token.raw), rawTokens); - assert.equal(newContextMatch.headTokensRemoved, 0); - assert.equal(newContextMatch.tailTokensAdded, 0); + let baseState = new ContextState(existingContext, plainModel); + let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform)); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + + if(!newContextMatch.final.tokenization.alignment.canAlign) { + // Done this way b/c TS can infer types correctly afterward. + assert.fail("context alignment failed"); + } + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0); }); // Needs improved context-state management (due to 2x tokens) it("properly matches and aligns when a 'wordbreak' is added", function() { - let existingContext = models.tokenize(defaultBreaker, { - left: "an apple a day keeps the doctor" - }); + let existingContext = { + left: "an apple a day keeps the doctor", + startOfBuffer: true, + endOfBuffer: true + }; let transform = { insert: ' ', deleteLeft: 0 } - let newContext = models.tokenize(defaultBreaker, { - left: "an apple a day keeps the doctor " - }); let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""]; - let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel); - let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform)); - assert.isNotNull(newContextMatch?.state); - assert.deepEqual(newContextMatch.state.tokens.map(token => token.raw), rawTokens); + let baseState = new ContextState(existingContext, plainModel); + let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform)); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); // We want to preserve the added whitespace when predicting a token that follows after it. assert.deepEqual(newContextMatch.preservationTransform, { insert: ' ', deleteLeft: 0 }); // The 'wordbreak' transform - let state = newContextMatch?.state; - assert.isNotEmpty(state.tokens[state.tokens.length - 2].transformDistributions); - assert.isEmpty(state.tokens[state.tokens.length - 1].transformDistributions); - assert.equal(newContextMatch.headTokensRemoved, 0); - assert.equal(newContextMatch.tailTokensAdded, 2); + let state = newContextMatch?.final; + assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence); + assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence); + + if(!newContextMatch.final.tokenization.alignment.canAlign) { + // Done this way b/c TS can infer types correctly afterward. + assert.fail("context alignment failed"); + } + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 2); }); it("properly matches and aligns when a 'wordbreak' is removed via backspace", function() { - let existingContext = models.tokenize(defaultBreaker, { - left: "an apple a day keeps the doctor " - }); + let existingContext = { + left: "an apple a day keeps the doctor ", + startOfBuffer: true, + endOfBuffer: true + }; let transform = { insert: '', deleteLeft: 1 } - let newContext = models.tokenize(defaultBreaker, { - left: "an apple a day keeps the doctor" - }); let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"]; - let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel); - let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform)); - assert.isOk(newContextMatch?.state); - assert.deepEqual(newContextMatch?.state.tokens.map(token => token.raw), rawTokens); + let baseState = new ContextState(existingContext, plainModel); + let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform)); + assert.isOk(newContextMatch?.final); + assert.deepEqual(newContextMatch?.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); - // The 'wordbreak' transform - assert.equal(newContextMatch.headTokensRemoved, 0); - assert.equal(newContextMatch.tailTokensAdded, 0); + if(!newContextMatch.final.tokenization.alignment.canAlign) { + // Done this way b/c TS can infer types correctly afterward. + assert.fail("context alignment failed"); + } + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, -2); }); it("properly matches and aligns when an implied 'wordbreak' occurs (as when following \"'\")", function() { - let existingContext = models.tokenize(defaultBreaker, { - left: "'" - }); + let existingContext = { + left: "'", + startOfBuffer: true, + endOfBuffer: true + }; let transform = { insert: 'a', deleteLeft: 0 } - let newContext = models.tokenize(defaultBreaker, { - left: "'a" - }); let rawTokens = ["'", "a"]; - let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel); - let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform)); - assert.isNotNull(newContextMatch?.state); - assert.deepEqual(newContextMatch.state.tokens.map(token => token.raw), rawTokens); + let baseState = new ContextState(existingContext, plainModel); + let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform)); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); assert.deepEqual(newContextMatch.preservationTransform, { insert: '', deleteLeft: 0 }); // The 'wordbreak' transform - let state = newContextMatch.state; - assert.isNotEmpty(state.tokens[state.tokens.length - 2].transformDistributions); - assert.isNotEmpty(state.tokens[state.tokens.length - 1].transformDistributions); + let state = newContextMatch.final; + assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence); + assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence); - assert.equal(newContextMatch.headTokensRemoved, 0); - assert.equal(newContextMatch.tailTokensAdded, 1); + if(!newContextMatch.final.tokenization.alignment.canAlign) { + // Done this way b/c TS can infer types correctly afterward. + assert.fail("context alignment failed"); + } + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 1); }) // Needs improved context-state management (due to 2x tokens) it("properly matches and aligns when lead token is removed AND a 'wordbreak' is added'", function() { - let existingContext = models.tokenize(defaultBreaker, { - left: "an apple a day keeps the doctor" - }); + let existingContext = { + left: "an apple a day keeps the doctor", + startOfBuffer: true, + endOfBuffer: true + }; let transform = { insert: ' ', deleteLeft: 0 } - let newContext = models.tokenize(defaultBreaker, { - left: "apple a day keeps the doctor " - }); + let newContext = { + left: "apple a day keeps the doctor ", + startOfBuffer: true, + endOfBuffer: true + }; let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""]; - let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel); - let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform)); - assert.isNotNull(newContextMatch?.state); - assert.deepEqual(newContextMatch.state.tokens.map(token => token.raw), rawTokens); + let baseState = new ContextState(existingContext, plainModel); + let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform)); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); // We want to preserve the added whitespace when predicting a token that follows after it. assert.deepEqual(newContextMatch.preservationTransform, { insert: ' ', deleteLeft: 0 }); // The 'wordbreak' transform - let state = newContextMatch.state; - assert.isNotEmpty(state.tokens[state.tokens.length - 2].transformDistributions); - assert.isEmpty(state.tokens[state.tokens.length - 1].transformDistributions); + let state = newContextMatch.final; + assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence); + assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence); - assert.equal(newContextMatch.headTokensRemoved, 2); - assert.equal(newContextMatch.tailTokensAdded, 2); + if(!newContextMatch.final.tokenization.alignment.canAlign) { + // Done this way b/c TS can infer types correctly afterward. + assert.fail("context alignment failed"); + } + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, -2); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 2); }); it("properly matches and aligns when initial token is modified AND a 'wordbreak' is added'", function() { - let existingContext = models.tokenize(defaultBreaker, { - left: "an" - }); + let existingContext = { + left: "an", + startOfBuffer: true, + endOfBuffer: true + }; let transform = { insert: 'd ', deleteLeft: 0 } - let newContext = models.tokenize(defaultBreaker, { - left: "and " - }); let rawTokens = ["and", " ", ""]; - let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel); + let baseState = new ContextState(existingContext, plainModel); let newContextMatch = ContextTracker.attemptMatchContext( - newContext.left, - baseContextMatch, - tokenizeTransformDistribution(tokenizer, {left: "an", startOfBuffer: true, endOfBuffer: true}, [{sample: transform, p: 1}]) + existingContext, + plainModel, + baseState, + [{sample: transform, p: 1}] ); - assert.isNotNull(newContextMatch?.state); - assert.deepEqual(newContextMatch.state.tokens.map(token => token.raw), rawTokens); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); // We want to preserve all text preceding the new token when applying a suggestion. - assert.deepEqual(newContextMatch.preservationTransform, { insert: 'd ', deleteLeft: 0, deleteRight: 0}); + assert.deepEqual(newContextMatch.preservationTransform, { insert: 'd ', deleteLeft: 0}); // The 'wordbreak' transform - let state = newContextMatch.state; - assert.isNotEmpty(state.tokens[state.tokens.length - 2].transformDistributions); - assert.isEmpty(state.tokens[state.tokens.length - 1].transformDistributions); + let state = newContextMatch.final; + assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence); + assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence); - assert.equal(newContextMatch.headTokensRemoved, 0); - assert.equal(newContextMatch.tailTokensAdded, 2); + if(!newContextMatch.final.tokenization.alignment.canAlign) { + // Done this way b/c TS can infer types correctly afterward. + assert.fail("context alignment failed"); + } + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 2); }); it("properly matches and aligns when tail token is modified AND a 'wordbreak' is added'", function() { - let existingContext = models.tokenize(defaultBreaker, { - left: "apple a day keeps the doc" - }); + let existingContext = { + left: "apple a day keeps the doc", + startOfBuffer: true, + endOfBuffer: true + }; let transform = { insert: 'tor ', deleteLeft: 0 } - let newContext = models.tokenize(defaultBreaker, { - left: "apple a day keeps the doctor " - }); let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""]; - let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel); + let baseState = new ContextState(existingContext, plainModel); let newContextMatch = ContextTracker.attemptMatchContext( - newContext.left, - baseContextMatch, - tokenizeTransformDistribution(tokenizer, {left: "apple a day keeps the doc", startOfBuffer: true, endOfBuffer: true}, [{sample: transform, p: 1}]) + existingContext, + plainModel, + baseState, + [{sample: transform, p: 1}] ); - assert.isNotNull(newContextMatch?.state); - assert.deepEqual(newContextMatch.state.tokens.map(token => token.raw), rawTokens); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); // We want to preserve all text preceding the new token when applying a suggestion. - assert.deepEqual(newContextMatch.preservationTransform, { insert: 'tor ', deleteLeft: 0, deleteRight: 0 }); + assert.deepEqual(newContextMatch.preservationTransform, { insert: 'tor ', deleteLeft: 0 }); // The 'wordbreak' transform - let state = newContextMatch.state; - assert.isNotEmpty(state.tokens[state.tokens.length - 2].transformDistributions); - assert.isEmpty(state.tokens[state.tokens.length - 1].transformDistributions); + let state = newContextMatch.final; + assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence); + assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence); - assert.equal(newContextMatch.headTokensRemoved, 0); - assert.equal(newContextMatch.tailTokensAdded, 2); + if(!newContextMatch.final.tokenization.alignment.canAlign) { + // Done this way b/c TS can infer types correctly afterward. + assert.fail("context alignment failed"); + } + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 2); }); it('rejects hard-to-handle case: tail token is split into three rather than two', function() { let baseContext = models.tokenize(defaultBreaker, { - left: "text'" + left: "text'", + startOfBuffer: true, + endOfBuffer: true }); assert.equal(baseContext.left.length, 1); - let baseContextMatch = ContextTracker.modelContextState(baseContext.left, plainModel); + + let baseState = new ContextState({ left: "text'", startOfBuffer: true, endOfBuffer: true }, plainModel); // Now the actual check. let newContext = models.tokenize(defaultBreaker, { - left: "text'\"" + left: "text'\"", + startOfBuffer: true, + endOfBuffer: true }); // The reason it's a problem - current internal logic isn't prepared to shift // from 1 to 3 tokens in a single step. @@ -283,9 +343,10 @@ describe('ContextTracker', function() { deleteLeft: 0 } let problemContextMatch = ContextTracker.attemptMatchContext( - newContext.left, - baseContextMatch, - tokenizeTransformDistribution(tokenizer, {left: "text'", startOfBuffer: true, endOfBuffer: true}, [{sample: transform, p: 1}]) + {left: "text'", startOfBuffer: true, endOfBuffer: true}, + plainModel, + baseState, + [{sample: transform, p: 1}] ); assert.isNull(problemContextMatch); }); @@ -293,29 +354,27 @@ describe('ContextTracker', function() { describe('modelContextState', function() { it('models without final wordbreak', function() { - let tokenized = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"].map((entry) => { - return { - text: entry, - isWhitespace: entry == " " - }; - }); + let context = { + left: "an apple a day keeps the doctor", + startOfBuffer: true, + endOfBuffer: true + }; let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"]; - let state = ContextTracker.modelContextState(tokenized, plainModel); - assert.deepEqual(state.tokens.map(token => token.raw), rawTokens); + let state = new ContextState(context, plainModel); + assert.deepEqual(state.tokenization.tokens.map(token => token.exampleInput), rawTokens); }); it('models with final wordbreak', function() { - let tokenized = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""].map((entry) => { - return { - text: entry, - isWhitespace: entry == " " - }; - }); + let context = { + left: "an apple a day keeps the doctor ", + startOfBuffer: true, + endOfBuffer: true + }; let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""]; - let state = ContextTracker.modelContextState(tokenized, plainModel); - assert.deepEqual(state.tokens.map(token => token.raw), rawTokens); + let state = new ContextState(context, plainModel); + assert.deepEqual(state.tokenization.tokens.map(token => token.exampleInput), rawTokens); }); }); @@ -327,7 +386,7 @@ describe('ContextTracker', function() { // Needs improved context-state management (due to 2x tokens) it('tracks an accepted suggestion', function() { - let baseSuggestion = { + let baseSuggestion: Suggestion = { transform: { insert: 'world ', deleteLeft: 3, @@ -357,15 +416,12 @@ describe('ContextTracker', function() { let compositor = new ModelCompositor(model); let baseContextMatch = compositor.contextTracker.analyzeState(model, baseContext); - baseContextMatch.state.tail.replacements = [{ - suggestion: baseSuggestion, - tokenWidth: 1 - }]; + baseContextMatch.final.tokenization.tail.suggestions = [ baseSuggestion ]; let reversion = compositor.acceptSuggestion(baseSuggestion, baseContext, postTransform); // Actual test assertion - was the replacement tracked? - assert.equal(baseContextMatch.state.tail.activeReplacementId, baseSuggestion.id); + assert.equal(baseContextMatch.final.tokenization.tail.appliedSuggestionId, baseSuggestion.id); assert.equal(reversion.id, -baseSuggestion.id); // Next step - on the followup context, is the replacement still active? @@ -373,10 +429,10 @@ describe('ContextTracker', function() { let postContextMatch = compositor.contextTracker.analyzeState(model, postContext); // Penultimate token corresponds to whitespace, which does not have a 'raw' representation. - assert.equal(postContextMatch.state.tokens[postContextMatch.state.tokens.length - 2].raw, ' '); + assert.equal(postContextMatch.final.tokenization.tokens[postContextMatch.final.tokenization.tokens.length - 2].exampleInput, ' '); // Final token is empty (follows a wordbreak) - assert.equal(postContextMatch.state.tail.raw, ''); + assert.equal(postContextMatch.final.tokenization.tail.exampleInput, ''); }); }); }); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts index d1ba7b9dee..d32326e843 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts @@ -20,7 +20,7 @@ describe('predictionAutoSelect', () => { const predictions: CorrectionPredictionTuple[] = [ { correction: { - sample: 'apple', // can be null / "mocked out" + sample: 'apple', p: 1 }, prediction: { @@ -47,11 +47,60 @@ describe('predictionAutoSelect', () => { assert.isOk(autoselected); }); + it(`does not select suggestions if the root correction has no letters`, () => { + const predictions: CorrectionPredictionTuple[] = [ + { + correction: { + sample: '5', + p: 1 + }, + prediction: { + sample: { + tag: 'keep', + transform: { + insert: '5', + deleteLeft: 0 + }, + matchesModel: false, + displayAs: '5' + }, + p: 0.01 + }, + totalProb: 0.01 + }, + { + correction: { + sample: '5', + p: 1 + }, + prediction: { + sample: { + transform: { + insert: '5th', + deleteLeft: 0 + }, + matchesModel: true, + displayAs: '5th' + }, + p: 0.8 + }, + totalProb: 0.8 + } + ]; + + const originalPredictions = [...predictions]; + assert.doesNotThrow(() => predictionAutoSelect(predictions)); + assert.sameDeepOrderedMembers(predictions, originalPredictions); + + const autoselected = predictions.find((entry) => entry.prediction.sample.autoAccept); + assert.isNotOk(autoselected); + }); + it(`does not select solitary 'keep' suggestion that doesn't match the model`, () => { const predictions: CorrectionPredictionTuple[] = [ { correction: { - sample: 'appl', // can be null / "mocked out" + sample: 'appl', p: 1 }, prediction: { @@ -81,7 +130,7 @@ describe('predictionAutoSelect', () => { it(`selects 'keep' suggestion that does match the model over any alternatives`, () => { const keepSuggestion: CorrectionPredictionTuple = { correction: { - sample: 'thin', // can be null / "mocked out" + sample: 'thin', p: .8 }, prediction: { @@ -101,7 +150,7 @@ describe('predictionAutoSelect', () => { const highestNonKeepSuggestion: CorrectionPredictionTuple = { correction: { - sample: 'thin', // can be null / "mocked out" + sample: 'thin', p: .8 }, prediction: { @@ -122,7 +171,7 @@ describe('predictionAutoSelect', () => { highestNonKeepSuggestion, { correction: { - sample: 'thin', // can be null / "mocked out" + sample: 'thin', p: .8 }, prediction: { @@ -139,7 +188,7 @@ describe('predictionAutoSelect', () => { }, { correction: { - sample: 'thic', // can be null / "mocked out" + sample: 'thic', p: .2 }, prediction: { @@ -167,7 +216,7 @@ describe('predictionAutoSelect', () => { it(`selects solitary non-'keep' suggestion when 'keep' does not match model`, () => { const keepSuggestion: CorrectionPredictionTuple = { correction: { - sample: 'thin', // can be null / "mocked out" + sample: 'thin', p: .8 }, prediction: { @@ -191,7 +240,7 @@ describe('predictionAutoSelect', () => { // Refer to AUTOSELECT_PROPORTION_THRESHOLD in predict-helpers.ts. const onlyNonKeepSuggestion: CorrectionPredictionTuple = { correction: { - sample: 'thin', // can be null / "mocked out" + sample: 'thin', p: .8 }, prediction: { @@ -228,7 +277,7 @@ describe('predictionAutoSelect', () => { it(`does not select non-'keep' without sufficient winning probability`, () => { const keepSuggestion: CorrectionPredictionTuple = { correction: { - sample: 'thin', // can be null / "mocked out" + sample: 'thin', p: .8 }, prediction: { @@ -252,7 +301,7 @@ describe('predictionAutoSelect', () => { // Refer to AUTOSELECT_PROPORTION_THRESHOLD in predict-helpers.ts. const highestNonKeepSuggestion: CorrectionPredictionTuple = { correction: { - sample: 'thin', // can be null / "mocked out" + sample: 'thin', p: .8 }, prediction: { @@ -273,7 +322,7 @@ describe('predictionAutoSelect', () => { highestNonKeepSuggestion, { correction: { - sample: 'thin', // can be null / "mocked out" + sample: 'thin', p: .8 }, prediction: { @@ -290,7 +339,7 @@ describe('predictionAutoSelect', () => { }, { correction: { - sample: 'thic', // can be null / "mocked out" + sample: 'thic', p: .2 }, prediction: { @@ -323,7 +372,7 @@ describe('predictionAutoSelect', () => { it(`does select non-'keep' with sufficient winning probability`, () => { const keepSuggestion: CorrectionPredictionTuple = { correction: { - sample: 'thin', // can be null / "mocked out" + sample: 'thin', p: .8 }, prediction: { @@ -343,7 +392,7 @@ describe('predictionAutoSelect', () => { const highestNonKeepSuggestion: CorrectionPredictionTuple = { correction: { - sample: 'thin', // can be null / "mocked out" + sample: 'thin', p: .9 }, prediction: { @@ -364,7 +413,7 @@ describe('predictionAutoSelect', () => { highestNonKeepSuggestion, { correction: { - sample: 'thin', // can be null / "mocked out" + sample: 'thin', p: .9 }, prediction: { @@ -381,7 +430,7 @@ describe('predictionAutoSelect', () => { }, { correction: { - sample: 'thic', // can be null / "mocked out" + sample: 'thic', p: .1 }, prediction: { @@ -412,7 +461,7 @@ describe('predictionAutoSelect', () => { it('ignores non key-matched suggestions when key-matched suggestions exist', () => { const keepSuggestion: CorrectionPredictionTuple = { correction: { - sample: 'cant', // can be null / "mocked out" + sample: 'cant', p: 1 }, prediction: { @@ -433,7 +482,7 @@ describe('predictionAutoSelect', () => { const expectedSuggestion: CorrectionPredictionTuple = { correction: { - sample: 'cant', // can be null / "mocked out" + sample: 'cant', p: 1 }, prediction: { @@ -455,7 +504,7 @@ describe('predictionAutoSelect', () => { expectedSuggestion, { correction: { - sample: 'cant', // can be null / "mocked out" + sample: 'cant', p: 1 }, prediction: { @@ -487,7 +536,7 @@ describe('predictionAutoSelect', () => { it('does not auto-select suggestion if its root correction is not most likely', () => { const keepSuggestion: CorrectionPredictionTuple = { correction: { - sample: 'thi', // can be null / "mocked out" + sample: 'thi', p: .7 }, prediction: { @@ -507,7 +556,7 @@ describe('predictionAutoSelect', () => { const highestCorrectionSuggestion: CorrectionPredictionTuple = { correction: { - sample: 'thi', // can be null / "mocked out" + sample: 'thi', p: .7 }, prediction: { @@ -525,7 +574,7 @@ describe('predictionAutoSelect', () => { const highestNonKeepSuggestion: CorrectionPredictionTuple = { correction: { - sample: 'the', // can be null / "mocked out" + sample: 'the', p: .3 }, prediction: { diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-finalization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-finalization.tests.ts index e79af94304..4e63055a10 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-finalization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-finalization.tests.ts @@ -167,7 +167,10 @@ describe('finalizeSuggestions', () => { const { unfinalized, expected } = build_its_is_set(); const finalized = finalizeSuggestions(testModelWithSpacing, unfinalized, context, transform, false); - expected.forEach((entry) => entry.transform.insert += testModelWithSpacing.punctuation.insertAfterWord); + expected.forEach((entry) => entry.appendedTransform = { + insert: testModelWithSpacing.punctuation.insertAfterWord, + deleteLeft: 0 + }); assert.sameDeepOrderedMembers(finalized, expected); }); @@ -216,7 +219,10 @@ describe('finalizeSuggestions', () => { // The character after the caret isn't the whitespace we'd usually insert, // so we don't swallow it this time. - expected.forEach((entry) => entry.transform.insert += testModelWithSpacing.punctuation.insertAfterWord); + expected.forEach((entry) => entry.appendedTransform = { + insert: testModelWithSpacing.punctuation.insertAfterWord, + deleteLeft: 0 + }); assert.sameDeepOrderedMembers(finalized, expected); }); @@ -241,7 +247,10 @@ describe('finalizeSuggestions', () => { const { unfinalized, expected } = build_its_is_set(); const finalized = finalizeSuggestions(testModelWithSpacing, unfinalized, context, transform, false); - expected.forEach((entry) => entry.transform.insert += testModelWithSpacing.punctuation.insertAfterWord); + expected.forEach((entry) => entry.appendedTransform = { + insert: testModelWithSpacing.punctuation.insertAfterWord, + deleteLeft: 0 + }); assert.sameDeepOrderedMembers(finalized, expected); }); }); @@ -340,7 +349,10 @@ describe('finalizeSuggestions', () => { const { unfinalized, expected } = build_its_is_set('verbose'); const finalized = finalizeSuggestions(testModelWithSpacing, unfinalized, context, transform, /* verbose */ true); - expected.forEach((entry) => entry.transform.insert += testModelWithSpacing.punctuation.insertAfterWord); + expected.forEach((entry) => entry.appendedTransform = { + insert: testModelWithSpacing.punctuation.insertAfterWord, + deleteLeft: 0 + }); assert.sameDeepOrderedMembers(finalized, expected); }); @@ -363,7 +375,10 @@ describe('finalizeSuggestions', () => { const { unfinalized, expected } = build_its_is_set(); const finalized = finalizeSuggestions(testModelWithSpacing, unfinalized, context, transform, false); - expected.forEach((entry) => entry.transform.insert += testModelWithSpacing.punctuation.insertAfterWord); + expected.forEach((entry) => entry.appendedTransform = { + insert: testModelWithSpacing.punctuation.insertAfterWord, + deleteLeft: 0 + }); assert.sameDeepOrderedMembers(finalized, expected); }); }); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-custom-punctuation.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-custom-punctuation.tests.ts index dce4924004..9b9ab2c312 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-custom-punctuation.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-custom-punctuation.tests.ts @@ -94,7 +94,7 @@ describe('Custom Punctuation', function () { // Check that it has been changed: for (var i = 0; i < dummySuggestions.length; i++) { - assert.isTrue(suggestions[i].transform.insert.endsWith(' ')); + assert.isTrue(suggestions[i].appendedTransform.insert == ' '); } }); }) diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts index bad4a869d9..9aa6940b3b 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts @@ -96,13 +96,14 @@ describe('ModelCompositor', function() { }); assert.isDefined(keep); - assert.equal(keep.transform.insert, 'the '); + assert.equal(keep.transform.insert, 'the'); + assert.isDefined(keep.appendedTransform?.insert, ' '); // Expect an appended space. - let expectedEntries = ['they ', 'there ', 'their ', 'these ', 'themselves ']; + let expectedEntries = ['they', 'there', 'their', 'these', 'themselves']; expectedEntries.forEach(function(entry) { assert.isDefined(suggestions.find(function(suggestion) { - return suggestion.transform.insert == entry; + return suggestion.transform.insert == entry && suggestion.appendedTransform?.insert == ' '; })); }); }); @@ -813,10 +814,14 @@ describe('ModelCompositor', function() { assert.equal(suggestions.length, 1); let expectedTransform = { - insert: 'hi ', // Keeps current context the same, though it adds a wordbreak. + insert: 'hi', // Keeps current context the same, though it adds a wordbreak. deleteLeft: 2 } assert.deepEqual(suggestions[0].transform, expectedTransform); + assert.deepEqual(suggestions[0].appendedTransform, { + insert: ' ', + deleteLeft: 0 + }); }); it('model with traversals: returns appropriate suggestions upon reversion', async function() { @@ -893,7 +898,7 @@ describe('ModelCompositor', function() { assert.equal(compositor.contextTracker.count, 3); // The replacement should be marked on the context-tracking token. - assert.isOk(suggestionContextState.tail.replacement); + assert.isAtLeast(suggestionContextState.tokenization.tail.appliedSuggestionId, 0); let appliedContext = models.applyTransform(baseSuggestion.transform, baseContext); compositor.applyReversion(reversion, appliedContext); @@ -903,7 +908,7 @@ describe('ModelCompositor', function() { assert.equal(compositor.contextTracker.item(1), suggestionContextState); // The replacement should no longer be marked for the context-tracking token. - assert.isNotOk(suggestionContextState.tail.replacement); + assert.isNotOk(suggestionContextState.tokenization.tail.appliedSuggestionId); }); }); });