From b029b1993b66588c6cacdf27fd7c79a48e9290b6 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 22 Jun 2026 13:52:41 -0500 Subject: [PATCH 01/21] fix(web): do not auto-select model-matching keep suggestions Fixes: #12312 Do, however, prevent auto-selection of any other suggestion when a model-matching 'keep' is available. Build-bot: skip build:web build:android build:ios --- .../worker-thread/src/main/predict-helpers.ts | 8 +++++--- .../engine/src/interfaces/prediction/predictionContext.ts | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 9739adba85..2f480e7535 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -965,9 +965,11 @@ export function predictionAutoSelect(suggestionDistribution: CorrectionPredictio const keepOption = suggestionDistribution[0].prediction.sample as Outcome; if(keepOption.tag == 'keep' && keepOption.matchesModel) { - // Auto-select it for auto-acceptance; we don't correct away from perfectly-valid - // lexical entries, even if they are comparatively low-frequency. - keepOption.autoAccept = true; + // Do not auto-select 'keep' suggestions'; there's no need to apply them. + // + // Do, however, block auto-selection of any other suggestions if we would + // have auto-selected the 'keep'; even if it is comparatively unlikely / + // low-frequency, we 'keep' the current context. return; } else if(suggestionDistribution.length == 1) { return; diff --git a/web/src/engine/src/interfaces/prediction/predictionContext.ts b/web/src/engine/src/interfaces/prediction/predictionContext.ts index 97182a9bf1..b374f504fb 100644 --- a/web/src/engine/src/interfaces/prediction/predictionContext.ts +++ b/web/src/engine/src/interfaces/prediction/predictionContext.ts @@ -266,7 +266,7 @@ export class PredictionContext extends EventEmitter { this._revertSuggestion = s as Reversion; } - if (this.langProcessor.mayAutoCorrect && s.autoAccept && !this.selected) { + if (this.langProcessor.mayAutoCorrect && s.autoAccept && !this.selected && s.tag != 'keep') { this.selected = s; } } From a7371ea3ec9cf5c74ccc6157a0254deb28831bd2 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 23 Jun 2026 12:22:43 -0500 Subject: [PATCH 02/21] feat(ios): add per-language autocorrect toggle Does not actually pass the toggle's setting to KMW yet. Build-bot: skip build:ios --- .../KMEI/KeymanEngine/Classes/Constants.swift | 1 + .../Extension/UserDefaults+Types.swift | 29 ++++ .../Keyboard/KeymanWebViewController.swift | 5 +- .../LanguageSettingsViewController.swift | 132 +++++++++++------- .../KeymanEngine/en.lproj/Localizable.strings | 3 + .../Contents/Resources/ios-host.js | 3 +- .../src/main/headless/languageProcessor.ts | 15 ++ 7 files changed, 138 insertions(+), 50 deletions(-) diff --git a/ios/engine/KMEI/KeymanEngine/Classes/Constants.swift b/ios/engine/KMEI/KeymanEngine/Classes/Constants.swift index 01d2862838..2d820d710e 100644 --- a/ios/engine/KMEI/KeymanEngine/Classes/Constants.swift +++ b/ios/engine/KMEI/KeymanEngine/Classes/Constants.swift @@ -33,6 +33,7 @@ public enum Key { /// Dictionary of prediction/correction toggle settings keyed by language id in UserDefaults static let userPredictSettings = "UserPredictionEnablementSettings" static let userCorrectSettings = "UserCorrectionEnablementSettings" + static let userAutocorrectSettings = "UserAutocorrectionEnablementSettings" // Internal user defaults keys static let engineVersion = "KeymanEngineVersion" diff --git a/ios/engine/KMEI/KeymanEngine/Classes/Extension/UserDefaults+Types.swift b/ios/engine/KMEI/KeymanEngine/Classes/Extension/UserDefaults+Types.swift index 8991660dae..4158dc51f2 100644 --- a/ios/engine/KMEI/KeymanEngine/Classes/Extension/UserDefaults+Types.swift +++ b/ios/engine/KMEI/KeymanEngine/Classes/Extension/UserDefaults+Types.swift @@ -331,6 +331,17 @@ public extension UserDefaults { set(prefs, forKey: Key.userCorrectSettings) } } + + // stores a dictionary of autocorrection-enablement settings keyed to language ids, i.e., [langID: Bool] + var autocorrectionEnablements: [String: Bool]? { + get { + return dictionary(forKey: Key.userAutocorrectSettings) as? [String : Bool] + } + + set(prefs) { + set(prefs, forKey: Key.userAutocorrectSettings) + } + } var portraitKeyboardHeight: Double { get { @@ -387,4 +398,22 @@ public extension UserDefaults { prefs?[forLanguageID] = correctSetting correctionEnablements = prefs } + + func autocorrectSettingForLanguage(languageID: String) -> Bool { + if let dict = autocorrectionEnablements { + return dict[languageID] ?? true + } else { + return true + } + } + + func set(autocorrectSetting: Bool, forLanguageID: String) { + var prefs: [String: Bool]? + prefs = autocorrectionEnablements + if prefs == nil { + prefs = [String: Bool]() + } + prefs?[forLanguageID] = autocorrectSetting + autocorrectionEnablements = prefs + } } diff --git a/ios/engine/KMEI/KeymanEngine/Classes/Keyboard/KeymanWebViewController.swift b/ios/engine/KMEI/KeymanEngine/Classes/Keyboard/KeymanWebViewController.swift index 4bd5375bc9..53e4e327ea 100644 --- a/ios/engine/KMEI/KeymanEngine/Classes/Keyboard/KeymanWebViewController.swift +++ b/ios/engine/KMEI/KeymanEngine/Classes/Keyboard/KeymanWebViewController.swift @@ -182,7 +182,7 @@ extension KeymanWebViewController { } view = nil } - + func languageMenuPosition(_ completion: @escaping (CGRect) -> Void) { webView!.evaluateJavaScript("langMenuPos();") { result, _ in guard let result = result as? String, !result.isEmpty else { @@ -422,10 +422,11 @@ extension KeymanWebViewController { let predict = userDefaults.predictSettingForLanguage(languageID: lexicalModel.languageID) let correct = userDefaults.correctSettingForLanguage(languageID: lexicalModel.languageID) + let autocorrect = userDefaults.autocorrectSettingForLanguage(languageID: lexicalModel.languageID) // Pass these off to KMW! // We do these first so that they're automatically set for the to-be-registered model in advance. - webView!.evaluateJavaScript("enableSuggestions(\(stubString), \(predict), \(correct))") + webView!.evaluateJavaScript("enableSuggestions(\(stubString), \(predict), \(correct), \(autocorrect))") self.activeModel = predict } else { // We're registering a model in the background - don't change settings. webView!.evaluateJavaScript("keyman.addModel(\(stubString));", completionHandler: nil) diff --git a/ios/engine/KMEI/KeymanEngine/Classes/Settings/LanguageSettingsViewController.swift b/ios/engine/KMEI/KeymanEngine/Classes/Settings/LanguageSettingsViewController.swift index 0c98a1cacc..816a7aea2c 100644 --- a/ios/engine/KMEI/KeymanEngine/Classes/Settings/LanguageSettingsViewController.swift +++ b/ios/engine/KMEI/KeymanEngine/Classes/Settings/LanguageSettingsViewController.swift @@ -18,8 +18,11 @@ class LanguageSettingsViewController: UITableViewController { private var doPredictionsSwitch: UISwitch? private var doCorrectionsSwitch: UISwitch? + private var doAutocorrectionsSwitch: UISwitch? private var doCorrectionsLabel: UILabel? + private var doAutocorrectionsLabel: UILabel? private var correctionsCell: UITableViewCell? + private var autocorrectionsCell: UITableViewCell? public init(_ inLanguage: Language) { language = inLanguage @@ -87,17 +90,8 @@ class LanguageSettingsViewController: UITableViewController { return switchFrame } - @objc - func predictionSwitchValueChanged(source: UISwitch) { - let value = source.isOn; + func refreshModelIfNeeded() { let userDefaults = Storage.active.userDefaults - userDefaults.set(predictSetting: value, forLanguageID: self.language.id) - - // Reactively set the corrections switch interactivity state. - self.doCorrectionsSwitch?.isHidden = !value - self.doCorrectionsLabel?.isEnabled = value - self.correctionsCell?.isUserInteractionEnabled = value - if let lm = Manager.shared.preferredLexicalModel(userDefaults, forLanguage: self.language.id) { if Manager.shared.currentKeyboardID?.languageID == self.language.id { // re-register the model - that'll enact the settings. @@ -111,23 +105,59 @@ class LanguageSettingsViewController: UITableViewController { } } + @objc + func predictionSwitchValueChanged(source: UISwitch) { + let value = source.isOn; + let userDefaults = Storage.active.userDefaults + userDefaults.set(predictSetting: value, forLanguageID: self.language.id) + + // Reactively set the corrections switch interactivity state. + self.doCorrectionsSwitch?.isHidden = !value + self.doCorrectionsLabel?.isEnabled = value + + let mayCorrect = userDefaults.autocorrectSettingForLanguage(languageID: self.language.id) + self.doAutocorrectionsSwitch?.isHidden = !(value && mayCorrect) + self.doAutocorrectionsLabel?.isEnabled = value && mayCorrect + self.correctionsCell?.isUserInteractionEnabled = value + } + @objc func correctionSwitchValueChanged(source: UISwitch) { let value = source.isOn; let userDefaults = Storage.active.userDefaults userDefaults.set(correctSetting: value, forLanguageID: self.language.id) - if let lm = Manager.shared.preferredLexicalModel(userDefaults, forLanguage: self.language.id) { - if Manager.shared.currentKeyboardID?.languageID == self.language.id { - // re-register the model - that'll enact the settings. - _ = Manager.shared.registerLexicalModel(lm) - } - // Based on how Manager chooses the model in setKeyboard. - } else if let lm = userDefaults.userLexicalModels?.first(where: { $0.languageID == self.language.id }) { - if Manager.shared.currentKeyboardID?.languageID == self.language.id { - _ = Manager.shared.registerLexicalModel(lm) - } - } + // This may only be triggered if the predict toggle is on, + // so we can rely on just the input value. + self.doAutocorrectionsSwitch?.isHidden = !value + self.doAutocorrectionsLabel?.isEnabled = value + + refreshModelIfNeeded() + } + + @objc + func autocorrectionSwitchValueChanged(source: UISwitch) { + let value = source.isOn; + let userDefaults = Storage.active.userDefaults + userDefaults.set(autocorrectSetting: value, forLanguageID: self.language.id) + + refreshModelIfNeeded() + } + + func addSwitchToTableCell(_ toggle: UISwitch, cell: UITableViewCell, isOn: Bool, selector: Selector) { + cell.accessoryType = .none + toggle.translatesAutoresizingMaskIntoConstraints = false + + let switchFrame = frameAtRightOfCell(cell: cell.frame, controlSize: toggle.frame.size) + toggle.frame = switchFrame + + toggle.isOn = isOn + toggle.addTarget(self, action: selector, for: .valueChanged) + cell.addSubview(toggle) + cell.contentView.isUserInteractionEnabled = false + + toggle.rightAnchor.constraint(equalTo: cell.layoutMarginsGuide.rightAnchor).isActive = true + toggle.centerYAnchor.constraint(equalTo: cell.layoutMarginsGuide.centerYAnchor).isActive = true } override func tableView(_ tableView: UITableView, cellForRowAt indexPath: IndexPath) -> UITableViewCell { @@ -143,40 +173,44 @@ class LanguageSettingsViewController: UITableViewController { let cell = UITableViewCell(style: .subtitle, reuseIdentifier: cellIdentifier) if 1 == indexPath.section { if 0 == indexPath.row { - cell.accessoryType = .none doPredictionsSwitch = UISwitch() - doPredictionsSwitch!.translatesAutoresizingMaskIntoConstraints = false - let switchFrame = frameAtRightOfCell(cell: cell.frame, controlSize: doPredictionsSwitch!.frame.size) - doPredictionsSwitch!.frame = switchFrame - - doPredictionsSwitch!.isOn = userDefaults.predictSettingForLanguage(languageID: self.language.id) - doPredictionsSwitch!.addTarget(self, action: #selector(self.predictionSwitchValueChanged), for: .valueChanged) - cell.addSubview(doPredictionsSwitch!) - cell.contentView.isUserInteractionEnabled = false - - doPredictionsSwitch!.rightAnchor.constraint(equalTo: cell.layoutMarginsGuide.rightAnchor).isActive = true - doPredictionsSwitch!.centerYAnchor.constraint(equalTo: cell.layoutMarginsGuide.centerYAnchor).isActive = true + self.addSwitchToTableCell( + doPredictionsSwitch!, + cell: cell, + isOn: userDefaults.predictSettingForLanguage(languageID: self.language.id), + selector: #selector(self.predictionSwitchValueChanged) + ) } else if 1 == indexPath.row { correctionsCell = cell - cell.accessoryType = .none doCorrectionsSwitch = UISwitch() - doCorrectionsSwitch!.translatesAutoresizingMaskIntoConstraints = false - let switchFrame = frameAtRightOfCell(cell: cell.frame, controlSize: doCorrectionsSwitch!.frame.size) - doCorrectionsSwitch!.frame = switchFrame - - doCorrectionsSwitch!.isOn = userDefaults.correctSettingForLanguage(languageID: self.language.id) - doCorrectionsSwitch!.addTarget(self, action: #selector(self.correctionSwitchValueChanged), for: .valueChanged) - cell.addSubview(doCorrectionsSwitch!) - cell.contentView.isUserInteractionEnabled = false - - doCorrectionsSwitch!.rightAnchor.constraint(equalTo: cell.layoutMarginsGuide.rightAnchor).isActive = true - doCorrectionsSwitch!.centerYAnchor.constraint(equalTo: cell.layoutMarginsGuide.centerYAnchor).isActive = true + self.addSwitchToTableCell( + doCorrectionsSwitch!, + cell: cell, + isOn: userDefaults.correctSettingForLanguage(languageID: self.language.id), + selector: #selector(self.correctionSwitchValueChanged) + ) // Disable interactivity if the prediction toggle is set to 'off'. doCorrectionsSwitch!.isHidden = !userDefaults.predictSettingForLanguage(languageID: self.language.id) cell.isUserInteractionEnabled = userDefaults.predictSettingForLanguage(languageID: self.language.id) + } else if 2 == indexPath.row { + autocorrectionsCell = cell + doAutocorrectionsSwitch = UISwitch() + + self.addSwitchToTableCell( + doAutocorrectionsSwitch!, + cell: cell, + isOn: userDefaults.autocorrectSettingForLanguage(languageID: self.language.id), + selector: #selector(self.autocorrectionSwitchValueChanged) + ) + + // Disable interactivity if the prediction or correction toggle is set to 'off'. + let mayPredict = userDefaults.predictSettingForLanguage(languageID: self.language.id) + let mayCorrect = userDefaults.correctSettingForLanguage(languageID: self.language.id) + doAutocorrectionsSwitch!.isHidden = !(mayPredict && mayCorrect) + cell.isUserInteractionEnabled = mayPredict && mayCorrect } else { // rows 3 and 4 cell.accessoryType = .disclosureIndicator } @@ -245,6 +279,10 @@ class LanguageSettingsViewController: UITableViewController { cell.textLabel?.text = NSLocalizedString("menu-langsettings-toggle-correct", bundle: engineBundle, comment: "") cell.textLabel?.isEnabled = !(doCorrectionsSwitch?.isHidden ?? false) case 2: + doAutocorrectionsLabel = cell.textLabel + cell.textLabel?.text = NSLocalizedString("menu-langsettings-toggle-autocorrect", bundle: engineBundle, comment: "") + cell.textLabel?.isEnabled = !(doAutocorrectionsSwitch?.isHidden ?? false) + case 3: cell.textLabel?.text = NSLocalizedString("menu-langsettings-label-lexical-models", bundle: engineBundle, comment: "") cell.accessoryType = .disclosureIndicator let modelCt = language.lexicalModels?.count ?? 0 @@ -332,8 +370,8 @@ class LanguageSettingsViewController: UITableViewController { showKeyboardInfoView(kb: (language.keyboards?[safe: indexPath.row])!) case 1: switch indexPath.row { - // case 0, 1: the toggles - but a general 'click' not on the toggle itself. - case 2: + // case 0, 1, 2: the toggles - but a general 'click' not on the toggle itself. + case 3: showLexicalModelsView() default: break diff --git a/ios/engine/KMEI/KeymanEngine/en.lproj/Localizable.strings b/ios/engine/KMEI/KeymanEngine/en.lproj/Localizable.strings index 05a1723ebb..0e5b70769b 100644 --- a/ios/engine/KMEI/KeymanEngine/en.lproj/Localizable.strings +++ b/ios/engine/KMEI/KeymanEngine/en.lproj/Localizable.strings @@ -154,6 +154,9 @@ /* Label for the toggle that enables corrections that is displayed within a language-specific settings menu */ "menu-langsettings-toggle-correct" = "Enable corrections"; +/* Label for the toggle that enables auto-corrections that is displayed within a language-specific settings menu */ +"menu-langsettings-toggle-autocorrect" = "Enable autocorrections"; + /* Label for the toggle that enables predictions that is displayed within a language-specific settings menu */ "menu-langsettings-toggle-predict" = "Enable predictions"; diff --git a/ios/engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/ios-host.js b/ios/engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/ios-host.js index 25af94b5bd..afb561b176 100644 --- a/ios/engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/ios-host.js +++ b/ios/engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/ios-host.js @@ -326,11 +326,12 @@ function toHex(theString) { return hexString.substr(0, hexString.length-1); } -function enableSuggestions(model, mayPredict, mayCorrect) { +function enableSuggestions(model, mayPredict, mayCorrect, mayAutocorrect) { // Set the options first so that KMW's ModelCache can properly handle model enablement states // the moment we actually register the new model. keyman.core.languageProcessor.mayPredict = mayPredict; keyman.core.languageProcessor.mayCorrect = mayCorrect; + keyman.core.languageProcessor.mayAutocorrect = mayAutocorrect; keyman.addModel(model); } diff --git a/web/src/engine/src/main/headless/languageProcessor.ts b/web/src/engine/src/main/headless/languageProcessor.ts index b98d4246df..c5d08c5ca9 100644 --- a/web/src/engine/src/main/headless/languageProcessor.ts +++ b/web/src/engine/src/main/headless/languageProcessor.ts @@ -166,6 +166,12 @@ export class LanguageProcessor extends EventEmitter { return !!this.recentTranscriptions.get(transitionId); } + /** + * When called, this requests the worker to generate predictions for the transcribed + * context transition. + * @param transcription + * @param layerId + */ public predict(transcription: Transcription, layerId: string): Promise { if(!this.isActive) { return null; @@ -373,6 +379,9 @@ export class LanguageProcessor extends EventEmitter { this.lmEngine.resetContext(context, transcription.token); } + // The "may correct" setting is enforced here, in the main Web engine - not + // in the worker. As the desired setting may vary language-to-language, + // we'd rather do this than reconfigure the worker constantly. let alternates = transcription.alternates; if(!this.mayCorrect || !alternates || alternates.length == 0) { alternates = [{ @@ -386,6 +395,12 @@ export class LanguageProcessor extends EventEmitter { return promise.then((suggestions: Suggestion[]) => { if(promise == this.currentPromise) { + if(!this.mayAutoCorrect) { + // We disable the auto-accept flag here, rather than in the worker - + // that way, we don't need to reconfigure the worker when settings + // change. + suggestions.forEach((s) => delete s.autoAccept); + } const result = new ReadySuggestions(suggestions, transform.id); this.emit("suggestionsready", result); this.currentPromise = null; From b14fe90894bffe61af158f98c2268a93f457ee12 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Fri, 26 Jun 2026 09:01:16 -0500 Subject: [PATCH 03/21] change(web): polish documentation + uses of transition ID in predictive text - Drops the `Suggestion.transformId` field in favor of just reusing `.transform.id` - Updates documentation on .transform.id as a unique identifier of the transition, not specific to a single Transform. Does not modify the `.transform` or `.id` properties because we've been publishing lexical model types, and that would change the public model API. Build-bot: skip build:web Test-bot: skip --- common/web/types/src/lexical-model-types.ts | 18 +++++----- .../predictive-text/templates/src/common.ts | 4 --- .../src/main/correction/context-state.ts | 2 +- .../src/main/correction/context-token.ts | 10 +++--- .../main/correction/context-tokenization.ts | 4 +-- .../src/main/correction/context-transition.ts | 12 +++---- .../src/main/model-compositor.ts | 31 +++++++++-------- .../src/main/models/dummy-model.ts | 3 +- .../worker-thread/src/main/predict-helpers.ts | 15 +++------ .../prediction/predictionContext.ts | 2 +- .../src/main/headless/inputProcessor.ts | 2 +- .../src/main/headless/languageProcessor.ts | 6 ++-- .../engine/main/inputProcessor.tests.ts | 3 +- .../predictive-text/templates/common.tests.ts | 10 +++--- .../context/context-tokenization.tests.ts | 32 +++++++++--------- .../context/context-tracker.tests.ts | 2 +- .../context/context-transition.tests.ts | 32 +++++++----------- .../base-context-state.tests.ts | 2 -- ...ine-suggestion-context-transition.tests.ts | 21 +++++------- .../predict-from-corrections.tests.ts | 1 - .../worker-model-compositor.tests.ts | 33 ++++++++----------- 21 files changed, 107 insertions(+), 138 deletions(-) diff --git a/common/web/types/src/lexical-model-types.ts b/common/web/types/src/lexical-model-types.ts index 37a761608d..a3c8df1c98 100644 --- a/common/web/types/src/lexical-model-types.ts +++ b/common/web/types/src/lexical-model-types.ts @@ -253,9 +253,14 @@ export interface LexicalModel { */ export interface Transform { /** - * Facilitates use of unique identifiers for tracking the Transform and - * any related data from its original source, as the reference cannot be - * preserved across WebWorker boundaries. + * Facilitates use of unique identifiers for tracking data about the context + * transition to which the Transform belongs. More than one Transform may + * hold the same `id` if they are alternate interpretations of the same + * transition event - say, the resulting effects of neighbor keys that may + * have been missed due to "fat fingering". + * + * Also note that the Transform reference cannot be preserved across WebWorker + * boundaries, but this ID may. * * This is *separate* from any LMLayer-internal identification values. */ @@ -287,13 +292,6 @@ export interface Transform { * A concrete suggestion */ export interface Suggestion { - /** - * Indicates the externally-supplied id of the Transform that prompted - * the Suggestion. Automatically handled by the LMLayer; models should - * not handle this field. - */ - transformId?: number; - /** * A unique identifier for the Suggestion itself, not shared with any others - * even for Suggestions sourced from the same Transform. diff --git a/web/src/engine/predictive-text/templates/src/common.ts b/web/src/engine/predictive-text/templates/src/common.ts index ec6e8a09e5..d141f1f9ae 100644 --- a/web/src/engine/predictive-text/templates/src/common.ts +++ b/web/src/engine/predictive-text/templates/src/common.ts @@ -132,10 +132,6 @@ export function transformToSuggestion(transform: Transform, p?: number): Outcome displayAs: transform.insert }; - if(transform.id !== undefined) { - suggestion.transformId = transform.id; - } - if(p === 0 || p) { suggestion.p = p; } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts index 673acf5493..bc9b412bf3 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts @@ -89,7 +89,7 @@ export class ContextState { return undefined; } - return this.suggestions.find(s => s.id == this.appliedSuggestionId)?.transformId; + return this.suggestions.find(s => s.id == this.appliedSuggestionId)?.transform.id; } /** diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts index 942773fa57..0a772514f3 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts @@ -27,12 +27,12 @@ import Transform = LexicalModelTypes.Transform; * any prior cached data or for rewriting its probabilities after * receiving backspace input. * @param text - * @param transformId + * @param transitionId * @returns */ -function textToCharTransforms(text: string, transformId?: number): Transform[] { - return transformId ? - [...text].map(insert => ({insert, deleteLeft: 0, id: transformId})) : +function textToCharTransforms(text: string, transitionId?: number): Transform[] { + return transitionId ? + [...text].map(insert => ({insert, deleteLeft: 0, id: transitionId})) : [...text].map(insert => ({insert, deleteLeft: 0})); } @@ -100,7 +100,7 @@ export class ContextToken { static fromRawText(model: LexicalModel, rawText: string, isPartial?: boolean) { rawText ||= ''; - // Supports the old pathway for: updateWithBackspace(tokenText: string, transformId: number) + // Supports the old pathway for: updateWithBackspace(tokenText: string, transitionId: number) // Build a token that represents the current text with no ambiguity - probability at max (1.0) let searchModule: SearchQuotientNode = new LegacyQuotientRoot(model); const BASE_PROBABILITY = 1; diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index dcd7b0a295..ee8b22e53f 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -613,8 +613,8 @@ export class ContextTokenization { * @param transitionEdge Batched results from one or more * `precomputeTokenizationAfterInput` calls on this instance, all with the * same alignment values. - * @param transitionId The id of the Transform associated with the keystroke - * triggering the transition. + * @param transitionId The id of the context transition associated with the + * Transition * @param bestProbFromSet The probability of the single most likely input * transform in the overall transformDistribution associated with the * keystroke triggering the transition. It need not be represented by the diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts index a75245908e..c65f348996 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts @@ -36,7 +36,7 @@ export class ContextTransition { */ inputDistribution?: Distribution; - // The transform ID in play. + // The transition ID in play. private _transitionId?: number; /** @@ -177,7 +177,7 @@ export class ContextTransition { // We won't try to partially revert a multi-word suggestion; reversions // are only supported at the end of the last word of the main suggestion // body and after any appended whitespace. - resultingTokenization.tail.appliedTransitionId = suggestion.transformId; + resultingTokenization.tail.appliedTransitionId = suggestion.transform.id; const resultingState = new ContextState(applyTransform(transformToApply, baseState.context), lexicalModel); resultingState.tokenization = resultingTokenization; // [resultingTokenization].concat(preservedVariations); @@ -185,12 +185,12 @@ export class ContextTransition { resultingState.appliedSuggestionId = suggestion.id; resultingState.suggestions = this.final.suggestions; - // Use the transform's ID for the transition. Note that when applying the + // Use the transition ID tracked on the Transform. Note that when applying the // `appendedTransform` component of a suggestion, this will differ from - // suggestion.transformId. + // suggestion.transitionId. const resultingTransition = new ContextTransition(baseState, transformToApply.id); resultingTransition.finalize(resultingState, inputDistribution); - resultingTransition.revertableTransitionId = suggestion.transformId; + resultingTransition.revertableTransitionId = suggestion.transform.id; // .finalize unsets _.transitionId; re-assign it. resultingTransition._transitionId = transformToApply.id; @@ -232,7 +232,7 @@ export class ContextTransition { const baseTokenizationLength = results.transition.final.displayTokenization.tokens.length; const appliedTokenization = appendingTransition.final.displayTokenization; for(let i = baseTokenizationLength; i < appliedTokenization.tokens.length; i++) { - appliedTokenization.tokens[i].appliedTransitionId = suggestion.transformId; + appliedTokenization.tokens[i].appliedTransitionId = suggestion.transform.id; } return { diff --git a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts index 5361c198e0..64d6ae4e20 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts @@ -81,12 +81,12 @@ export class ModelCompositor { this.configuration = config; } - initContextTracker(context: Context, transformId: number) { + initContextTracker(context: Context, transitionId: number) { if(this.contextTracker || !this.lexicalModel.traverseFromRoot) { return; } - this._contextTracker = new ContextTracker(this.lexicalModel, context, transformId, this.configuration); + this._contextTracker = new ContextTracker(this.lexicalModel, context, transitionId, this.configuration); } async predict(transformDistribution: Transform | Distribution, context: Context): Promise[]> { @@ -122,8 +122,8 @@ export class ModelCompositor { // Only allow new-word suggestions if space was the most likely keypress. // const allowSpace = TransformUtils.isWhitespace(inputTransform); const inputTransform = transformDistribution[0].sample; - const transformId = inputTransform.id; - this.initContextTracker(context, transformId); + const transitionId = inputTransform.id; + this.initContextTracker(context, transitionId); const allowBksp = TransformUtils.isBackspace(inputTransform); const allowWhitespace = TransformUtils.isWhitespace(inputTransform); @@ -240,6 +240,8 @@ export class ModelCompositor { // The Web engine will restore the original state of the context before accepting // and before reverting; all we need to do is put the original keystroke back in place. let reversionTransform: Transform = originalInput ?? { insert: '', deleteLeft: 0 }; + // clone the transform to prevent aliasing + reversionTransform = {...reversionTransform}; // Step 2: building the proper 'displayAs' string for the Reversion const postContext = originalInput ? models.applyTransform(originalInput, context) : context; @@ -263,8 +265,8 @@ export class ModelCompositor { // Since we're outside of the standard `predict` control path, we'll need to // set the Reversion's ID directly. let reversion = toAnnotatedSuggestion(this.lexicalModel, firstConversion, 'revert'); - if(suggestion.transformId != null) { - reversion.transformId = -suggestion.transformId; + if(suggestion.transform.id != null) { + reversion.transform.id = -suggestion.transform.id; } if(suggestion.id != null) { // Since a reversion inverts its source suggestion, we set its ID to be the @@ -287,11 +289,11 @@ export class ModelCompositor { } } else { let originalTransition = this.contextTracker.latest; - if(originalTransition.transitionId != suggestion.transformId) { - originalTransition = this.contextTracker.findAndRevert(suggestion.transformId); + if(originalTransition.transitionId != suggestion.transform.id) { + originalTransition = this.contextTracker.findAndRevert(suggestion.transform.id); } if(!originalTransition) { - this.contextTracker.reset(context, suggestion.transformId); + this.contextTracker.reset(context, suggestion.transform.id); originalTransition = this.contextTracker.latest; } @@ -333,7 +335,7 @@ export class ModelCompositor { suggestions.forEach(function(suggestion) { // A reversion's transform ID is the additive inverse of its original suggestion; // we revert to the state of said original suggestion. - suggestion.transformId = -reversion.transformId; + suggestion.transform.id = -reversion.transform.id; // Prevent auto-selection of any suggestion immediately after a reversion. // It's fine after at least one keystroke, but not before. suggestion.autoAccept = false; @@ -347,18 +349,19 @@ export class ModelCompositor { } // When the context is tracked, we prefer the tracked information. - // Note that the base reversion's .transformId will predate the appendedTransform id + // Note that the base reversion's .transform.id will predate the appendedTransform id // used to add whitespace (if one existed), so reverting to the base ID's associated // context also reverts the appendedTransform. - let originalTransition = this.contextTracker.findAndRevert(-reversion.transformId); + const baseTransitionId = -reversion.transform.id; + let originalTransition = this.contextTracker.findAndRevert(baseTransitionId); if(appendedOnly) { this.contextTracker.latest = originalTransition; return Promise.resolve([]); } - if(!originalTransition) { - this.contextTracker.reset(context, -reversion.transformId); + if(!originalTransition || originalTransition.transitionId != baseTransitionId) { + this.contextTracker.reset(context, baseTransitionId); originalTransition = this.contextTracker.latest; suggestions = fallbackSuggestions(); diff --git a/web/src/engine/predictive-text/worker-thread/src/main/models/dummy-model.ts b/web/src/engine/predictive-text/worker-thread/src/main/models/dummy-model.ts index 3b5854ef58..e1651557c4 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/models/dummy-model.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/models/dummy-model.ts @@ -98,12 +98,11 @@ export class DummyModel implements LexicalModel { predict(transform: Transform, context: Context, injectedSuggestions?: Outcome[]): Distribution { let makeUniformDistribution = function(suggestions: Outcome[]): Distribution { let distribution: Distribution = []; + const transitionId = transform.id; for(let s of suggestions) { - const transitionId = transform.id; if(transitionId !== undefined) { // Set the transform ID to match the incoming transform ID if one exists. - s.transformId = transitionId; if(s.transform) { s.transform.id = transitionId; } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 9739adba85..7df559e93e 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -697,7 +697,7 @@ export function predictFromCorrections( // Let's not rely on the model to copy transform IDs. // Only bother is there IS an ID to copy. if(correctionTransform.id !== undefined) { - pair.sample.transformId = correctionTransform.id; + pair.sample.transform.id = correctionTransform.id; } let tuple: CorrectionPredictionTuple = { @@ -829,7 +829,7 @@ export function processSimilarity( for(let tuple of suggestionDistribution) { // Don't set it unnecessarily; this can have side-effects in some automated tests. if(inputTransform.id !== undefined) { - tuple.prediction.sample.transformId = inputTransform.id; + tuple.prediction.sample.transform.id = inputTransform.id; } const predictedWord = wordbreak(models.applyTransform(tuple.prediction.sample.transform, context)); @@ -907,7 +907,7 @@ export function createDefaultKeep( let keepOption = toAnnotatedSuggestion(lexicalModel, keepSuggestion, 'keep'); if(inputTransform.id !== undefined) { - keepOption.transformId = inputTransform.id; + keepOption.transform.id = inputTransform.id; } keepOption.matchesModel = false; @@ -1084,11 +1084,6 @@ export function finalizeSuggestions( mutableSuggestion.transform = mergedTransform; } - // Is sometimes not set during unit tests. - if(prediction.sample.transformId !== undefined) { - prediction.sample.transform.id = prediction.sample.transformId; - } - if(!verbose) { return { ...prediction.sample, @@ -1189,8 +1184,8 @@ export function toAnnotatedSuggestion( result.appendedTransform = suggestion.appendedTransform; } - if(suggestion.transformId !== undefined) { - result.transformId = suggestion.transformId; + if(suggestion.transform.id !== undefined) { + result.transform.id = suggestion.transform.id; } return result; diff --git a/web/src/engine/src/interfaces/prediction/predictionContext.ts b/web/src/engine/src/interfaces/prediction/predictionContext.ts index 97182a9bf1..2ec526fb37 100644 --- a/web/src/engine/src/interfaces/prediction/predictionContext.ts +++ b/web/src/engine/src/interfaces/prediction/predictionContext.ts @@ -273,7 +273,7 @@ export class PredictionContext extends EventEmitter { // Verify that the transition IDs are still valid and remove special entries. this._currentSuggestions = suggestions.filter(s => { - return this.langProcessor.hasState(Math.abs(s.transformId)) && + return this.langProcessor.hasState(Math.abs(s.transform.id)) && s != this._keepSuggestion && s != this._revertSuggestion }); diff --git a/web/src/engine/src/main/headless/inputProcessor.ts b/web/src/engine/src/main/headless/inputProcessor.ts index 1099b09145..5018e4de02 100644 --- a/web/src/engine/src/main/headless/inputProcessor.ts +++ b/web/src/engine/src/main/headless/inputProcessor.ts @@ -386,7 +386,7 @@ export class InputProcessor { // If so, since it behaves the same in either case, and it's a known word-breaking mark, // let's apply the selected suggestion automatically. if(postApplyTransform.insert == ruleTransform.insert && transformMatchesPattern(postApplyTransform, breakingMarks)) { - const baseTransition = this.contextCache.get(predictionContext.selected.transformId); + const baseTransition = this.contextCache.get(predictionContext.selected.transform.id); // Somehow, the base state is out of context - abort! if(!baseTransition) { diff --git a/web/src/engine/src/main/headless/languageProcessor.ts b/web/src/engine/src/main/headless/languageProcessor.ts index b98d4246df..3130fcaf27 100644 --- a/web/src/engine/src/main/headless/languageProcessor.ts +++ b/web/src/engine/src/main/headless/languageProcessor.ts @@ -218,13 +218,13 @@ export class LanguageProcessor extends EventEmitter { // Find the state of the context at the time the suggestion was generated. // This may refer to the context before an input keystroke or before application // of a predictive suggestion. - const original = this.getPredictionState(suggestion.transformId); + const original = this.getPredictionState(suggestion.transform.id); if(!original) { console.warn("Could not apply the Suggestion!"); return null; } - this.recentTranscriptions.rewindTo(suggestion.transformId); + this.recentTranscriptions.rewindTo(suggestion.transform.id); // Apply the Suggestion! @@ -309,7 +309,7 @@ export class LanguageProcessor extends EventEmitter { // // Reversions use the additive inverse of the id token of the Transcription being // reverted to. - const reversionId = appendedOnly ? reversion.appendedTransform.id : -reversion.transformId; + const reversionId = appendedOnly ? reversion.appendedTransform.id : -reversion.transform.id; const original = this.getPredictionState(reversionId); if(!original) { console.warn("Could not apply the Suggestion!"); diff --git a/web/src/test/auto/headless/engine/main/inputProcessor.tests.ts b/web/src/test/auto/headless/engine/main/inputProcessor.tests.ts index 1d63669537..bb469c723e 100644 --- a/web/src/test/auto/headless/engine/main/inputProcessor.tests.ts +++ b/web/src/test/auto/headless/engine/main/inputProcessor.tests.ts @@ -283,9 +283,8 @@ describe('InputProcessor', function() { [], [ { - transform: { insert: 'testing', deleteLeft: 4 }, + transform: { insert: 'testing', deleteLeft: 4, id: 0 }, appendedTransform: { insert: ' ', deleteLeft: 0 }, - transformId: 0, // will be overwritten by the DummyModel to match the transition ID. id: 1, displayAs: 'testing' } diff --git a/web/src/test/auto/headless/engine/predictive-text/templates/common.tests.ts b/web/src/test/auto/headless/engine/predictive-text/templates/common.tests.ts index 5abd27cd75..0c7b010586 100644 --- a/web/src/test/auto/headless/engine/predictive-text/templates/common.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/templates/common.tests.ts @@ -191,7 +191,7 @@ describe('Common utility functions', function() { deleteLeft: 0, id: 0 }, - transformId: 0, + transitionId: 0, displayAs: 'hello' }; @@ -205,7 +205,7 @@ describe('Common utility functions', function() { deleteLeft: 0, id: 0 }, - transformId: 0, + transitionId: 0, displayAs: 'hello', p: 0 }; @@ -220,7 +220,7 @@ describe('Common utility functions', function() { deleteLeft: 0, id: 0 }, - transformId: 0, + transitionId: 0, displayAs: 'hello', p: 0.5 }; @@ -228,14 +228,14 @@ describe('Common utility functions', function() { assert.deepEqual(models.transformToSuggestion(suggestion.transform, 0.5), suggestion); }); - it('properly handles the transformId', function() { + it('properly handles the transitionId', function() { let suggestion = { transform: { insert: 'hello', deleteLeft: 0, id: 3 }, - transformId: 3, // Ensures there isn't a separate ID seed in use. + transitionId: 3, // Ensures there isn't a separate ID seed in use. displayAs: 'hello' }; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index 3c8351f7d0..dcef6f8f61 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -45,8 +45,8 @@ function toToken(text: string) { } let TOKEN_TRANSFORM_SEED = 0; -function toTransformToken(text: string, transformId?: number) { - let idSeed = transformId === undefined ? TOKEN_TRANSFORM_SEED++ : transformId; +function toTransitionToken(text: string, transitionId?: number) { + let idSeed = transitionId === undefined ? TOKEN_TRANSFORM_SEED++ : transitionId; let isWhitespace = text == ' '; let token = ContextToken.fromRawText(plainModel, ''); const textAsTransform = { insert: text, deleteLeft: 0, id: idSeed }; @@ -104,7 +104,7 @@ describe('ContextTokenization', function() { it("constructs from a token array + alignment data", () => { const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; - const tokens = rawTextTokens.map((text => toTransformToken(text))); + const tokens = rawTextTokens.map((text => toTransitionToken(text))); const emptyTransform = { insert: '', deleteLeft: 0, deleteRight: 0 }; // We _could_ flesh this out a bit more... but it's not really needed for this test. @@ -141,7 +141,7 @@ describe('ContextTokenization', function() { it('clones', () => { const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; - const tokens = rawTextTokens.map((text => toTransformToken(text))); + const tokens = rawTextTokens.map((text => toTransitionToken(text))); const emptyTransform = { insert: '', deleteLeft: 0, deleteRight: 0 }; // We _could_ flesh this out a bit more... but it's not really needed for this test. @@ -830,7 +830,7 @@ describe('ContextTokenization', function() { it('handles empty contexts', () => { const baseTokens = ['']; const idSeed = TOKEN_TRANSFORM_SEED; - const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t))); const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 0 }, true, testEdgeWindowSpec); assert.deepEqual(results, { @@ -850,7 +850,7 @@ describe('ContextTokenization', function() { it('handles empty contexts and invalid Transforms', () => { const baseTokens = ['']; const idSeed = TOKEN_TRANSFORM_SEED; - const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t))); const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 2 }, true, testEdgeWindowSpec); assert.deepEqual(results, { @@ -870,7 +870,7 @@ describe('ContextTokenization', function() { it('builds edge windows for the start of context with no edits', () => { const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; const idSeed = TOKEN_TRANSFORM_SEED; - const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t))); const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 0 }, true, testEdgeWindowSpec); assert.deepEqual(results, { @@ -910,7 +910,7 @@ describe('ContextTokenization', function() { it('builds edge windows for the start of context with deletion edits (1)', () => { const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; const idSeed = TOKEN_TRANSFORM_SEED; - const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t))); const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 2 }, true, testEdgeWindowSpec); assert.deepEqual(results, { @@ -950,7 +950,7 @@ describe('ContextTokenization', function() { it('builds edge windows for the start of context with deletion edits (2)', () => { const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; const idSeed = TOKEN_TRANSFORM_SEED; - const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t))); const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 4 }, true, testEdgeWindowSpec); assert.deepEqual(results, { @@ -970,7 +970,7 @@ describe('ContextTokenization', function() { it('builds edge windows for the end of context with no edits', () => { const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; const idSeed = TOKEN_TRANSFORM_SEED; - const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t))); baseTokenization.tail.isPartial = true; const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 0 }, false, testEdgeWindowSpec); @@ -991,7 +991,7 @@ describe('ContextTokenization', function() { it('builds edge windows for the end of context with no edits, trailing whitespace', () => { const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day', ' ', '']; const idSeed = TOKEN_TRANSFORM_SEED; - const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t))); const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 0 }, false, testEdgeWindowSpec); assert.deepEqual(results, { @@ -1778,7 +1778,7 @@ describe('ContextTokenization', function() { it('returns the standard edge window for empty transform inputs', () => { const baseTokens = ['quick', ' ', 'brown', ' ', 'fox']; const idSeed = TOKEN_TRANSFORM_SEED; - const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t))); const editTransform = { insert: '', @@ -1812,7 +1812,7 @@ describe('ContextTokenization', function() { it('returns the standard edge window for empty transforms with context-final whitespace', () => { const baseTokens = ['quick', ' ', 'brown', ' ', 'fox', ' ']; const idSeed = TOKEN_TRANSFORM_SEED; - const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t))); const editTransform = { insert: '', @@ -1847,7 +1847,7 @@ describe('ContextTokenization', function() { it('returns the standard edge window for pure transform w insert inputs', () => { const baseTokens = ['quick', ' ', 'brown', ' ', 'fox']; const idSeed = TOKEN_TRANSFORM_SEED; - const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t))); const editTransform = { insert: ' jumped', @@ -1881,7 +1881,7 @@ describe('ContextTokenization', function() { it('returns the proper edge window for transforms w deleteLeft inputs (1)', () => { const baseTokens = ['quick', ' ', 'brown', ' ', 'fox']; const idSeed = TOKEN_TRANSFORM_SEED; - const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t))); const editTransform = { insert: 'rog', @@ -1916,7 +1916,7 @@ describe('ContextTokenization', function() { it('returns the proper edge window for transforms w deleteLeft inputs (2)', () => { const baseTokens = ['quick', ' ', 'brown', ' ', 'fox']; const idSeed = TOKEN_TRANSFORM_SEED; - const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t))); const editTransform = { insert: 'fox and brown fox', // => quick fox and brown fox diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts index f27ebd72f7..050ab56c4a 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts @@ -34,7 +34,7 @@ describe('ContextTracker', function() { deleteLeft: 0, id: 15 }, - transformId: 2, + transitionId: 2, id: 1, displayAs: 'world' }; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-transition.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-transition.tests.ts index 263a24aa47..2477a80f18 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-transition.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-transition.tests.ts @@ -118,9 +118,8 @@ describe('ContextTransition', () => { appendedTransform: { insert: ' ', deleteLeft: 0, - id: 2 + id: 3 }, - transformId: 2, id: 10, displayAs: 'world' }, { @@ -132,9 +131,8 @@ describe('ContextTransition', () => { appendedTransform: { insert: ' ', deleteLeft: 0, - id: 2 + id: 3 }, - transformId: 2, id: 11, displayAs: 'won' }]; @@ -155,7 +153,7 @@ describe('ContextTransition', () => { // 3 long, only last token was edited. appliedTransition.base.final.tokenization.tokens.forEach((token, index) => { if(index >= 2) { - assert.equal(token.appliedTransitionId, suggestions[0].transformId); + assert.equal(token.appliedTransitionId, suggestions[0].transform.id); } else { assert.isUndefined(token.appliedTransitionId); } @@ -163,7 +161,7 @@ describe('ContextTransition', () => { appliedTransition.appended.final.tokenization.tokens.forEach((token, index) => { if(index >= 2) { - assert.equal(token.appliedTransitionId, suggestions[0].transformId); + assert.equal(token.appliedTransitionId, suggestions[0].transform.id); } else { assert.isUndefined(token.appliedTransitionId); } @@ -202,9 +200,8 @@ describe('ContextTransition', () => { appendedTransform: { insert: ' ', deleteLeft: 0, - id: 2 + id: 3 }, - transformId: 2, id: 10, displayAs: 'the' }, { @@ -216,9 +213,8 @@ describe('ContextTransition', () => { appendedTransform: { insert: ' ', deleteLeft: 0, - id: 2 + id: 3 }, - transformId: 2, id: 11, displayAs: 'and' }]; @@ -239,7 +235,7 @@ describe('ContextTransition', () => { // 3 long, only last token was edited. appliedTransition.base.final.tokenization.tokens.forEach((token, index) => { if(index >= 4) { - assert.equal(token.appliedTransitionId, suggestions[0].transformId); + assert.equal(token.appliedTransitionId, suggestions[0].transform.id); } else { assert.isUndefined(token.appliedTransitionId); } @@ -247,7 +243,7 @@ describe('ContextTransition', () => { appliedTransition.appended.final.tokenization.tokens.forEach((token, index) => { if(index >= 4) { - assert.equal(token.appliedTransitionId, suggestions[0].transformId); + assert.equal(token.appliedTransitionId, suggestions[0].transform.id); } else { assert.isUndefined(token.appliedTransitionId); } @@ -287,9 +283,8 @@ describe('ContextTransition', () => { appendedTransform: { insert: ' ', deleteLeft: 0, - id: 2 + id: 3 }, - transformId: 2, id: 10, displayAs: 'world' }, { @@ -301,9 +296,8 @@ describe('ContextTransition', () => { appendedTransform: { insert: ' ', deleteLeft: 0, - id: 2 + id: 3 }, - transformId: 2, id: 11, displayAs: 'won' }]; @@ -337,9 +331,8 @@ describe('ContextTransition', () => { appendedTransform: { insert: ' ', deleteLeft: 0, - id: 2 + id: 3 }, - transformId: 2, id: 10, displayAs: 'world' }, { @@ -351,9 +344,8 @@ describe('ContextTransition', () => { appendedTransform: { insert: ' ', deleteLeft: 0, - id: 2 + id: 3 }, - transformId: 2, id: 11, displayAs: 'won' }]; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/base-context-state.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/base-context-state.tests.ts index 6b36685093..a38b4e3012 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/base-context-state.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/base-context-state.tests.ts @@ -168,7 +168,6 @@ describe('matchBaseContextState', () => { const suggestion: Suggestion= { transform: { insert: 'dramatically', deleteLeft: 0, id: 3 }, displayAs: 'dramatically', - transformId: 3, id: 5 } @@ -234,7 +233,6 @@ describe('matchBaseContextState', () => { const suggestion: Suggestion= { transform: { insert: '', deleteLeft: 'dramatically'.length, id: 3 }, displayAs: '""', - transformId: 3, id: 5 } diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts index 218f18313a..0be3b6525e 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts @@ -137,7 +137,6 @@ describe('determineContextTransition', () => { insert: ' ', deleteLeft: 0 }, - transformId: 0, displayAs: 'testing' }; baseTransition.final.suggestions = [pred_testing]; @@ -192,7 +191,7 @@ describe('determineContextTransition', () => { transform: { insert: 'testing', deleteLeft: 4, - id: 1 + id: 0 }, appendedTransform: { insert: ' ', @@ -200,7 +199,6 @@ describe('determineContextTransition', () => { id: 2 }, id: 4, - transformId: 0, displayAs: 'testing' }; baseTransition.final.suggestions = [pred_testing]; @@ -226,8 +224,8 @@ describe('determineContextTransition', () => { assert.notEqual(extendingTransition, baseTransition); // These values support delayed reversions. - assert.equal(extendingTransition.final.tokenization.tokens[6].appliedTransitionId, pred_testing.transformId); - assert.equal(extendingTransition.final.tokenization.tokens[7].appliedTransitionId, pred_testing.transformId); + assert.equal(extendingTransition.final.tokenization.tokens[6].appliedTransitionId, pred_testing.transform.id); + assert.equal(extendingTransition.final.tokenization.tokens[7].appliedTransitionId, pred_testing.transform.id); // We start a new token here, rather than continue (and/or replace) an old one; // this shouldn't be set here yet. @@ -253,14 +251,13 @@ describe('determineContextTransition', () => { transform: { insert: 'testing', deleteLeft: 4, - id: 1 + id: 0 }, appendedTransform: { insert: ' ', deleteLeft: 0, id: 2 }, - transformId: 0, displayAs: 'testing' }; baseTransition.final.suggestions = [pred_testing]; @@ -293,7 +290,7 @@ describe('determineContextTransition', () => { [{sample: { insert: '', deleteLeft: 1 }, p: 1}] ); - assert.equal(extensionDeletingTransition.revertableTransitionId, pred_testing.transformId); + assert.equal(extensionDeletingTransition.revertableTransitionId, pred_testing.transform.id); } finally { warningEmitterSpy.restore(); } @@ -315,14 +312,13 @@ describe('determineContextTransition', () => { transform: { insert: 'testing', deleteLeft: 4, - id: 1 + id: 0 }, appendedTransform: { insert: ' ', deleteLeft: 0, id: 2 }, - transformId: 0, displayAs: 'testing' }; baseTransition.final.suggestions = [pred_testing]; @@ -367,7 +363,7 @@ describe('determineContextTransition', () => { [{sample: { insert: '', deleteLeft: 1 }, p: 1}] ); - assert.equal(appendDeletingTransition.revertableTransitionId, pred_testing.transformId); + assert.equal(appendDeletingTransition.revertableTransitionId, pred_testing.transform.id); } finally { warningEmitterSpy.restore(); } @@ -389,14 +385,13 @@ describe('determineContextTransition', () => { transform: { insert: 'testing', deleteLeft: 4, - id: 1 + id: 0 }, appendedTransform: { insert: ' ', deleteLeft: 0, id: 2 }, - transformId: 0, displayAs: 'testing' }; baseTransition.final.suggestions = [pred_testing]; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-corrections.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-corrections.tests.ts index 2351ff9932..31f5063c15 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-corrections.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-corrections.tests.ts @@ -172,7 +172,6 @@ describe('predictFromCorrections', () => { assert.sameOrderedMembers(predictions.map((entry) => entry.prediction.sample.displayAs), ["it's", "its"]); assert.sameDeepOrderedMembers(predictions.map((entry) => entry.prediction.sample), dummied_suggestions.map((entry) => { entry = deepCopy(entry); - entry.transformId = 314159; return entry; })); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts index 928b6e75c4..5343760d56 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts @@ -588,7 +588,6 @@ describe('ModelCompositor', function() { deleteLeft: 0, id: 0 }, - transformId: 0, displayAs: 'hello' }; @@ -668,13 +667,12 @@ describe('ModelCompositor', function() { } it('first word of context, postTransform provided, .deleteLeft = 0', function() { - let baseSuggestion = { + let baseSuggestion: Suggestion = { transform: { insert: 'hello ', deleteLeft: 2, id: 0 }, - transformId: 0, id: 1, displayAs: 'hello' }; @@ -691,7 +689,7 @@ describe('ModelCompositor', function() { } let reversion = acceptanceTest(englishPunctuation, baseSuggestion, baseContext, postTransform); - assert.equal(reversion.transformId, baseSuggestion.transformId); + assert.equal(reversion.transform.id, baseSuggestion.transform.id); assert.equal(reversion.id, -baseSuggestion.id); // Check #1: Does the returned reversion properly revert the context to its pre-application state? @@ -711,13 +709,12 @@ describe('ModelCompositor', function() { }); it('second word of context, postTransform provided, .deleteLeft = 0', function() { - let baseSuggestion = { + let baseSuggestion: Suggestion = { transform: { insert: 'world ', deleteLeft: 3, id: 0 }, - transformId: 0, id: 0, displayAs: 'world' }; @@ -734,7 +731,7 @@ describe('ModelCompositor', function() { } let reversion = acceptanceTest(englishPunctuation, baseSuggestion, baseContext, postTransform); - assert.equal(reversion.transformId, baseSuggestion.transformId); + assert.equal(reversion.transform.id, baseSuggestion.transform.id); assert.equal(reversion.id, -baseSuggestion.id); // Check #1: Does the returned reversion properly revert the context to its pre-application state? @@ -754,13 +751,12 @@ describe('ModelCompositor', function() { }); it('second word of context, postTransform undefined', function() { - let baseSuggestion = { + let baseSuggestion: Suggestion = { transform: { insert: 'world ', deleteLeft: 3, id: 0 }, - transformId: 0, id: 0, displayAs: 'world' }; @@ -770,7 +766,7 @@ describe('ModelCompositor', function() { } let reversion = acceptanceTest(englishPunctuation, baseSuggestion, baseContext); - assert.equal(reversion.transformId, baseSuggestion.transformId); + assert.equal(reversion.transform.id, baseSuggestion.transform.id); assert.equal(reversion.id, -baseSuggestion.id); // Check #1: Does the returned reversion properly revert the context to its pre-application state? @@ -790,13 +786,12 @@ describe('ModelCompositor', function() { }); it('first word of context + postTransform provided, .deleteLeft > 0', function() { - let baseSuggestion = { + let baseSuggestion: Suggestion = { transform: { insert: 'hello ', deleteLeft: 2, id: 0 }, - transformId: 0, id: 0, displayAs: 'hello' }; @@ -813,7 +808,7 @@ describe('ModelCompositor', function() { } let reversion = acceptanceTest(englishPunctuation, baseSuggestion, baseContext, postTransform); - assert.equal(reversion.transformId, baseSuggestion.transformId); + assert.equal(reversion.transform.id, baseSuggestion.transform.id); assert.equal(reversion.id, -baseSuggestion.id); // Check #1: Does the returned reversion properly revert the context to its pre-application state? @@ -847,13 +842,12 @@ describe('ModelCompositor', function() { // it('first word of context + postTransform provided, .deleteLeft > 0') // seen earlier in the file. - let baseSuggestion = { + let baseSuggestion: Suggestion = { transform: { insert: 'hello ', deleteLeft: 2, id: 0 }, - transformId: 0, id: 0, displayAs: 'hello' }; @@ -873,7 +867,7 @@ describe('ModelCompositor', function() { let compositor = new ModelCompositor(model, true); let reversion = compositor.acceptSuggestion(baseSuggestion, baseContext, postTransform); - assert.equal(reversion.transformId, baseSuggestion.transformId); + assert.equal(reversion.transform.id, baseSuggestion.transform.id); assert.equal(reversion.id, -baseSuggestion.id); let appliedContext = models.applyTransform(baseSuggestion.transform, baseContext); @@ -887,7 +881,8 @@ describe('ModelCompositor', function() { let expectedTransform = { insert: 'hi', // Keeps current context the same, though it adds a wordbreak. - deleteLeft: 2 + deleteLeft: 2, + id: 0 } assert.deepEqual(suggestions[0].transform, expectedTransform); assert.deepEqual(suggestions[0].appendedTransform, { @@ -923,7 +918,7 @@ describe('ModelCompositor', function() { let baseSuggestion = initialSuggestions[1]; baseSuggestion.appendedTransform.id = 15; // set an id for the applied transform. let reversion = compositor.acceptSuggestion(baseSuggestion, baseContext, postTransform); - assert.equal(reversion.transformId, -baseSuggestion.transformId); + assert.equal(reversion.transform.id, -baseSuggestion.transform.id); assert.equal(reversion.id, -baseSuggestion.id); let appliedContext = models.applyTransform(baseSuggestion.transform, baseContext); @@ -970,7 +965,7 @@ describe('ModelCompositor', function() { assert.equal(compositor.contextTracker.unitTestEndPoints.cache().size, 2); let contextIds = compositor.contextTracker.unitTestEndPoints.cache().keys(); - assert.equal(reversion.transformId, -baseSuggestion.transformId); + assert.equal(reversion.transform.id, -baseSuggestion.transform.id); assert.equal(reversion.id, -baseSuggestion.id); const appliedContextState = compositor.contextTracker.unitTestEndPoints.cache().get(15); From 6d8ae47ab5cc6c5201827f3830710d960340731b Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Fri, 26 Jun 2026 14:27:15 -0500 Subject: [PATCH 04/21] fix(web): drop .transformId refs from engine:main unit tests --- .../engine/predictive-text/templates/common.tests.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/web/src/test/auto/headless/engine/predictive-text/templates/common.tests.ts b/web/src/test/auto/headless/engine/predictive-text/templates/common.tests.ts index 0c7b010586..95db19df8d 100644 --- a/web/src/test/auto/headless/engine/predictive-text/templates/common.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/templates/common.tests.ts @@ -191,7 +191,6 @@ describe('Common utility functions', function() { deleteLeft: 0, id: 0 }, - transitionId: 0, displayAs: 'hello' }; @@ -205,7 +204,6 @@ describe('Common utility functions', function() { deleteLeft: 0, id: 0 }, - transitionId: 0, displayAs: 'hello', p: 0 }; @@ -220,7 +218,6 @@ describe('Common utility functions', function() { deleteLeft: 0, id: 0 }, - transitionId: 0, displayAs: 'hello', p: 0.5 }; @@ -228,14 +225,13 @@ describe('Common utility functions', function() { assert.deepEqual(models.transformToSuggestion(suggestion.transform, 0.5), suggestion); }); - it('properly handles the transitionId', function() { + it('properly handles the transition ID', function() { let suggestion = { transform: { insert: 'hello', deleteLeft: 0, id: 3 }, - transitionId: 3, // Ensures there isn't a separate ID seed in use. displayAs: 'hello' }; From 4fbfcb99402a51437b71cef8c88255506bf07b95 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 30 Jun 2026 10:40:00 -0500 Subject: [PATCH 05/21] change(web): clarify merge, split edit token-mapping types Build-bot: skip build:web Test-bot: skip --- .../src/main/correction/context-token.ts | 4 +- .../main/correction/context-tokenization.ts | 51 +++++++++++++------ 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts index 942773fa57..4d4247a086 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts @@ -10,7 +10,7 @@ import { LexicalModelTypes } from '@keymanapp/common-types'; import { SearchQuotientNode, PathInputProperties } from "./search-quotient-node.js"; -import { TokenSplitMap } from "./context-tokenization.js"; +import { TokenSplitMapping } from "./context-tokenization.js"; import { LegacyQuotientSpur } from "./legacy-quotient-spur.js"; import { LegacyQuotientRoot } from "./legacy-quotient-root.js"; import { generateSubsetId } from './tokenization-subsets.js'; @@ -199,7 +199,7 @@ export class ContextToken { * @param lexicalModel * @returns */ - split(split: TokenSplitMap): ContextToken[] { + split(split: TokenSplitMapping): ContextToken[] { // Split from tail to head - leave as much 'head' intact as possible at each // step, rather than needing to reconstruct the tail multiple times. const splitSpecs = split.matches.slice(); diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index dcd7b0a295..6933e3c836 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -31,7 +31,7 @@ const MIN_CHARS_TO_RECONSIDER_FOR_TOKENIZATION = 8; * This type is used to indicate properties of tokens affected by merges and * splits during a context transition. */ -interface EditTokenMap { +interface EditTokenMappingHalf { /** * The index of the affected token. */ @@ -42,10 +42,31 @@ interface EditTokenMap { text: string }; +/** + * This type is used to indicate properties of tokens affected by splits during + * a context transition. + */ +interface SplitTokenMappingHalf extends EditTokenMappingHalf { + /** + * The index of the affected token. + */ + index: number, + /** + * The token's most likely represented text. + */ + text: string, + + /** + * The codepoint index within the original token at which the split-off token + * begins. + */ + textOffset: number +}; + /** * This type represents mappings for tokens affected by merge edit operations. */ -interface TokenMergeMap { +interface TokenMergeMapping { /** * Entries here represent source-context tokens that are combined as part of * the merge edit operation. @@ -53,7 +74,7 @@ interface TokenMergeMap { * Entries will appear in the same ordering as the codepoints they represent * appear in the underlying context. */ - inputs: EditTokenMap[], + inputs: EditTokenMappingHalf[], /** * This entry represents the post-transition context token produced from the @@ -62,14 +83,14 @@ interface TokenMergeMap { * Note that it is possible for extra codepoints to exist here that were not * represented in the original source-context tokens. */ - match: EditTokenMap + match: EditTokenMappingHalf }; /** * This type represents mappings for tokens affected by split edit operations. */ -export interface TokenSplitMap { +export interface TokenSplitMapping { /** * This entry represents the source-context token that was split as part of * the split edit operation. @@ -77,7 +98,7 @@ export interface TokenSplitMap { * Note that it is possible for codepoints to exist here but not be * represented in the post-transition context tokens resulting from the split. */ - input: EditTokenMap, + input: EditTokenMappingHalf, /** * Entries here represent post-transition tokens that represent pieces of the @@ -86,7 +107,7 @@ export interface TokenSplitMap { * Entries will appear in the same ordering as the codepoints they represent * appear in the underlying context. */ - matches: (EditTokenMap & { textOffset: number })[] + matches: SplitTokenMappingHalf[] }; /** @@ -99,11 +120,11 @@ export interface TransitionEdgeAlignment { /** * Denotes any token merge edits needed after applying the Transform. */ - merges: TokenMergeMap[]; + merges: TokenMergeMapping[]; /** * Denotes any token split edits needed after applying the Transform. */ - splits: TokenSplitMap[]; + splits: TokenSplitMapping[]; /** * Denotes any further token edits needed that cannot be attributed to * 'merge's, 'split's, or edits from the input `Transform`. @@ -1036,13 +1057,13 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo * generality, if two separate groups of tokens are merged, two groups will be * defined - one for each token resulting from a merge. */ - merges: TokenMergeMap[], + merges: TokenMergeMapping[], /** * Indicates groupings of directly related splits. Without loss of * generality, if two separate tokens are split, two groups will be defined - * one for each source token split. */ - splits: TokenSplitMap[] + splits: TokenSplitMapping[] } { // We've found the root token to which changes may apply. // We've found the last post-application token to which transform changes contributed. @@ -1080,8 +1101,8 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo const mappedPath: EditTuple[] = []; let mergeOffset = 0; let splitOffset = 0; - const merges: TokenMergeMap[] = []; - const splits: TokenSplitMap[] = []; + const merges: TokenMergeMapping[] = []; + const splits: TokenSplitMapping[] = []; while(queueIndex < editPath.length) { const edit = editPath[queueIndex]; const { input, match } = edit; @@ -1091,7 +1112,7 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo let matchOffset: number = 0; if(op == 'merge') { const mergeTarget = resultTokenization[match]; - const merge: TokenMergeMap = { + const merge: TokenMergeMapping = { match: { index: match, text: mergeTarget @@ -1122,7 +1143,7 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo merges.push(merge); } else if(op == 'split') { const splitTarget = preTokenization[input]; - const split: TokenSplitMap = { + const split: TokenSplitMapping = { input: { index: input, text: splitTarget From 6f72c2f75c59298865205224ab4698ff2511a7fa Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 6 Jul 2026 08:28:39 -0500 Subject: [PATCH 06/21] change(web): drops repeated fields from extended class --- .../src/main/correction/context-tokenization.ts | 9 --------- 1 file changed, 9 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 6933e3c836..9e130f6caa 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -47,15 +47,6 @@ interface EditTokenMappingHalf { * a context transition. */ interface SplitTokenMappingHalf extends EditTokenMappingHalf { - /** - * The index of the affected token. - */ - index: number, - /** - * The token's most likely represented text. - */ - text: string, - /** * The codepoint index within the original token at which the split-off token * begins. From cfd731511a77bcc20bfd16e00687d0a2859522f2 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 6 Jul 2026 08:43:09 -0500 Subject: [PATCH 07/21] fix(web): adjust unit tests that previously auto-selected keep --- .../prediction-helpers/auto-correct.tests.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts index d32326e843..882390591a 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts @@ -16,7 +16,7 @@ describe('predictionAutoSelect', () => { assert.sameDeepOrderedMembers(predictions, originalPredictions); }); - it(`selects solitary 'keep' suggestion that does match the model`, () => { + it(`selects nothing if solitary 'keep' suggestion does match the model`, () => { const predictions: CorrectionPredictionTuple[] = [ { correction: { @@ -44,7 +44,7 @@ describe('predictionAutoSelect', () => { assert.sameDeepOrderedMembers(predictions, originalPredictions); const autoselected = predictions.find((entry) => entry.prediction.sample.autoAccept); - assert.isOk(autoselected); + assert.isNotOk(autoselected); }); it(`does not select suggestions if the root correction has no letters`, () => { @@ -127,7 +127,7 @@ describe('predictionAutoSelect', () => { assert.isNotOk(autoselected); }); - it(`selects 'keep' suggestion that does match the model over any alternatives`, () => { + it(`selects nothing for 'keep' suggestion that does match the model even with alternatives`, () => { const keepSuggestion: CorrectionPredictionTuple = { correction: { sample: 'thin', @@ -210,7 +210,7 @@ describe('predictionAutoSelect', () => { assert.sameDeepMembers(predictions, originalPredictions); const autoselected = predictions.find((entry) => entry.prediction.sample.autoAccept); - assert.equal(autoselected, keepSuggestion); + assert.isNotOk(autoselected); }); it(`selects solitary non-'keep' suggestion when 'keep' does not match model`, () => { From 94e67290e11bbe32fee59eb44a29800b21a8ceae Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 6 Jul 2026 10:07:15 -0500 Subject: [PATCH 08/21] change(web): stop negation of reversion transition IDs Build-bot: skip build:web Test-bot: skip --- .../worker-thread/src/main/model-compositor.ts | 12 +++--------- .../src/main/headless/languageProcessor.ts | 5 +---- ...ermine-suggestion-context-transition.tests.ts | 10 +++++----- .../worker-model-compositor.tests.ts | 16 ++++++++++------ 4 files changed, 19 insertions(+), 24 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts index 64d6ae4e20..5e43a14728 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts @@ -239,7 +239,7 @@ export class ModelCompositor { // Step 1: re-use the original input Transform as the reversion's Transform. // The Web engine will restore the original state of the context before accepting // and before reverting; all we need to do is put the original keystroke back in place. - let reversionTransform: Transform = originalInput ?? { insert: '', deleteLeft: 0 }; + let reversionTransform: Transform = originalInput ?? { insert: '', deleteLeft: 0, id: suggestion.transform.id }; // clone the transform to prevent aliasing reversionTransform = {...reversionTransform}; @@ -265,9 +265,6 @@ export class ModelCompositor { // Since we're outside of the standard `predict` control path, we'll need to // set the Reversion's ID directly. let reversion = toAnnotatedSuggestion(this.lexicalModel, firstConversion, 'revert'); - if(suggestion.transform.id != null) { - reversion.transform.id = -suggestion.transform.id; - } if(suggestion.id != null) { // Since a reversion inverts its source suggestion, we set its ID to be the // additive inverse of the source suggestion's ID. Makes easy mapping / @@ -331,11 +328,8 @@ export class ModelCompositor { let compositor = this; let suggestions: Promise; let fallbackSuggestions = async function() { - const suggestions = await compositor.predict({insert: '', deleteLeft: 0}, context); + const suggestions = await compositor.predict({insert: '', deleteLeft: 0, id: reversion.transform.id}, context); suggestions.forEach(function(suggestion) { - // A reversion's transform ID is the additive inverse of its original suggestion; - // we revert to the state of said original suggestion. - suggestion.transform.id = -reversion.transform.id; // Prevent auto-selection of any suggestion immediately after a reversion. // It's fine after at least one keystroke, but not before. suggestion.autoAccept = false; @@ -352,7 +346,7 @@ export class ModelCompositor { // Note that the base reversion's .transform.id will predate the appendedTransform id // used to add whitespace (if one existed), so reverting to the base ID's associated // context also reverts the appendedTransform. - const baseTransitionId = -reversion.transform.id; + const baseTransitionId = reversion.transform.id; let originalTransition = this.contextTracker.findAndRevert(baseTransitionId); if(appendedOnly) { diff --git a/web/src/engine/src/main/headless/languageProcessor.ts b/web/src/engine/src/main/headless/languageProcessor.ts index 3130fcaf27..4436b8d2f5 100644 --- a/web/src/engine/src/main/headless/languageProcessor.ts +++ b/web/src/engine/src/main/headless/languageProcessor.ts @@ -306,10 +306,7 @@ export class LanguageProcessor extends EventEmitter { // Find the state of the context at the time the suggestion was generated. // This may refer to the context before an input keystroke or before application // of a predictive suggestion. - // - // Reversions use the additive inverse of the id token of the Transcription being - // reverted to. - const reversionId = appendedOnly ? reversion.appendedTransform.id : -reversion.transform.id; + const reversionId = appendedOnly ? reversion.appendedTransform.id : reversion.transform.id; const original = this.getPredictionState(reversionId); if(!original) { console.warn("Could not apply the Suggestion!"); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts index 0be3b6525e..e911021537 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts @@ -150,7 +150,7 @@ describe('determineContextTransition', () => { } }; - compositor.acceptSuggestion(applied_testing, baseContext, { insert: '', deleteLeft: 0 }); + compositor.acceptSuggestion(applied_testing, baseContext, { insert: '', deleteLeft: 0, id: applied_testing.transform.id }); const acceptingTransition = tracker.latest; const inputDistribution: Distribution = [{sample: applied_testing.appendedTransform, p: 1}]; @@ -203,7 +203,7 @@ describe('determineContextTransition', () => { }; baseTransition.final.suggestions = [pred_testing]; - compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0 }); + compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0, id: pred_testing.transform.id }); const inputDistribution: Distribution = [{sample: { insert: 'a', deleteLeft: 0, id: 5 }, p: 1}]; @@ -262,7 +262,7 @@ describe('determineContextTransition', () => { }; baseTransition.final.suggestions = [pred_testing]; - compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0 }); + compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0, id: pred_testing.transform.id }); const inputDistribution: Distribution = [{sample: { insert: 'a', deleteLeft: 0, id: 5 }, p: 1}]; @@ -323,7 +323,7 @@ describe('determineContextTransition', () => { }; baseTransition.final.suggestions = [pred_testing]; - compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0 }); + compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0, id: pred_testing.transform.id }); const inputDistribution: Distribution = [{sample: { insert: 'a', deleteLeft: 0, id: 5 }, p: 1}]; @@ -396,7 +396,7 @@ describe('determineContextTransition', () => { }; baseTransition.final.suggestions = [pred_testing]; - compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0 }); + compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0, id: pred_testing.transform.id }); const inputDistribution: Distribution = [{sample: { insert: 'a', deleteLeft: 0, id: 5 }, p: 1}]; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts index 5343760d56..4acc343f98 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts @@ -685,7 +685,8 @@ describe('ModelCompositor', function() { // of the Context when the suggestion is built. let postTransform = { insert: 'l', - deleteLeft: 0 + deleteLeft: 0, + id: baseSuggestion.transform.id } let reversion = acceptanceTest(englishPunctuation, baseSuggestion, baseContext, postTransform); @@ -727,7 +728,8 @@ describe('ModelCompositor', function() { // of the Context when the suggestion is built. let postTransform = { insert: 'l', - deleteLeft: 0 + deleteLeft: 0, + id: baseSuggestion.id } let reversion = acceptanceTest(englishPunctuation, baseSuggestion, baseContext, postTransform); @@ -804,7 +806,8 @@ describe('ModelCompositor', function() { // of the Context when the suggestion is built. let postTransform = { insert: 'i', - deleteLeft: 1 + deleteLeft: 1, + id: baseSuggestion.transform.id } let reversion = acceptanceTest(englishPunctuation, baseSuggestion, baseContext, postTransform); @@ -860,7 +863,8 @@ describe('ModelCompositor', function() { // of the Context when the suggestion is built. let postTransform = { insert: 'i', - deleteLeft: 1 + deleteLeft: 1, + id: baseSuggestion.transform.id } let model = new models.DummyModel({punctuation: englishPunctuation}); @@ -918,7 +922,7 @@ describe('ModelCompositor', function() { let baseSuggestion = initialSuggestions[1]; baseSuggestion.appendedTransform.id = 15; // set an id for the applied transform. let reversion = compositor.acceptSuggestion(baseSuggestion, baseContext, postTransform); - assert.equal(reversion.transform.id, -baseSuggestion.transform.id); + assert.equal(reversion.transform.id, baseSuggestion.transform.id); assert.equal(reversion.id, -baseSuggestion.id); let appliedContext = models.applyTransform(baseSuggestion.transform, baseContext); @@ -965,7 +969,7 @@ describe('ModelCompositor', function() { assert.equal(compositor.contextTracker.unitTestEndPoints.cache().size, 2); let contextIds = compositor.contextTracker.unitTestEndPoints.cache().keys(); - assert.equal(reversion.transform.id, -baseSuggestion.transform.id); + assert.equal(reversion.transform.id, baseSuggestion.transform.id); assert.equal(reversion.id, -baseSuggestion.id); const appliedContextState = compositor.contextTracker.unitTestEndPoints.cache().get(15); From 267cac21c2efd10c40790b81216cc787f8d86e51 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 6 Jul 2026 17:50:14 -0500 Subject: [PATCH 09/21] fix(web): refresh model when "Enable predictions" is toggled --- .../Classes/Settings/LanguageSettingsViewController.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ios/engine/KMEI/KeymanEngine/Classes/Settings/LanguageSettingsViewController.swift b/ios/engine/KMEI/KeymanEngine/Classes/Settings/LanguageSettingsViewController.swift index 816a7aea2c..1593c7d3b3 100644 --- a/ios/engine/KMEI/KeymanEngine/Classes/Settings/LanguageSettingsViewController.swift +++ b/ios/engine/KMEI/KeymanEngine/Classes/Settings/LanguageSettingsViewController.swift @@ -119,6 +119,8 @@ class LanguageSettingsViewController: UITableViewController { self.doAutocorrectionsSwitch?.isHidden = !(value && mayCorrect) self.doAutocorrectionsLabel?.isEnabled = value && mayCorrect self.correctionsCell?.isUserInteractionEnabled = value + + refreshModelIfNeeded() } @objc From 21c18a4ca5e55adebcbbfef5fe00c19606e3d690 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 9 Jul 2026 00:51:10 +0700 Subject: [PATCH 10/21] change(web): Apply suggestion from code review Co-authored-by: Marc Durdin --- .../worker-thread/src/main/model-compositor.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts index 5e43a14728..8e12b96ed4 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts @@ -239,9 +239,9 @@ export class ModelCompositor { // Step 1: re-use the original input Transform as the reversion's Transform. // The Web engine will restore the original state of the context before accepting // and before reverting; all we need to do is put the original keystroke back in place. - let reversionTransform: Transform = originalInput ?? { insert: '', deleteLeft: 0, id: suggestion.transform.id }; - // clone the transform to prevent aliasing - reversionTransform = {...reversionTransform}; + let reversionTransform: Transform = originalInput + ? { ...originalInput } + : { insert: '', deleteLeft: 0, id: suggestion.transform.id }; // Step 2: building the proper 'displayAs' string for the Reversion const postContext = originalInput ? models.applyTransform(originalInput, context) : context; From 86cf687519e9dc89041be2988ca5757face7eaf2 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 9 Jul 2026 09:57:42 -0500 Subject: [PATCH 11/21] fix(web): correct casing of autocorrect permission property name --- .../resources/Keyman.bundle/Contents/Resources/ios-host.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ios/engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/ios-host.js b/ios/engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/ios-host.js index afb561b176..fb5c4c67cb 100644 --- a/ios/engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/ios-host.js +++ b/ios/engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/ios-host.js @@ -331,7 +331,7 @@ function enableSuggestions(model, mayPredict, mayCorrect, mayAutocorrect) { // the moment we actually register the new model. keyman.core.languageProcessor.mayPredict = mayPredict; keyman.core.languageProcessor.mayCorrect = mayCorrect; - keyman.core.languageProcessor.mayAutocorrect = mayAutocorrect; + keyman.core.languageProcessor.mayAutoCorrect = mayAutocorrect; keyman.addModel(model); } From f1a6f6efdf5bccaa9eb6a7a13029f8ea1edd9af4 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 9 Jul 2026 10:37:42 -0500 Subject: [PATCH 12/21] change(ios): apply suggestions from review --- .../LanguageSettingsViewController.swift | 29 ++++++++++--------- .../KeymanEngine/en.lproj/Localizable.strings | 6 ++-- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/ios/engine/KMEI/KeymanEngine/Classes/Settings/LanguageSettingsViewController.swift b/ios/engine/KMEI/KeymanEngine/Classes/Settings/LanguageSettingsViewController.swift index 1593c7d3b3..40968726cf 100644 --- a/ios/engine/KMEI/KeymanEngine/Classes/Settings/LanguageSettingsViewController.swift +++ b/ios/engine/KMEI/KeymanEngine/Classes/Settings/LanguageSettingsViewController.swift @@ -105,21 +105,26 @@ class LanguageSettingsViewController: UITableViewController { } } + func refreshSwitchVisibility() { + let userDefaults = Storage.active.userDefaults + + let mayCorrect = userDefaults.correctSettingForLanguage(languageID: self.language.id) + self.doCorrectionsSwitch?.isHidden = !mayCorrect + self.doCorrectionsLabel?.isEnabled = mayCorrect + + let mayAutoCorrect = userDefaults.autocorrectSettingForLanguage(languageID: self.language.id) + self.doAutocorrectionsSwitch?.isHidden = !(mayCorrect && mayAutoCorrect) + self.doAutocorrectionsLabel?.isEnabled = mayCorrect && mayAutoCorrect + self.correctionsCell?.isUserInteractionEnabled = mayCorrect + } + @objc func predictionSwitchValueChanged(source: UISwitch) { let value = source.isOn; let userDefaults = Storage.active.userDefaults userDefaults.set(predictSetting: value, forLanguageID: self.language.id) - // Reactively set the corrections switch interactivity state. - self.doCorrectionsSwitch?.isHidden = !value - self.doCorrectionsLabel?.isEnabled = value - - let mayCorrect = userDefaults.autocorrectSettingForLanguage(languageID: self.language.id) - self.doAutocorrectionsSwitch?.isHidden = !(value && mayCorrect) - self.doAutocorrectionsLabel?.isEnabled = value && mayCorrect - self.correctionsCell?.isUserInteractionEnabled = value - + refreshSwitchVisibility() refreshModelIfNeeded() } @@ -129,11 +134,7 @@ class LanguageSettingsViewController: UITableViewController { let userDefaults = Storage.active.userDefaults userDefaults.set(correctSetting: value, forLanguageID: self.language.id) - // This may only be triggered if the predict toggle is on, - // so we can rely on just the input value. - self.doAutocorrectionsSwitch?.isHidden = !value - self.doAutocorrectionsLabel?.isEnabled = value - + refreshSwitchVisibility() refreshModelIfNeeded() } diff --git a/ios/engine/KMEI/KeymanEngine/en.lproj/Localizable.strings b/ios/engine/KMEI/KeymanEngine/en.lproj/Localizable.strings index 0e5b70769b..e45e70f853 100644 --- a/ios/engine/KMEI/KeymanEngine/en.lproj/Localizable.strings +++ b/ios/engine/KMEI/KeymanEngine/en.lproj/Localizable.strings @@ -152,13 +152,13 @@ "menu-langsettings-title" = "%@ Settings"; /* Label for the toggle that enables corrections that is displayed within a language-specific settings menu */ -"menu-langsettings-toggle-correct" = "Enable corrections"; +"menu-langsettings-toggle-correct" = "Offer corrections"; /* Label for the toggle that enables auto-corrections that is displayed within a language-specific settings menu */ -"menu-langsettings-toggle-autocorrect" = "Enable autocorrections"; +"menu-langsettings-toggle-autocorrect" = "Apply corrections automatically"; /* Label for the toggle that enables predictions that is displayed within a language-specific settings menu */ -"menu-langsettings-toggle-predict" = "Enable predictions"; +"menu-langsettings-toggle-predict" = "Offer word completions"; /* Help message for a prompt that appears for confirming a lexical model download: language (1): lexical model (dictionary) name (2) */ "menu-lexical-model-install-message" = "Would you like to install this dictionary?"; From ccb2e7057a317b307bed72a91d896bb65a30f78c Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 9 Jul 2026 10:48:00 -0500 Subject: [PATCH 13/21] fix(ios): further adjust + fix language toggle logic --- .../LanguageSettingsViewController.swift | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/ios/engine/KMEI/KeymanEngine/Classes/Settings/LanguageSettingsViewController.swift b/ios/engine/KMEI/KeymanEngine/Classes/Settings/LanguageSettingsViewController.swift index 40968726cf..8ee69ed231 100644 --- a/ios/engine/KMEI/KeymanEngine/Classes/Settings/LanguageSettingsViewController.swift +++ b/ios/engine/KMEI/KeymanEngine/Classes/Settings/LanguageSettingsViewController.swift @@ -108,14 +108,15 @@ class LanguageSettingsViewController: UITableViewController { func refreshSwitchVisibility() { let userDefaults = Storage.active.userDefaults - let mayCorrect = userDefaults.correctSettingForLanguage(languageID: self.language.id) - self.doCorrectionsSwitch?.isHidden = !mayCorrect - self.doCorrectionsLabel?.isEnabled = mayCorrect + let mayPredict = userDefaults.predictSettingForLanguage(languageID: self.language.id) + self.doCorrectionsSwitch?.isHidden = !mayPredict + self.doCorrectionsLabel?.isEnabled = mayPredict + self.correctionsCell?.isUserInteractionEnabled = mayPredict - let mayAutoCorrect = userDefaults.autocorrectSettingForLanguage(languageID: self.language.id) - self.doAutocorrectionsSwitch?.isHidden = !(mayCorrect && mayAutoCorrect) - self.doAutocorrectionsLabel?.isEnabled = mayCorrect && mayAutoCorrect - self.correctionsCell?.isUserInteractionEnabled = mayCorrect + let mayCorrect = userDefaults.correctSettingForLanguage(languageID: self.language.id) + self.doAutocorrectionsSwitch?.isHidden = !(mayPredict && mayCorrect) + self.doAutocorrectionsLabel?.isEnabled = mayPredict && mayCorrect + self.autocorrectionsCell?.isUserInteractionEnabled = mayPredict && mayCorrect } @objc @@ -195,9 +196,7 @@ class LanguageSettingsViewController: UITableViewController { selector: #selector(self.correctionSwitchValueChanged) ) - // Disable interactivity if the prediction toggle is set to 'off'. - doCorrectionsSwitch!.isHidden = !userDefaults.predictSettingForLanguage(languageID: self.language.id) - cell.isUserInteractionEnabled = userDefaults.predictSettingForLanguage(languageID: self.language.id) + refreshSwitchVisibility() } else if 2 == indexPath.row { autocorrectionsCell = cell doAutocorrectionsSwitch = UISwitch() @@ -209,11 +208,7 @@ class LanguageSettingsViewController: UITableViewController { selector: #selector(self.autocorrectionSwitchValueChanged) ) - // Disable interactivity if the prediction or correction toggle is set to 'off'. - let mayPredict = userDefaults.predictSettingForLanguage(languageID: self.language.id) - let mayCorrect = userDefaults.correctSettingForLanguage(languageID: self.language.id) - doAutocorrectionsSwitch!.isHidden = !(mayPredict && mayCorrect) - cell.isUserInteractionEnabled = mayPredict && mayCorrect + refreshSwitchVisibility() } else { // rows 3 and 4 cell.accessoryType = .disclosureIndicator } From 13af7821c3ec3a1742ffd2901b58ec938bbd4467 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 20 Jul 2026 13:17:05 -0500 Subject: [PATCH 14/21] fix(web): prevent backspace from clustering with output keys --- .../src/main/correction/tokenization-subsets.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-subsets.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-subsets.ts index bfbadef182..a8617a7c0f 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-subsets.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-subsets.ts @@ -135,7 +135,8 @@ export function legacySubsetKeyer(tokenizationEdits: TokenizationTransitionEdits // Now, based on the transform tokenization. We want to force uniqueness for // all variations of result length on each tokenized transform resulting from // the precomputation's represented keystroke. - for(const {0: relativeIndex} of tokenizedTransform.entries()) { + for(const {0: relativeIndex, 1: transform} of tokenizedTransform.entries()) { + const insertLen = KMWString.length(transform.insert); if(relativeIndex > 0) { // The true boundary lie before the insert if the value is non-zero; // don't differentiate here! @@ -148,10 +149,10 @@ export function legacySubsetKeyer(tokenizationEdits: TokenizationTransitionEdits // // IMPORTANT: update unit tests manually if the BI marker here changes // or the use of SENTINEL_CODE_UNIT as a key component separator changes. - components.push(`BI@${relativeIndex}`); + components.push(`BI@${relativeIndex}-${boundaryTextLen + insertLen}`); boundaryTextLen = 0; } else { - components.push(`I@${relativeIndex}`); + components.push(`I@${relativeIndex}-${boundaryTextLen + insertLen}`); } } From 5a1e23d72711275122b5d303bccd83dd1978b84b Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 21 Jul 2026 13:39:11 -0500 Subject: [PATCH 15/21] fix(web): handle prediction whitespace-input, backspace-input edge cases better Fixes: #16271 Fixes: #16272 Build-bot: skip release:web,android,ios --- .../src/main/correction/context-tokenization.ts | 13 ++++++++++++- .../worker-thread/src/main/predict-helpers.ts | 14 +++++++++----- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 52c365dd7e..4ccd8cb86d 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -708,8 +708,19 @@ export class ContextTokenization { affectedToken = null; } + // Backspace handling - emptying context via backspace or erasing _part_ of + // a whitespace token can erase the tokenization-final empty token usually + // used for word-initial suggestions. + // + // We re-add it here so that suggestions can be presented to the user as + // normal. + const tokenSequence = this.tokens.slice(0, sliceIndex).concat(tailTokenization); + if(tokenSequence.length == 0 || tokenSequence[tokenSequence.length - 1]?.isWhitespace) { + tokenSequence.push(new ContextToken(new LegacyQuotientRoot(lexicalModel))); + } + return new ContextTokenization( - this.tokens.slice(0, sliceIndex).concat(tailTokenization), + tokenSequence, null, determineTaillessTrueKeystroke(transitionEdge) ); diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index ac0de6c503..665db610ef 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -578,11 +578,6 @@ export async function correctAndEnumerate( // Corrections obtained: now to predict from them! const tokenization = tokenizations.find(t => t.spaceId == match.spaceId); - // If our 'match' results in fully deleting the new token, reject it and try again. - if(match.matchSequence.length == 0 && match.inputSequence.length != 0) { - continue; - } - // If our 'match' fully replaces the token, reject it and try again. if(match.matchSequence.length != 0 && match.matchSequence.length == match.knownCost) { continue; @@ -609,6 +604,12 @@ export async function correctAndEnumerate( const predictions = buildAndMapPredictions(transition, tokenization, match, costFactor); + // Backspaces that cause the whole context to become empty do not reflect the backspace transform + // within the search space. We correct for that here. + if(tokenization.tail.searchModule.codepointLength == 0 && inputTransform.deleteLeft > 0) { + predictions.forEach((p) => p.prediction.sample.transform.deleteLeft += inputTransform.deleteLeft); + } + // Only set 'best correction' cost when a correction ACTUALLY YIELDS predictions. if(predictions.length > 0 && bestCorrectionCost === undefined) { bestCorrectionCost = match.totalCost * costFactor; @@ -1077,6 +1078,9 @@ export function finalizeSuggestions( if(presDL > 0) { mergedTransform.deleteLeft -= presDL; } + if(prediction.sample.transform.id !== undefined) { + mergedTransform.id = prediction.sample.transform.id; + } // Temporarily and locally drops 'readonly' semantics so that we can reassign the transform. // See https://www.typescriptlang.org/docs/handbook/release-notes/typescript-2-8.html#improved-control-over-mapped-type-modifiers From 26703a4e6d097264dd64033600be9b638e6e2728 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Wed, 22 Jul 2026 15:39:53 -0500 Subject: [PATCH 16/21] change(web): double the permitted transcription-cache size This will help in cases where a user wishes to revert a suggestion after numerous other edits. --- web/src/engine/src/main/headless/transcriptionCache.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/engine/src/main/headless/transcriptionCache.ts b/web/src/engine/src/main/headless/transcriptionCache.ts index 4a3e6408a6..7ff3832b9b 100644 --- a/web/src/engine/src/main/headless/transcriptionCache.ts +++ b/web/src/engine/src/main/headless/transcriptionCache.ts @@ -1,7 +1,7 @@ import { Transcription } from "keyman/engine/keyboard"; import { RewindableCache } from "keyman/common/web-utils"; -const TRANSCRIPTION_BUFFER_SIZE = 10; +const TRANSCRIPTION_BUFFER_SIZE = 20; export class TranscriptionCache extends RewindableCache { constructor() { From e225b26f10b7c2a0bc9fcbd6a429d4ff978e6bfd Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Thu, 23 Jul 2026 15:52:53 -0500 Subject: [PATCH 17/21] change(web): add unit tests for issues fixed by this PR --- .../worker-thread/src/main/predict-helpers.ts | 17 +- .../context/context-tokenization.tests.ts | 37 +++ .../context/tokenization-subsets.tests.ts | 51 +++- .../build-and-map-predictions.tests.ts | 220 ++++++++++++++++++ 4 files changed, 316 insertions(+), 9 deletions(-) create mode 100644 web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/build-and-map-predictions.tests.ts diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 665db610ef..5aa73200ee 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -13,7 +13,6 @@ import { ContextTransition } from './correction/context-transition.js'; import { ExecutionTimer } from './correction/execution-timer.js'; import { ModelCompositor } from './model-compositor.js'; import { getBestTokenMatches } from './correction/distance-modeler.js'; -import { TokenResultMapping } from './correction/token-result-mapping.js'; import CasingForm = LexicalModelTypes.CasingForm; import Context = LexicalModelTypes.Context; @@ -459,7 +458,8 @@ export function determineSuggestionRange( export function buildAndMapPredictions( transition: ContextTransition, tokenization: ContextTokenization, - match: Readonly, + // Originally, Readonly - but we only need these two components here. + match: Readonly<{matchString: string, totalCost: number}>, costFactor: number ): CorrectionPredictionTuple[] { const model = transition.final.model; @@ -493,6 +493,13 @@ export function buildAndMapPredictions( // entry.baseTokenization = transition.final.tokenizationSourceMap.get(tokenization); }); + // Backspaces that shorten a multi-codepoint whitespace token are not handled well by default. + // As a new empty token is placed at the end for such cases, we can detect and handle such cases. + const inputTransform = transition.inputDistribution?.[0].sample ?? { insert: '', deleteLeft: 0 }; + if(tokenization.tokens.length > 1 && tokenization.tail.searchModule.codepointLength == 0 && inputTransform.deleteLeft > 0) { + predictions.forEach((p) => p.prediction.sample.transform.deleteLeft += inputTransform.deleteLeft); + } + return predictions; } @@ -604,12 +611,6 @@ export async function correctAndEnumerate( const predictions = buildAndMapPredictions(transition, tokenization, match, costFactor); - // Backspaces that cause the whole context to become empty do not reflect the backspace transform - // within the search space. We correct for that here. - if(tokenization.tail.searchModule.codepointLength == 0 && inputTransform.deleteLeft > 0) { - predictions.forEach((p) => p.prediction.sample.transform.deleteLeft += inputTransform.deleteLeft); - } - // Only set 'best correction' cost when a correction ACTUALLY YIELDS predictions. if(predictions.length > 0 && bestCorrectionCost === undefined) { bestCorrectionCost = match.totalCost * costFactor; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index dcef6f8f61..8f93bd20f4 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -397,6 +397,43 @@ describe('ContextTokenization', function() { ); }); + it('handles simple case - deletion of final context content via backspace', () => { + const baseTokens = ['a']; + const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t))); + + const targetTokens = [''].map((t) => ({text: t, isWhitespace: t == ' '})); + const inputTransform = { insert: '', deleteLeft: 1, deleteRight: 0, id: 42 }; + const inputTransformMap: Map = new Map(); + inputTransformMap.set(0, { insert: '', deleteLeft: 1, id: 42 }); + + const edgeWindow = buildEdgeWindow(baseTokenization.tokens, inputTransform, false, testEdgeWindowSpec); + const tokenization = baseTokenization.evaluateTransition({ + alignment: { + merges: [], + splits: [], + unmappedEdits: [], + edgeWindow: { + ...edgeWindow, + // The range within the window constructed by the prior call for its parameterization. + // Any adjustments on the boundary token itself are included here. + retokenization: [...targetTokens.slice(edgeWindow.sliceIndex).map(t => t.text)] + }, + removedTokenCount: 1 + }, + inputs: [{ sample: inputTransformMap, p: 1 }], + inputSubsetId: generateSubsetId() + }, + inputTransform.id, + 1 + ); + + assert.isOk(tokenization); + assert.equal(tokenization.tokens.length, targetTokens.length); + assert.deepEqual(tokenization.tokens.map((t) => ({text: t.exampleInput, isWhitespace: t.isWhitespace})), + targetTokens + ); + }); + it('handles simple case - new character added to last token', () => { const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'da']; const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t))); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts index 04647123c0..b924d9422c 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts @@ -21,6 +21,7 @@ import { ContextToken, ContextTokenization, generateSubsetId, + legacySubsetKeyer, models, precomputationSubsetKeyer, TokenizationTransitionEdits, @@ -31,7 +32,7 @@ import Distribution = LexicalModelTypes.Distribution; import Transform = LexicalModelTypes.Transform; import TrieModel = models.TrieModel; -var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'), +const plainModel = new TrieModel(jsonFixture('models/tries/english-1000'), {wordBreaker: defaultBreaker}); function toToken(text: string) { @@ -41,6 +42,54 @@ function toToken(text: string) { return token; } +describe('legacySubsetKeyer', () => { + it('does not map backspace inputs to same result as standard key inputs', () => { + const appleToken = ContextToken.fromRawText(plainModel, 'apple', true); + + const bksp = { insert: '', deleteLeft: 1, deleteRight: 0, id: 3 }; + const bkspKey = legacySubsetKeyer({ + alignment: { + merges: [], + splits: [], + unmappedEdits: [], + edgeWindow: { + ...buildEdgeWindow([appleToken], bksp, false), + retokenization: ['appl'], + retokenizationText: 'appl' + }, + removedTokenCount: 0 + }, + tokenizedTransform: (() => { + const map: Map = new Map(); + map.set(0, bksp); + return map; + })() + }); + + const input = { insert: 's', deleteLeft: 0, deleteRight: 0, id: bksp.id }; + const inputKey = legacySubsetKeyer({ + alignment: { + merges: [], + splits: [], + unmappedEdits: [], + edgeWindow: { + ...buildEdgeWindow([appleToken], input, false), + retokenization: ['apples'], + retokenizationText: 'apples' + }, + removedTokenCount: 0 + }, + tokenizedTransform: (() => { + const map: Map = new Map(); + map.set(0, input); + return map; + })() + }); + + assert.notEqual(bkspKey, inputKey); + }); +}); + describe('precomputationSubsetKeyer', function() { it("safely generates keys for empty transition + empty contexts", () => { const rawTextTokens = ['']; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/build-and-map-predictions.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/build-and-map-predictions.tests.ts new file mode 100644 index 0000000000..73d8276e13 --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/build-and-map-predictions.tests.ts @@ -0,0 +1,220 @@ + +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2026-07-23 + * + * This file contains tests designed to validate the behavior of the + * buildAndMapPredictions helper function class and its integration with the + * lower-level predictive-text helpers. + */ + + +import { assert } from 'chai'; + +import { default as defaultBreaker } from '@keymanapp/models-wordbreakers'; +import { LexicalModelTypes } from '@keymanapp/common-types'; +import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; +import { TrieModel } from '@keymanapp/models-templates'; + +import { buildAndMapPredictions, buildEdgeWindow, ContextState, ContextToken, ContextTokenization, ContextTransition, generateSubsetId, LegacyQuotientRoot, LegacyQuotientSpur, models, predictFromCorrections } from "@keymanapp/lm-worker/test-index"; + +import Context = LexicalModelTypes.Context; +import Distribution = LexicalModelTypes.Distribution; +import ProbabilityMass = LexicalModelTypes.ProbabilityMass; +import Transform = LexicalModelTypes.Transform; + +const plainModel = new TrieModel(jsonFixture('models/tries/english-1000'), + {wordBreaker: defaultBreaker}); + +describe('buildAndMapPredictions', () => { + it('adds the preservation transform to all generated predictions', () => { + const context: Context = { + left: 'th', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const correctionDistribution: Distribution = [{ + sample: { + insert: 'e', + deleteLeft: 0 + }, + p: 0.6 + } + ]; + + const basePredictions = predictFromCorrections(plainModel, correctionDistribution, context); + basePredictions.forEach((entry) => assert.isNotOk(entry.preservationTransform)); + + // must construct the taillessTrueKeystroke appropriately. + const tailless = { insert: 'TEST', deleteLeft: 0 }; + const tokenization = new ContextTokenization([ContextToken.fromRawText(plainModel, 'th', true)], null, tailless); + const transition = new ContextTransition(new ContextState(context, plainModel, tokenization), 0); + + const targetTokenization = new ContextTokenization([new ContextToken(new LegacyQuotientSpur(tokenization.tail.searchModule, correctionDistribution, correctionDistribution[0]))]); + transition.finalize(new ContextState(models.applyTransform(correctionDistribution[0].sample, context), plainModel, targetTokenization), correctionDistribution); + + const mappedPredictions = buildAndMapPredictions( + transition, + transition.base.displayTokenization, + {matchString: 'the', totalCost: 0}, + 1 + ); + + assert.deepEqual(mappedPredictions.map((tuple) => tuple.prediction), basePredictions.map((tuple) => tuple.prediction)); + mappedPredictions.forEach((tuple) => assert.isOk(tuple.preservationTransform)); + mappedPredictions.forEach((tuple) => tuple.preservationTransform == tailless); + }); + + it('properly handles empty prediction roots from deleted same-token codepoints', () => { + const context: Context = { + left: 'the a', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const correctionDistribution: Distribution = [{ + sample: { + insert: '', + deleteLeft: 1 + }, + p: 1 + } + ]; + + const basePredictions = predictFromCorrections(plainModel, correctionDistribution, context); + + // must construct the taillessTrueKeystroke appropriately. + const tokenization = new ContextTokenization([ + ContextToken.fromRawText(plainModel, 'the', false), + ContextToken.fromRawText(plainModel, ' ', false), + ContextToken.fromRawText(plainModel, 'a', true) + ]); + const transition = new ContextTransition(new ContextState(context, plainModel, tokenization), 0); + + const targetTokenization = new ContextTokenization([ + tokenization.tokens[0], + tokenization.tokens[1], + new ContextToken(new LegacyQuotientRoot(plainModel)) + ]); + transition.finalize(new ContextState(models.applyTransform(correctionDistribution[0].sample, context), plainModel, targetTokenization), correctionDistribution); + + const mappedPredictions = buildAndMapPredictions( + transition, + transition.base.displayTokenization, + {matchString: '', totalCost: 0}, + 1 + ); + + assert.deepEqual(mappedPredictions.map((tuple) => tuple.prediction), basePredictions.map((tuple) => tuple.prediction)); + }); + + it('properly handles empty prediction roots caused by backspacing one of multiple spaces', () => { + const context: Context = { + left: 'the ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const correctionDistribution: Distribution = [{ + sample: { + insert: '', + deleteLeft: 1 + }, + p: 1 + } + ]; + + const basePredictions = predictFromCorrections(plainModel, correctionDistribution, context); + + // must construct the taillessTrueKeystroke appropriately. + const tokenization = new ContextTokenization([ + ContextToken.fromRawText(plainModel, 'the', false), + ContextToken.fromRawText(plainModel, ' ', false), + ContextToken.fromRawText(plainModel, '', true) + ]); + const transition = new ContextTransition(new ContextState(context, plainModel, tokenization), 0); + + const targetTokenization = new ContextTokenization([ + tokenization.tokens[0], + new ContextToken(new LegacyQuotientSpur(tokenization.tokens[1].searchModule, correctionDistribution, correctionDistribution[0])), + new ContextToken(new LegacyQuotientRoot(plainModel)) + ]); + transition.finalize(new ContextState(models.applyTransform(correctionDistribution[0].sample, context), plainModel, targetTokenization), correctionDistribution); + + const mappedPredictions = buildAndMapPredictions( + transition, + transition.base.displayTokenization, + {matchString: '', totalCost: 0}, + 1 + ); + + assert.deepEqual(mappedPredictions.map((tuple) => tuple.prediction), basePredictions.map((tuple) => tuple.prediction)); + }); + + it('properly handles contexts made empty by input backspace', () => { + const context: Context = { + left: 't', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const correctionDistribution = [{ + sample: { + insert: '', + deleteLeft: 1, + deleteRight: 0 + }, + p: 1 + } + ]; + + // must construct the taillessTrueKeystroke appropriately. + const tokenization = new ContextTokenization([ + ContextToken.fromRawText(plainModel, 't', true) + ]); + const transition = new ContextTransition(new ContextState(context, plainModel, tokenization), 0); + + const targetTokenization = new ContextTokenization([ + new ContextToken(new LegacyQuotientRoot(plainModel)) + ], { + alignment: { + merges: [], + splits: [], + unmappedEdits: [], + edgeWindow: { + ...buildEdgeWindow(tokenization.tokens, correctionDistribution[0].sample, false), + retokenization: [''], + retokenizationText: '' + }, + removedTokenCount: 0 + }, + inputs: (() => { + const val: ProbabilityMass>[] = [{ + sample: new Map(), + p: correctionDistribution[0].p + }]; + + val[0].sample.set(0, correctionDistribution[0].sample); + + return val; + })(), + inputSubsetId: generateSubsetId() + }, null); + transition.finalize(new ContextState(models.applyTransform(correctionDistribution[0].sample, context), plainModel, targetTokenization), correctionDistribution); + + const mappedPredictions = buildAndMapPredictions( + transition, + transition.final.displayTokenization, + {matchString: '', totalCost: 0}, + 1 + ); + + mappedPredictions.forEach((tuple) => assert.equal(tuple.prediction.sample.transform.deleteLeft, 1)); + }); +}); \ No newline at end of file From f864bf293a473b3eeccf0b0d23af10cecfac3375 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 27 Jul 2026 12:11:50 -0500 Subject: [PATCH 18/21] change(web): spin off method, add unit tests to verify handling of keep vs revert suggestions --- .../src/main/correction/context-transition.ts | 9 ++ .../src/main/model-compositor.ts | 20 +--- .../worker-thread/src/main/predict-helpers.ts | 33 +++++- .../worker-thread/src/main/test-index.ts | 2 +- .../prepend-reversion.tests.ts | 101 ++++++++++++++++++ 5 files changed, 148 insertions(+), 17 deletions(-) create mode 100644 web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/prepend-reversion.tests.ts diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts index c65f348996..ede7125a9c 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts @@ -19,6 +19,15 @@ import Reversion = LexicalModelTypes.Reversion; import Suggestion = LexicalModelTypes.Suggestion; import Transform = LexicalModelTypes.Transform; +export interface TransitionReversionView extends Pick { + /** + * Gets the context state resulting from the context transition event, + * including any generated suggestions and data regarding potential + * application thereof. + */ + final: Pick +} + /** * Represents the transition between two context states as triggered * by input keystrokes or applied suggestions. diff --git a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts index 8e12b96ed4..1a514cad64 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts @@ -2,7 +2,7 @@ import * as models from '@keymanapp/models-templates'; import { LexicalModelTypes } from '@keymanapp/common-types'; import { TransformUtils } from './transformUtils.js'; -import { applySuggestionCasing, correctAndEnumerate, createDefaultKeep, dedupeSuggestions, finalizeSuggestions, predictionAutoSelect, processSimilarity, toAnnotatedSuggestion, tupleDisplayOrderSort } from './predict-helpers.js'; +import { applySuggestionCasing, correctAndEnumerate, createDefaultKeep, dedupeSuggestions, finalizeSuggestions, predictionAutoSelect, prependReversion, processSimilarity, toAnnotatedSuggestion, tupleDisplayOrderSort } from './predict-helpers.js'; import { detectCurrentCasing, determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js'; import { ContextTracker } from './correction/context-tracker.js'; @@ -206,18 +206,8 @@ export class ModelCompositor { this.SUGGESTION_ID_SEED++; }); - if(revertableTransitionId) { - const reversion = this.contextTracker.peek(revertableTransitionId)?.reversion; - if(reversion) { - if(suggestions[0]?.tag == 'keep') { - const keep = suggestions.shift(); - suggestions.unshift(reversion); - suggestions.unshift(keep); - } else { - suggestions.unshift(reversion) - } - } - } + const transitionToRevert = this.contextTracker?.peek(revertableTransitionId); + prependReversion(suggestions, transitionToRevert); // Store the suggestions on the final token of the current context state (if it exists). // Or, once phrase-level suggestions are possible, on whichever token serves as each prediction's root. @@ -239,8 +229,8 @@ export class ModelCompositor { // Step 1: re-use the original input Transform as the reversion's Transform. // The Web engine will restore the original state of the context before accepting // and before reverting; all we need to do is put the original keystroke back in place. - let reversionTransform: Transform = originalInput - ? { ...originalInput } + let reversionTransform: Transform = originalInput + ? { ...originalInput } : { insert: '', deleteLeft: 0, id: suggestion.transform.id }; // Step 2: building the proper 'displayAs' string for the Reversion diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 5aa73200ee..9047e80a83 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -9,7 +9,7 @@ import { ContextTokenization } from './correction/context-tokenization.js'; import { ContextTracker } from './correction/context-tracker.js'; import { ContextToken } from './correction/context-token.js'; import { ContextState, determineContextSlideTransform } from './correction/context-state.js'; -import { ContextTransition } from './correction/context-transition.js'; +import { ContextTransition, TransitionReversionView } from './correction/context-transition.js'; import { ExecutionTimer } from './correction/execution-timer.js'; import { ModelCompositor } from './model-compositor.js'; import { getBestTokenMatches } from './correction/distance-modeler.js'; @@ -1196,4 +1196,35 @@ export function toAnnotatedSuggestion( } return result; +} + +/** + * For applicable scenarios, this mutates the passed-in suggestion array by + * prepending a predictive-text reversion that restores the context to a prior + * state. Otherwise, it leaves the suggestion array unaltered. + * @param suggestions + * @param transitionToRevert + * @returns + */ +export function prependReversion(suggestions: Suggestion[], transitionToRevert: TransitionReversionView) { + if(transitionToRevert) { + const reversion = transitionToRevert.reversion; + if(reversion) { + if(suggestions[0]?.tag == 'keep') { + const appliedId = -reversion.id; + const appliedSuggestion = transitionToRevert.final.suggestions.find((s) => s.id == appliedId); + // If the selected suggestion was itself a `keep`, we don't need a + // reversion. They'd do the same thing. + if(appliedSuggestion.tag != 'keep') { + const keep = suggestions.shift(); + suggestions.unshift(reversion); + suggestions.unshift(keep); + } + } else { + suggestions.unshift(reversion); + } + } + } + + return suggestions; } \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts b/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts index e0c5d513f9..bcaf4692bf 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts @@ -3,7 +3,7 @@ export * from './correction/context-state.js'; export * from './correction/context-token.js'; export * from './correction/context-tokenization.js'; export { ContextTracker } from './correction/context-tracker.js'; -export { ContextTransition } from './correction/context-transition.js'; +export * from './correction/context-transition.js'; export * from './correction/correction-searchable.js'; export * from './correction/correction-result-mapping.js'; export * from './correction/distance-modeler.js'; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/prepend-reversion.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/prepend-reversion.tests.ts new file mode 100644 index 0000000000..5215705ed6 --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/prepend-reversion.tests.ts @@ -0,0 +1,101 @@ + +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2026-07-27 + * + * This file contains tests designed to ensure predictive text does not + * provide matching 'keep' and 'revert' suggestions in any context. + */ + + +import { assert } from 'chai'; + +import { LexicalModelTypes } from '@keymanapp/common-types'; +import { prependReversion, type TransitionReversionView } from "@keymanapp/lm-worker/test-index"; + +import Suggestion = LexicalModelTypes.Suggestion; + +describe('prependReversion', () => { + it(`prepends reversions when reverting a non-'keep' suggestion`, () => { + // context: Original was appl+u, corrected to apply. Reached via bksp. + const suggestions: Suggestion[] = [{ + tag: 'keep', + transform: { insert: 'apply', deleteLeft: 6, id: 3 }, + displayAs: '"apply"', + id: 5, + matchesModel: false + } as Suggestion]; + + const revertable: TransitionReversionView = { + reversion: { + tag: 'revert', + transform: { insert: 'u', deleteLeft: 0, id: 3 }, + id: -3, + displayAs: '"applu"' + }, + final: { + suggestions: [{ + tag: 'keep', + transform: { insert: 'applu', deleteLeft: 4, id: 3 }, + displayAs: '"applu"', + id: 2, + matchesModel: false + } as Suggestion, { + transform: { insert: 'apply', deleteLeft: 4, id: 3 }, + displayAs: 'apply', + id: 3 + }, { + transform: { insert: 'applied', deleteLeft: 4, id: 3 }, + displayAs: 'applied', + id: 4 + }] + } + }; + + prependReversion(suggestions, revertable); + + assert.includeMembers(suggestions, [revertable.reversion]); + }); + + it(`does not prepend reversions when reverting a 'keep' suggestion`, () => { + // context: Original was appl+u, corrected to apply + const suggestions: Suggestion[] = [{ + tag: 'keep', + transform: { insert: 'applu', deleteLeft: 5, id: 3 }, + displayAs: '"applu"', + id: 5, + matchesModel: false + } as Suggestion]; + + const revertable: TransitionReversionView = { + reversion: { + tag: 'revert', + transform: { insert: 'u', deleteLeft: 0, id: 3 }, + id: -2, + displayAs: '"applu"' + }, + final: { + suggestions: [{ + tag: 'keep', + transform: { insert: 'applu', deleteLeft: 4, id: 3 }, + displayAs: '"applu"', + id: 2, + matchesModel: false + } as Suggestion, { + transform: { insert: 'apply', deleteLeft: 4, id: 3 }, + displayAs: 'apply', + id: 3 + }, { + transform: { insert: 'applied', deleteLeft: 4, id: 3 }, + displayAs: 'applied', + id: 4 + }] + } + }; + + prependReversion(suggestions, revertable); + + assert.notIncludeMembers(suggestions, [revertable.reversion]); + }); +}); \ No newline at end of file From 7db38f93ec303f0c07774964b42237090b0c9792 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 28 Jul 2026 08:40:44 -0500 Subject: [PATCH 19/21] fix(web): do not remove applied transition ID from original token when editing --- .../src/main/correction/context-tokenization.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 4ccd8cb86d..61b06092ab 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -673,9 +673,6 @@ export class ContextTokenization { tailTokenization.splice(tokenIndex, 1, affectedToken); } - affectedToken.isPartial = true; - delete affectedToken.appliedTransitionId; - // If we are completely replacing a token via delete left, erase the deleteLeft; // that part applied to a _previous_ token that no longer exists. // We start at index 0 in the insert string for the "new" token. @@ -699,6 +696,11 @@ export class ContextTokenization { affectedToken = new ContextToken(affectedToken); affectedToken.addInput(inputSource, distribution); + // Do not adjust the original token, as it may be used by other transitions. + // Only adjust the new, extended token. + affectedToken.isPartial = true; + delete affectedToken.appliedTransitionId; + const tokenize = determineModelTokenizer(lexicalModel); affectedToken.isWhitespace = tokenize({left: affectedToken.exampleInput, startOfBuffer: false, endOfBuffer: false}).left[0]?.isWhitespace ?? false; // Do not re-use the previous token; the mutation may have unexpected From f2e160c232d5ecb6b600c2e86f5961096f7d3f17 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 28 Jul 2026 12:01:03 -0500 Subject: [PATCH 20/21] feat(web): add unit tests for prior commit --- .../context/context-tokenization.tests.ts | 192 ++++++++++++++++++ 1 file changed, 192 insertions(+) diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index 8f93bd20f4..0b10f90c89 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -860,6 +860,198 @@ describe('ContextTokenization', function() { assert.equal(preTail.exampleInput, '\''); assert.equal(tail.exampleInput, '.'); }); + + describe('properly handles previously-applied transition IDs', () => { + it('does not preserve applied transition IDs on edited tokens', () => { + const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day', ' ', 'can']; + const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t))); + + const REVERTABLE_TRANSITION_ID = 31415; + baseTokenization.tail.appliedTransitionId = REVERTABLE_TRANSITION_ID; + const NEW_TRANSITION_ID = REVERTABLE_TRANSITION_ID + 1; + + const dist = [ + { + sample: { insert: 't', deleteLeft: 0, deleteRight: 0, id: NEW_TRANSITION_ID }, + p: 1 + } + ]; + + const resultTokenization = baseTokenization.evaluateTransition( + { + alignment: { + merges: [], + splits: [], + unmappedEdits: [], + edgeWindow: { + ...buildEdgeWindow(baseTokenization.tokens, dist[0].sample, false), + retokenization: baseTokenization.tokens.map((t) => t.exampleInput), + retokenizationText: baseTokenization.tokens.map((t) => t.exampleInput).reduce((accum, curr) => accum + curr, '') + }, + removedTokenCount: 0 + }, + inputs: (() => { + const map: Map = new Map(); + map.set(0, dist[0].sample); + + return [ + {sample: map, p: 1} + ]; + })(), + inputSubsetId: 0 + }, + NEW_TRANSITION_ID, + 1 + ) + + resultTokenization.tokens.forEach((t) => { + assert.isUndefined(t.appliedTransitionId); + }); + }); + + it('preserves applied transition IDs on applicable tokens', () => { + const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day', ' ', 'can']; + const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t))); + + const REVERTABLE_TRANSITION_ID = 31415; + baseTokenization.tail.appliedTransitionId = REVERTABLE_TRANSITION_ID; + const NEW_TRANSITION_ID = REVERTABLE_TRANSITION_ID + 1; + + + const dist = [ + { + sample: { insert: ' ', deleteLeft: 0, deleteRight: 0, id: NEW_TRANSITION_ID }, + p: 1 + } + ]; + + const resultTokenization = baseTokenization.evaluateTransition( + { + alignment: { + merges: [], + splits: [], + unmappedEdits: [], + edgeWindow: { + ...buildEdgeWindow(baseTokenization.tokens, dist[0].sample, false), + retokenization: baseTokenization.tokens.map((t) => t.exampleInput), + retokenizationText: baseTokenization.tokens.map((t) => t.exampleInput).reduce((accum, curr) => accum + curr, '') + }, + removedTokenCount: 0 + }, + inputs: (() => { + const map: Map = new Map(); + map.set(1, dist[0].sample); + + return [ + {sample: map, p: 1} + ]; + })(), + inputSubsetId: 0 + }, + NEW_TRANSITION_ID, + 1 + ) + + const resultTokenLength = resultTokenization.tokens.length; + + resultTokenization.tokens.forEach((t, i) => { + // The space will add TWO tokens. + if(i == resultTokenLength - 3) { + assert.equal(t.appliedTransitionId, REVERTABLE_TRANSITION_ID); + } else { + assert.isUndefined(t.appliedTransitionId); + } + }); + }); + + // Performs the two above in sequence in a manner that could cause cross-effects + // if implemented incorrectly. + it('does not conflate effects between different tokenization transitions', () => { + const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day', ' ', 'can']; + const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t))); + + const REVERTABLE_TRANSITION_ID = 31415; + baseTokenization.tail.appliedTransitionId = REVERTABLE_TRANSITION_ID; + const NEW_TRANSITION_ID = REVERTABLE_TRANSITION_ID + 1; + + const dist = [ + { + sample: { insert: ' ', deleteLeft: 0, deleteRight: 0, id: NEW_TRANSITION_ID }, + p: .8 + }, { + sample: { insert: ' ', deleteLeft: 0, deleteRight: 0, id: NEW_TRANSITION_ID }, + p: .2 + } + ]; + + const baseTransitionEdge: TransitionEdge = { + alignment: { + merges: [], + splits: [], + unmappedEdits: [], + edgeWindow: { + ...buildEdgeWindow(baseTokenization.tokens, dist[0].sample, false), + retokenization: baseTokenization.tokens.map((t) => t.exampleInput), + retokenizationText: baseTokenization.tokens.map((t) => t.exampleInput).reduce((accum, curr) => accum + curr, '') + }, + removedTokenCount: 0 + }, + inputs: (() => { + const map: Map = new Map(); + map.set(1, dist[0].sample); + + return [ + {sample: map, p: dist[0].p} + ]; + })(), + inputSubsetId: 0 + }; + + // We don't care about the results here. What we care about is that + // this call doesn't remove the appliedTransitionId from the source + // token, preventing it from being marked on later tokenization + // transitions. + baseTokenization.evaluateTransition( + { + alignment: { + ...baseTransitionEdge.alignment, + edgeWindow: { + ...baseTransitionEdge.alignment.edgeWindow, + ...buildEdgeWindow(baseTokenization.tokens, dist[1].sample, false) + } + }, + inputs: (() => { + const map: Map = new Map(); + map.set(0, dist[1].sample); + + return [ + {sample: map, p: dist[1].p} + ]; + })(), + inputSubsetId: 0 + }, + NEW_TRANSITION_ID, + dist[1].p + ) + + const resultTokenization = baseTokenization.evaluateTransition( + baseTransitionEdge, + NEW_TRANSITION_ID, + dist[0].p + ); + + const resultTokenLength = resultTokenization.tokens.length; + + resultTokenization.tokens.forEach((t, i) => { + // The space will add TWO tokens. + if(i == resultTokenLength - 3) { + assert.equal(t.appliedTransitionId, REVERTABLE_TRANSITION_ID); + } else { + assert.isUndefined(t.appliedTransitionId); + } + }); + }); + }); }); describe('buildEdgeWindow', () => { From d2b5ac583264de321364010ec2eaacaf778b1479 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Fri, 31 Jul 2026 16:29:58 -0500 Subject: [PATCH 21/21] fix(web): fix broken unit test --- web/src/engine/predictive-text/templates/src/common.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/engine/predictive-text/templates/src/common.ts b/web/src/engine/predictive-text/templates/src/common.ts index d141f1f9ae..77d4826174 100644 --- a/web/src/engine/predictive-text/templates/src/common.ts +++ b/web/src/engine/predictive-text/templates/src/common.ts @@ -60,7 +60,7 @@ export function buildMergedTransform(first: Transform, second: Transform): Trans deleteLeft: first.deleteLeft + mergedSecondDelete } - if(first.id && first.id == second.id) { + if(first.id !== undefined && first.id == second.id) { returnedObj.id = first.id; }