diff --git a/common/web/types/src/lexical-model-types.ts b/common/web/types/src/lexical-model-types.ts index 37a761608d..a3c8df1c98 100644 --- a/common/web/types/src/lexical-model-types.ts +++ b/common/web/types/src/lexical-model-types.ts @@ -253,9 +253,14 @@ export interface LexicalModel { */ export interface Transform { /** - * Facilitates use of unique identifiers for tracking the Transform and - * any related data from its original source, as the reference cannot be - * preserved across WebWorker boundaries. + * Facilitates use of unique identifiers for tracking data about the context + * transition to which the Transform belongs. More than one Transform may + * hold the same `id` if they are alternate interpretations of the same + * transition event - say, the resulting effects of neighbor keys that may + * have been missed due to "fat fingering". + * + * Also note that the Transform reference cannot be preserved across WebWorker + * boundaries, but this ID may. * * This is *separate* from any LMLayer-internal identification values. */ @@ -287,13 +292,6 @@ export interface Transform { * A concrete suggestion */ export interface Suggestion { - /** - * Indicates the externally-supplied id of the Transform that prompted - * the Suggestion. Automatically handled by the LMLayer; models should - * not handle this field. - */ - transformId?: number; - /** * A unique identifier for the Suggestion itself, not shared with any others - * even for Suggestions sourced from the same Transform. diff --git a/ios/engine/KMEI/KeymanEngine/Classes/Constants.swift b/ios/engine/KMEI/KeymanEngine/Classes/Constants.swift index 01d2862838..2d820d710e 100644 --- a/ios/engine/KMEI/KeymanEngine/Classes/Constants.swift +++ b/ios/engine/KMEI/KeymanEngine/Classes/Constants.swift @@ -33,6 +33,7 @@ public enum Key { /// Dictionary of prediction/correction toggle settings keyed by language id in UserDefaults static let userPredictSettings = "UserPredictionEnablementSettings" static let userCorrectSettings = "UserCorrectionEnablementSettings" + static let userAutocorrectSettings = "UserAutocorrectionEnablementSettings" // Internal user defaults keys static let engineVersion = "KeymanEngineVersion" diff --git a/ios/engine/KMEI/KeymanEngine/Classes/Extension/UserDefaults+Types.swift b/ios/engine/KMEI/KeymanEngine/Classes/Extension/UserDefaults+Types.swift index 8991660dae..4158dc51f2 100644 --- a/ios/engine/KMEI/KeymanEngine/Classes/Extension/UserDefaults+Types.swift +++ b/ios/engine/KMEI/KeymanEngine/Classes/Extension/UserDefaults+Types.swift @@ -331,6 +331,17 @@ public extension UserDefaults { set(prefs, forKey: Key.userCorrectSettings) } } + + // stores a dictionary of autocorrection-enablement settings keyed to language ids, i.e., [langID: Bool] + var autocorrectionEnablements: [String: Bool]? { + get { + return dictionary(forKey: Key.userAutocorrectSettings) as? [String : Bool] + } + + set(prefs) { + set(prefs, forKey: Key.userAutocorrectSettings) + } + } var portraitKeyboardHeight: Double { get { @@ -387,4 +398,22 @@ public extension UserDefaults { prefs?[forLanguageID] = correctSetting correctionEnablements = prefs } + + func autocorrectSettingForLanguage(languageID: String) -> Bool { + if let dict = autocorrectionEnablements { + return dict[languageID] ?? true + } else { + return true + } + } + + func set(autocorrectSetting: Bool, forLanguageID: String) { + var prefs: [String: Bool]? + prefs = autocorrectionEnablements + if prefs == nil { + prefs = [String: Bool]() + } + prefs?[forLanguageID] = autocorrectSetting + autocorrectionEnablements = prefs + } } diff --git a/ios/engine/KMEI/KeymanEngine/Classes/Keyboard/KeymanWebViewController.swift b/ios/engine/KMEI/KeymanEngine/Classes/Keyboard/KeymanWebViewController.swift index 02560e6929..110566c326 100644 --- a/ios/engine/KMEI/KeymanEngine/Classes/Keyboard/KeymanWebViewController.swift +++ b/ios/engine/KMEI/KeymanEngine/Classes/Keyboard/KeymanWebViewController.swift @@ -433,10 +433,11 @@ extension KeymanWebViewController { let predict = userDefaults.predictSettingForLanguage(languageID: lexicalModel.languageID) let correct = userDefaults.correctSettingForLanguage(languageID: lexicalModel.languageID) + let autocorrect = userDefaults.autocorrectSettingForLanguage(languageID: lexicalModel.languageID) // Pass these off to KMW! // We do these first so that they're automatically set for the to-be-registered model in advance. - webView!.evaluateJavaScript("enableSuggestions(\(stubString), \(predict), \(correct))") + webView!.evaluateJavaScript("enableSuggestions(\(stubString), \(predict), \(correct), \(autocorrect))") self.activeModel = predict } else { // We're registering a model in the background - don't change settings. webView!.evaluateJavaScript("keyman.addModel(\(stubString));", completionHandler: nil) diff --git a/ios/engine/KMEI/KeymanEngine/Classes/Settings/LanguageSettingsViewController.swift b/ios/engine/KMEI/KeymanEngine/Classes/Settings/LanguageSettingsViewController.swift index 0c98a1cacc..8ee69ed231 100644 --- a/ios/engine/KMEI/KeymanEngine/Classes/Settings/LanguageSettingsViewController.swift +++ b/ios/engine/KMEI/KeymanEngine/Classes/Settings/LanguageSettingsViewController.swift @@ -18,8 +18,11 @@ class LanguageSettingsViewController: UITableViewController { private var doPredictionsSwitch: UISwitch? private var doCorrectionsSwitch: UISwitch? + private var doAutocorrectionsSwitch: UISwitch? private var doCorrectionsLabel: UILabel? + private var doAutocorrectionsLabel: UILabel? private var correctionsCell: UITableViewCell? + private var autocorrectionsCell: UITableViewCell? public init(_ inLanguage: Language) { language = inLanguage @@ -87,17 +90,8 @@ class LanguageSettingsViewController: UITableViewController { return switchFrame } - @objc - func predictionSwitchValueChanged(source: UISwitch) { - let value = source.isOn; + func refreshModelIfNeeded() { let userDefaults = Storage.active.userDefaults - userDefaults.set(predictSetting: value, forLanguageID: self.language.id) - - // Reactively set the corrections switch interactivity state. - self.doCorrectionsSwitch?.isHidden = !value - self.doCorrectionsLabel?.isEnabled = value - self.correctionsCell?.isUserInteractionEnabled = value - if let lm = Manager.shared.preferredLexicalModel(userDefaults, forLanguage: self.language.id) { if Manager.shared.currentKeyboardID?.languageID == self.language.id { // re-register the model - that'll enact the settings. @@ -111,23 +105,63 @@ class LanguageSettingsViewController: UITableViewController { } } + func refreshSwitchVisibility() { + let userDefaults = Storage.active.userDefaults + + let mayPredict = userDefaults.predictSettingForLanguage(languageID: self.language.id) + self.doCorrectionsSwitch?.isHidden = !mayPredict + self.doCorrectionsLabel?.isEnabled = mayPredict + self.correctionsCell?.isUserInteractionEnabled = mayPredict + + let mayCorrect = userDefaults.correctSettingForLanguage(languageID: self.language.id) + self.doAutocorrectionsSwitch?.isHidden = !(mayPredict && mayCorrect) + self.doAutocorrectionsLabel?.isEnabled = mayPredict && mayCorrect + self.autocorrectionsCell?.isUserInteractionEnabled = mayPredict && mayCorrect + } + + @objc + func predictionSwitchValueChanged(source: UISwitch) { + let value = source.isOn; + let userDefaults = Storage.active.userDefaults + userDefaults.set(predictSetting: value, forLanguageID: self.language.id) + + refreshSwitchVisibility() + refreshModelIfNeeded() + } + @objc func correctionSwitchValueChanged(source: UISwitch) { let value = source.isOn; let userDefaults = Storage.active.userDefaults userDefaults.set(correctSetting: value, forLanguageID: self.language.id) - if let lm = Manager.shared.preferredLexicalModel(userDefaults, forLanguage: self.language.id) { - if Manager.shared.currentKeyboardID?.languageID == self.language.id { - // re-register the model - that'll enact the settings. - _ = Manager.shared.registerLexicalModel(lm) - } - // Based on how Manager chooses the model in setKeyboard. - } else if let lm = userDefaults.userLexicalModels?.first(where: { $0.languageID == self.language.id }) { - if Manager.shared.currentKeyboardID?.languageID == self.language.id { - _ = Manager.shared.registerLexicalModel(lm) - } - } + refreshSwitchVisibility() + refreshModelIfNeeded() + } + + @objc + func autocorrectionSwitchValueChanged(source: UISwitch) { + let value = source.isOn; + let userDefaults = Storage.active.userDefaults + userDefaults.set(autocorrectSetting: value, forLanguageID: self.language.id) + + refreshModelIfNeeded() + } + + func addSwitchToTableCell(_ toggle: UISwitch, cell: UITableViewCell, isOn: Bool, selector: Selector) { + cell.accessoryType = .none + toggle.translatesAutoresizingMaskIntoConstraints = false + + let switchFrame = frameAtRightOfCell(cell: cell.frame, controlSize: toggle.frame.size) + toggle.frame = switchFrame + + toggle.isOn = isOn + toggle.addTarget(self, action: selector, for: .valueChanged) + cell.addSubview(toggle) + cell.contentView.isUserInteractionEnabled = false + + toggle.rightAnchor.constraint(equalTo: cell.layoutMarginsGuide.rightAnchor).isActive = true + toggle.centerYAnchor.constraint(equalTo: cell.layoutMarginsGuide.centerYAnchor).isActive = true } override func tableView(_ tableView: UITableView, cellForRowAt indexPath: IndexPath) -> UITableViewCell { @@ -143,40 +177,38 @@ class LanguageSettingsViewController: UITableViewController { let cell = UITableViewCell(style: .subtitle, reuseIdentifier: cellIdentifier) if 1 == indexPath.section { if 0 == indexPath.row { - cell.accessoryType = .none doPredictionsSwitch = UISwitch() - doPredictionsSwitch!.translatesAutoresizingMaskIntoConstraints = false - let switchFrame = frameAtRightOfCell(cell: cell.frame, controlSize: doPredictionsSwitch!.frame.size) - doPredictionsSwitch!.frame = switchFrame - - doPredictionsSwitch!.isOn = userDefaults.predictSettingForLanguage(languageID: self.language.id) - doPredictionsSwitch!.addTarget(self, action: #selector(self.predictionSwitchValueChanged), for: .valueChanged) - cell.addSubview(doPredictionsSwitch!) - cell.contentView.isUserInteractionEnabled = false - - doPredictionsSwitch!.rightAnchor.constraint(equalTo: cell.layoutMarginsGuide.rightAnchor).isActive = true - doPredictionsSwitch!.centerYAnchor.constraint(equalTo: cell.layoutMarginsGuide.centerYAnchor).isActive = true + self.addSwitchToTableCell( + doPredictionsSwitch!, + cell: cell, + isOn: userDefaults.predictSettingForLanguage(languageID: self.language.id), + selector: #selector(self.predictionSwitchValueChanged) + ) } else if 1 == indexPath.row { correctionsCell = cell - cell.accessoryType = .none doCorrectionsSwitch = UISwitch() - doCorrectionsSwitch!.translatesAutoresizingMaskIntoConstraints = false - let switchFrame = frameAtRightOfCell(cell: cell.frame, controlSize: doCorrectionsSwitch!.frame.size) - doCorrectionsSwitch!.frame = switchFrame + self.addSwitchToTableCell( + doCorrectionsSwitch!, + cell: cell, + isOn: userDefaults.correctSettingForLanguage(languageID: self.language.id), + selector: #selector(self.correctionSwitchValueChanged) + ) + + refreshSwitchVisibility() + } else if 2 == indexPath.row { + autocorrectionsCell = cell + doAutocorrectionsSwitch = UISwitch() - doCorrectionsSwitch!.isOn = userDefaults.correctSettingForLanguage(languageID: self.language.id) - doCorrectionsSwitch!.addTarget(self, action: #selector(self.correctionSwitchValueChanged), for: .valueChanged) - cell.addSubview(doCorrectionsSwitch!) - cell.contentView.isUserInteractionEnabled = false + self.addSwitchToTableCell( + doAutocorrectionsSwitch!, + cell: cell, + isOn: userDefaults.autocorrectSettingForLanguage(languageID: self.language.id), + selector: #selector(self.autocorrectionSwitchValueChanged) + ) - doCorrectionsSwitch!.rightAnchor.constraint(equalTo: cell.layoutMarginsGuide.rightAnchor).isActive = true - doCorrectionsSwitch!.centerYAnchor.constraint(equalTo: cell.layoutMarginsGuide.centerYAnchor).isActive = true - - // Disable interactivity if the prediction toggle is set to 'off'. - doCorrectionsSwitch!.isHidden = !userDefaults.predictSettingForLanguage(languageID: self.language.id) - cell.isUserInteractionEnabled = userDefaults.predictSettingForLanguage(languageID: self.language.id) + refreshSwitchVisibility() } else { // rows 3 and 4 cell.accessoryType = .disclosureIndicator } @@ -245,6 +277,10 @@ class LanguageSettingsViewController: UITableViewController { cell.textLabel?.text = NSLocalizedString("menu-langsettings-toggle-correct", bundle: engineBundle, comment: "") cell.textLabel?.isEnabled = !(doCorrectionsSwitch?.isHidden ?? false) case 2: + doAutocorrectionsLabel = cell.textLabel + cell.textLabel?.text = NSLocalizedString("menu-langsettings-toggle-autocorrect", bundle: engineBundle, comment: "") + cell.textLabel?.isEnabled = !(doAutocorrectionsSwitch?.isHidden ?? false) + case 3: cell.textLabel?.text = NSLocalizedString("menu-langsettings-label-lexical-models", bundle: engineBundle, comment: "") cell.accessoryType = .disclosureIndicator let modelCt = language.lexicalModels?.count ?? 0 @@ -332,8 +368,8 @@ class LanguageSettingsViewController: UITableViewController { showKeyboardInfoView(kb: (language.keyboards?[safe: indexPath.row])!) case 1: switch indexPath.row { - // case 0, 1: the toggles - but a general 'click' not on the toggle itself. - case 2: + // case 0, 1, 2: the toggles - but a general 'click' not on the toggle itself. + case 3: showLexicalModelsView() default: break diff --git a/ios/engine/KMEI/KeymanEngine/en.lproj/Localizable.strings b/ios/engine/KMEI/KeymanEngine/en.lproj/Localizable.strings index 05a1723ebb..e45e70f853 100644 --- a/ios/engine/KMEI/KeymanEngine/en.lproj/Localizable.strings +++ b/ios/engine/KMEI/KeymanEngine/en.lproj/Localizable.strings @@ -152,10 +152,13 @@ "menu-langsettings-title" = "%@ Settings"; /* Label for the toggle that enables corrections that is displayed within a language-specific settings menu */ -"menu-langsettings-toggle-correct" = "Enable corrections"; +"menu-langsettings-toggle-correct" = "Offer corrections"; + +/* Label for the toggle that enables auto-corrections that is displayed within a language-specific settings menu */ +"menu-langsettings-toggle-autocorrect" = "Apply corrections automatically"; /* Label for the toggle that enables predictions that is displayed within a language-specific settings menu */ -"menu-langsettings-toggle-predict" = "Enable predictions"; +"menu-langsettings-toggle-predict" = "Offer word completions"; /* Help message for a prompt that appears for confirming a lexical model download: language (1): lexical model (dictionary) name (2) */ "menu-lexical-model-install-message" = "Would you like to install this dictionary?"; diff --git a/ios/engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/ios-host.js b/ios/engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/ios-host.js index 25af94b5bd..fb5c4c67cb 100644 --- a/ios/engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/ios-host.js +++ b/ios/engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/ios-host.js @@ -326,11 +326,12 @@ function toHex(theString) { return hexString.substr(0, hexString.length-1); } -function enableSuggestions(model, mayPredict, mayCorrect) { +function enableSuggestions(model, mayPredict, mayCorrect, mayAutocorrect) { // Set the options first so that KMW's ModelCache can properly handle model enablement states // the moment we actually register the new model. keyman.core.languageProcessor.mayPredict = mayPredict; keyman.core.languageProcessor.mayCorrect = mayCorrect; + keyman.core.languageProcessor.mayAutoCorrect = mayAutocorrect; keyman.addModel(model); } diff --git a/web/src/engine/predictive-text/templates/src/common.ts b/web/src/engine/predictive-text/templates/src/common.ts index ec6e8a09e5..77d4826174 100644 --- a/web/src/engine/predictive-text/templates/src/common.ts +++ b/web/src/engine/predictive-text/templates/src/common.ts @@ -60,7 +60,7 @@ export function buildMergedTransform(first: Transform, second: Transform): Trans deleteLeft: first.deleteLeft + mergedSecondDelete } - if(first.id && first.id == second.id) { + if(first.id !== undefined && first.id == second.id) { returnedObj.id = first.id; } @@ -132,10 +132,6 @@ export function transformToSuggestion(transform: Transform, p?: number): Outcome displayAs: transform.insert }; - if(transform.id !== undefined) { - suggestion.transformId = transform.id; - } - if(p === 0 || p) { suggestion.p = p; } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts index 895970f285..19ae354477 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts @@ -95,7 +95,7 @@ export class ContextState { return undefined; } - return this.suggestions.find(s => s.id == this.appliedSuggestionId)?.transformId; + return this.suggestions.find(s => s.id == this.appliedSuggestionId)?.transform.id; } /** diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts index e21b9709b0..f3128c6120 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts @@ -10,7 +10,7 @@ import { LexicalModelTypes } from '@keymanapp/common-types'; import { SearchQuotientNode, PathInputProperties } from "./search-quotient-node.js"; -import { TokenSplitMap } from "./context-tokenization.js"; +import { TokenSplitMapping } from "./context-tokenization.js"; import { LegacyQuotientSpur } from "./legacy-quotient-spur.js"; import { LegacyQuotientRoot } from "./legacy-quotient-root.js"; import { generateSubsetId } from './tokenization-subsets.js'; @@ -26,12 +26,12 @@ import Transform = LexicalModelTypes.Transform; * any prior cached data or for rewriting its probabilities after * receiving backspace input. * @param text - * @param transformId + * @param transitionId * @returns */ -function textToCharTransforms(text: string, transformId?: number): Transform[] { - return transformId ? - [...text].map(insert => ({insert, deleteLeft: 0, id: transformId})) : +function textToCharTransforms(text: string, transitionId?: number): Transform[] { + return transitionId ? + [...text].map(insert => ({insert, deleteLeft: 0, id: transitionId})) : [...text].map(insert => ({insert, deleteLeft: 0})); } @@ -134,7 +134,7 @@ export class ContextToken implements ContextTokenLike { static fromRawText(model: LexicalModel, rawText: string, isPartial?: boolean) { rawText ||= ''; - // Supports the old pathway for: updateWithBackspace(tokenText: string, transformId: number) + // Supports the old pathway for: updateWithBackspace(tokenText: string, transitionId: number) // Build a token that represents the current text with no ambiguity - probability at max (1.0) let searchModule: SearchQuotientNode = new LegacyQuotientRoot(model); const BASE_PROBABILITY = 1; @@ -233,7 +233,7 @@ export class ContextToken implements ContextTokenLike { * @param lexicalModel * @returns */ - split(split: TokenSplitMap): ContextToken[] { + split(split: TokenSplitMapping): ContextToken[] { // Split from tail to head - leave as much 'head' intact as possible at each // step, rather than needing to reconstruct the tail multiple times. const splitSpecs = split.matches.slice(); diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 5dbb1af053..1a262c9ef3 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -32,7 +32,7 @@ const MIN_CHARS_TO_RECONSIDER_FOR_TOKENIZATION = 8; * This type is used to indicate properties of tokens affected by merges and * splits during a context transition. */ -interface EditTokenMap { +interface EditTokenMappingHalf { /** * The index of the affected token. */ @@ -43,10 +43,22 @@ interface EditTokenMap { text: string }; +/** + * This type is used to indicate properties of tokens affected by splits during + * a context transition. + */ +interface SplitTokenMappingHalf extends EditTokenMappingHalf { + /** + * The codepoint index within the original token at which the split-off token + * begins. + */ + textOffset: number +}; + /** * This type represents mappings for tokens affected by merge edit operations. */ -interface TokenMergeMap { +interface TokenMergeMapping { /** * Entries here represent source-context tokens that are combined as part of * the merge edit operation. @@ -54,7 +66,7 @@ interface TokenMergeMap { * Entries will appear in the same ordering as the codepoints they represent * appear in the underlying context. */ - inputs: EditTokenMap[], + inputs: EditTokenMappingHalf[], /** * This entry represents the post-transition context token produced from the @@ -63,14 +75,14 @@ interface TokenMergeMap { * Note that it is possible for extra codepoints to exist here that were not * represented in the original source-context tokens. */ - match: EditTokenMap + match: EditTokenMappingHalf }; /** * This type represents mappings for tokens affected by split edit operations. */ -export interface TokenSplitMap { +export interface TokenSplitMapping { /** * This entry represents the source-context token that was split as part of * the split edit operation. @@ -78,7 +90,7 @@ export interface TokenSplitMap { * Note that it is possible for codepoints to exist here but not be * represented in the post-transition context tokens resulting from the split. */ - input: EditTokenMap, + input: EditTokenMappingHalf, /** * Entries here represent post-transition tokens that represent pieces of the @@ -87,7 +99,7 @@ export interface TokenSplitMap { * Entries will appear in the same ordering as the codepoints they represent * appear in the underlying context. */ - matches: (EditTokenMap & { textOffset: number })[] + matches: SplitTokenMappingHalf[] }; /** @@ -100,11 +112,11 @@ export interface TransitionEdgeAlignment { /** * Denotes any token merge edits needed after applying the Transform. */ - merges: TokenMergeMap[]; + merges: TokenMergeMapping[]; /** * Denotes any token split edits needed after applying the Transform. */ - splits: TokenSplitMap[]; + splits: TokenSplitMapping[]; /** * Denotes any further token edits needed that cannot be attributed to * 'merge's, 'split's, or edits from the input `Transform`. @@ -431,8 +443,8 @@ export class ContextTokenization { * @param transitionEdge Batched results from one or more * `precomputeTokenizationAfterInput` calls on this instance, all with the * same alignment values. - * @param transitionId The id of the Transform associated with the keystroke - * triggering the transition. + * @param transitionId The id of the context transition associated with the + * Transition * @param bestProbFromSet The probability of the single most likely input * transform in the overall transformDistribution associated with the * keystroke triggering the transition. It need not be represented by the @@ -479,9 +491,6 @@ export class ContextTokenization { tailTokenization.splice(tokenIndex, 1, affectedToken); } - affectedToken.isPartial = true; - delete affectedToken.appliedTransitionId; - // If we are completely replacing a token via delete left, erase the deleteLeft; // that part applied to a _previous_ token that no longer exists. // We start at index 0 in the insert string for the "new" token. @@ -507,6 +516,11 @@ export class ContextTokenization { affectedToken.isPartial ); + // Do not adjust the original token, as it may be used by other transitions. + // Only adjust the new, extended token. + affectedToken.isPartial = true; + delete affectedToken.appliedTransitionId; + const tokenize = determineModelTokenizer(lexicalModel); affectedToken.isWhitespace = tokenize({left: affectedToken.exampleInput, startOfBuffer: false, endOfBuffer: false}).left[0]?.isWhitespace ?? false; // Do not re-use the previous token; the mutation may have unexpected @@ -516,7 +530,21 @@ export class ContextTokenization { affectedToken = null; } - return new ContextTokenization(this.tokens.slice(0, sliceIndex).concat(tailTokenization)); + // Backspace handling - emptying context via backspace or erasing _part_ of + // a whitespace token can erase the tokenization-final empty token usually + // used for word-initial suggestions. + // + // We re-add it here so that suggestions can be presented to the user as + // normal. + const tokenSequence = this.tokens.slice(0, sliceIndex).concat(tailTokenization); + if(tokenSequence.length == 0 || tokenSequence[tokenSequence.length - 1]?.isWhitespace) { + tokenSequence.push(new ContextToken(new LegacyQuotientRoot(lexicalModel))); + } + + return new ContextTokenization( + tokenSequence, + null + ); } } @@ -1030,13 +1058,13 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo * generality, if two separate groups of tokens are merged, two groups will be * defined - one for each token resulting from a merge. */ - merges: TokenMergeMap[], + merges: TokenMergeMapping[], /** * Indicates groupings of directly related splits. Without loss of * generality, if two separate tokens are split, two groups will be defined - * one for each source token split. */ - splits: TokenSplitMap[] + splits: TokenSplitMapping[] } { // We've found the root token to which changes may apply. // We've found the last post-application token to which transform changes contributed. @@ -1074,8 +1102,8 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo const mappedPath: EditTuple[] = []; let mergeOffset = 0; let splitOffset = 0; - const merges: TokenMergeMap[] = []; - const splits: TokenSplitMap[] = []; + const merges: TokenMergeMapping[] = []; + const splits: TokenSplitMapping[] = []; while(queueIndex < editPath.length) { const edit = editPath[queueIndex]; const { input, match } = edit; @@ -1085,7 +1113,7 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo let matchOffset: number = 0; if(op == 'merge') { const mergeTarget = resultTokenization[match]; - const merge: TokenMergeMap = { + const merge: TokenMergeMapping = { match: { index: match, text: mergeTarget @@ -1116,7 +1144,7 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo merges.push(merge); } else if(op == 'split') { const splitTarget = preTokenization[input]; - const split: TokenSplitMap = { + const split: TokenSplitMapping = { input: { index: input, text: splitTarget diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts index 1ffb277fc3..15a16bd96a 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts @@ -19,6 +19,15 @@ import Reversion = LexicalModelTypes.Reversion; import Suggestion = LexicalModelTypes.Suggestion; import Transform = LexicalModelTypes.Transform; +export interface TransitionReversionView extends Pick { + /** + * Gets the context state resulting from the context transition event, + * including any generated suggestions and data regarding potential + * application thereof. + */ + final: Pick +} + /** * Represents the transition between two context states as triggered * by input keystrokes or applied suggestions. @@ -36,7 +45,7 @@ export class ContextTransition { */ inputDistribution?: Distribution; - // The transform ID in play. + // The transition ID in play. private _transitionId?: number; /** @@ -177,7 +186,7 @@ export class ContextTransition { // We won't try to partially revert a multi-word suggestion; reversions // are only supported at the end of the last word of the main suggestion // body and after any appended whitespace. - resultingTokenization.tail.appliedTransitionId = suggestion.transformId; + resultingTokenization.tail.appliedTransitionId = suggestion.transform.id; const resultingState = new ContextState( applyTransform(transformToApply, baseState.context), @@ -189,12 +198,12 @@ export class ContextTransition { resultingState.appliedSuggestionId = suggestion.id; resultingState.suggestions = this.final.suggestions; - // Use the transform's ID for the transition. Note that when applying the + // Use the transition ID tracked on the Transform. Note that when applying the // `appendedTransform` component of a suggestion, this will differ from - // suggestion.transformId. + // suggestion.transitionId. const resultingTransition = new ContextTransition(baseState, transformToApply.id); resultingTransition.finalize(resultingState, inputDistribution); - resultingTransition.revertableTransitionId = suggestion.transformId; + resultingTransition.revertableTransitionId = suggestion.transform.id; // .finalize unsets _.transitionId; re-assign it. resultingTransition._transitionId = transformToApply.id; @@ -236,7 +245,7 @@ export class ContextTransition { const baseTokenizationLength = results.transition.final.displayTokenization.tokens.length; const appliedTokenization = appendingTransition.final.displayTokenization; for(let i = baseTokenizationLength; i < appliedTokenization.tokens.length; i++) { - appliedTokenization.tokens[i].appliedTransitionId = suggestion.transformId; + appliedTokenization.tokens[i].appliedTransitionId = suggestion.transform.id; } return { diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-subsets.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-subsets.ts index bfbadef182..a8617a7c0f 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-subsets.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-subsets.ts @@ -135,7 +135,8 @@ export function legacySubsetKeyer(tokenizationEdits: TokenizationTransitionEdits // Now, based on the transform tokenization. We want to force uniqueness for // all variations of result length on each tokenized transform resulting from // the precomputation's represented keystroke. - for(const {0: relativeIndex} of tokenizedTransform.entries()) { + for(const {0: relativeIndex, 1: transform} of tokenizedTransform.entries()) { + const insertLen = KMWString.length(transform.insert); if(relativeIndex > 0) { // The true boundary lie before the insert if the value is non-zero; // don't differentiate here! @@ -148,10 +149,10 @@ export function legacySubsetKeyer(tokenizationEdits: TokenizationTransitionEdits // // IMPORTANT: update unit tests manually if the BI marker here changes // or the use of SENTINEL_CODE_UNIT as a key component separator changes. - components.push(`BI@${relativeIndex}`); + components.push(`BI@${relativeIndex}-${boundaryTextLen + insertLen}`); boundaryTextLen = 0; } else { - components.push(`I@${relativeIndex}`); + components.push(`I@${relativeIndex}-${boundaryTextLen + insertLen}`); } } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts index 25863a5937..fb546e3196 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts @@ -1,7 +1,19 @@ import * as models from '@keymanapp/models-templates'; import { LexicalModelTypes } from '@keymanapp/common-types'; -import { applySuggestionCasing, composeIntermediatePredictions, correctAndEnumerate, createDefaultKeep, dedupeSuggestions, finalizeSuggestions, predictionAutoSelect, processSimilarity, toAnnotatedSuggestion, tupleDisplayOrderSort } from './predict-helpers.js'; +import { + applySuggestionCasing, + composeIntermediatePredictions, + correctAndEnumerate, + createDefaultKeep, + dedupeSuggestions, + finalizeSuggestions, + predictionAutoSelect, + prependReversion, + processSimilarity, + toAnnotatedSuggestion, + tupleDisplayOrderSort +} from './predict-helpers.js'; import { determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js'; import { ContextTracker } from './correction/context-tracker.js'; @@ -79,12 +91,12 @@ export class ModelCompositor { this.configuration = config; } - initContextTracker(context: Context, transformId: number) { + initContextTracker(context: Context, transitionId: number) { if(this.contextTracker || !this.lexicalModel.traverseFromRoot) { return; } - this._contextTracker = new ContextTracker(this.lexicalModel, context, transformId, this.configuration); + this._contextTracker = new ContextTracker(this.lexicalModel, context, transitionId, this.configuration); } async predict(transformDistribution: Transform | Distribution, context: Context): Promise[]> { @@ -120,8 +132,8 @@ export class ModelCompositor { // Only allow new-word suggestions if space was the most likely keypress. // const allowSpace = TransformUtils.isWhitespace(inputTransform); const inputTransform = transformDistribution[0].sample; - const transformId = inputTransform.id; - this.initContextTracker(context, transformId); + const transitionId = inputTransform.id; + this.initContextTracker(context, transitionId); // Section 1: determine 'prediction roots' - enumerate corrections from most to least likely, // searching for results that yield viable predictions from the model. @@ -190,18 +202,8 @@ export class ModelCompositor { this.SUGGESTION_ID_SEED++; }); - if(revertableTransitionId) { - const reversion = this.contextTracker.peek(revertableTransitionId)?.reversion; - if(reversion) { - if(suggestions[0]?.tag == 'keep') { - const keep = suggestions.shift(); - suggestions.unshift(reversion); - suggestions.unshift(keep); - } else { - suggestions.unshift(reversion) - } - } - } + const transitionToRevert = this.contextTracker?.peek(revertableTransitionId); + prependReversion(suggestions, transitionToRevert); if(suggestions.filter((s) => s.tag == 'keep').length > 1) { throw new Error(`Unexpected state: multiple keep suggestions exist: ${JSON.stringify(suggestions.filter((s) => s.tag == 'keep'))}`); @@ -229,7 +231,9 @@ export class ModelCompositor { // Step 1: re-use the original input Transform as the reversion's Transform. // The Web engine will restore the original state of the context before accepting // and before reverting; all we need to do is put the original keystroke back in place. - let reversionTransform: Transform = originalInput ?? { insert: '', deleteLeft: 0 }; + let reversionTransform: Transform = originalInput + ? { ...originalInput } + : { insert: '', deleteLeft: 0, id: suggestion.transform.id }; // Step 2: building the proper 'displayAs' string for the Reversion const postContext = originalInput ? models.applyTransform(originalInput, context) : context; @@ -253,9 +257,6 @@ export class ModelCompositor { // Since we're outside of the standard `predict` control path, we'll need to // set the Reversion's ID directly. let reversion = toAnnotatedSuggestion(this.lexicalModel, firstConversion, 'revert'); - if(suggestion.transformId != null) { - reversion.transformId = -suggestion.transformId; - } if(suggestion.id != null) { // Since a reversion inverts its source suggestion, we set its ID to be the // additive inverse of the source suggestion's ID. Makes easy mapping / @@ -277,11 +278,11 @@ export class ModelCompositor { } } else { let originalTransition = this.contextTracker.latest; - if(originalTransition.transitionId != suggestion.transformId) { - originalTransition = this.contextTracker.findAndRevert(suggestion.transformId); + if(originalTransition.transitionId != suggestion.transform.id) { + originalTransition = this.contextTracker.findAndRevert(suggestion.transform.id); } if(!originalTransition) { - this.contextTracker.reset(context, suggestion.transformId); + this.contextTracker.reset(context, suggestion.transform.id); originalTransition = this.contextTracker.latest; } @@ -319,11 +320,8 @@ export class ModelCompositor { let compositor = this; let suggestions: Promise; let fallbackSuggestions = async function() { - const suggestions = await compositor.predict({insert: '', deleteLeft: 0}, context); + const suggestions = await compositor.predict({insert: '', deleteLeft: 0, id: reversion.transform.id}, context); suggestions.forEach(function(suggestion) { - // A reversion's transform ID is the additive inverse of its original suggestion; - // we revert to the state of said original suggestion. - suggestion.transformId = -reversion.transformId; // Prevent auto-selection of any suggestion immediately after a reversion. // It's fine after at least one keystroke, but not before. suggestion.autoAccept = false; @@ -337,18 +335,19 @@ export class ModelCompositor { } // When the context is tracked, we prefer the tracked information. - // Note that the base reversion's .transformId will predate the appendedTransform id + // Note that the base reversion's .transform.id will predate the appendedTransform id // used to add whitespace (if one existed), so reverting to the base ID's associated // context also reverts the appendedTransform. - let originalTransition = this.contextTracker.findAndRevert(-reversion.transformId); + const baseTransitionId = reversion.transform.id; + let originalTransition = this.contextTracker.findAndRevert(baseTransitionId); if(appendedOnly) { this.contextTracker.latest = originalTransition; return Promise.resolve([]); } - if(!originalTransition) { - this.contextTracker.reset(context, -reversion.transformId); + if(!originalTransition || originalTransition.transitionId != baseTransitionId) { + this.contextTracker.reset(context, baseTransitionId); originalTransition = this.contextTracker.latest; suggestions = fallbackSuggestions(); diff --git a/web/src/engine/predictive-text/worker-thread/src/main/models/dummy-model.ts b/web/src/engine/predictive-text/worker-thread/src/main/models/dummy-model.ts index 3b5854ef58..e1651557c4 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/models/dummy-model.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/models/dummy-model.ts @@ -98,12 +98,11 @@ export class DummyModel implements LexicalModel { predict(transform: Transform, context: Context, injectedSuggestions?: Outcome[]): Distribution { let makeUniformDistribution = function(suggestions: Outcome[]): Distribution { let distribution: Distribution = []; + const transitionId = transform.id; for(let s of suggestions) { - const transitionId = transform.id; if(transitionId !== undefined) { // Set the transform ID to match the incoming transform ID if one exists. - s.transformId = transitionId; if(s.transform) { s.transform.id = transitionId; } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index c784b5f41d..bff2607b37 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -9,7 +9,7 @@ import { ContextToken, ContextTokenLike } from './correction/context-token.js'; import { ContextTokenization } from './correction/context-tokenization.js'; import { ContextTracker } from './correction/context-tracker.js'; import { ContextState, determineContextSlideTransform } from './correction/context-state.js'; -import { ContextTransition } from './correction/context-transition.js'; +import { ContextTransition, TransitionReversionView } from './correction/context-transition.js'; import { ExecutionTimer } from './correction/execution-timer.js'; import { ModelCompositor } from './model-compositor.js'; import { EDIT_DISTANCE_COST_SCALE, getBestTokenMatches } from './correction/distance-modeler.js'; @@ -764,11 +764,6 @@ export async function correctAndEnumerate( // Corrections obtained: now to predict from them! const tokenization = tokenizations.find(t => t.spaceId == match.spaceId); - // If our 'match' results in fully deleting the new token, reject it and try again. - if(match.matchSequence.length == 0 && match.inputSequence.length != 0) { - continue; - } - // If our 'match' fully replaces the token, reject it and try again. if(match.matchSequence.length != 0 && match.matchSequence.length == match.knownCost) { continue; @@ -896,7 +891,6 @@ export function predictFromCorrectionSequence( entry.sample.transform.deleteLeft = correctionTransform.deleteLeft; if(transitionId !== undefined) { - entry.sample.transformId = transitionId; entry.sample.transform.id = transitionId; } }); @@ -1020,7 +1014,7 @@ export function composeIntermediatePredictions(predictions: TokenizedIntermediat insert: '', deleteLeft: 0 } - const transformId = predictionData.components[0].prediction.transformId; + const transformId = predictionData.components[0].prediction.transform.id; if(transformId !== undefined) { reduceBaseTransform.id = transformId; } @@ -1214,7 +1208,6 @@ export function createDefaultKeep( let keepOption = toAnnotatedSuggestion(lexicalModel, keepSuggestion, 'keep'); if(inputTransform.id !== undefined) { - keepOption.transformId = inputTransform.id; keepOption.transform.id = inputTransform.id; } keepOption.matchesModel = false; @@ -1273,9 +1266,11 @@ export function predictionAutoSelect(suggestionDistribution: CompositedIntermedi const keepOption = suggestionDistribution[0].components.prediction as Outcome; if(keepOption.tag == 'keep' && keepOption.matchesModel) { - // Auto-select it for auto-acceptance; we don't correct away from perfectly-valid - // lexical entries, even if they are comparatively low-frequency. - keepOption.autoAccept = true; + // Do not auto-select 'keep' suggestions'; there's no need to apply them. + // + // Do, however, block auto-selection of any other suggestions if we would + // have auto-selected the 'keep'; even if it is comparatively unlikely / + // low-frequency, we 'keep' the current context. return; } else if(suggestionDistribution.length == 1) { return; @@ -1370,12 +1365,6 @@ export function finalizeSuggestions( const suggestions = deduplicatedSuggestionTuples.map((tuple) => { const prediction = tuple.components.prediction; - - // Is sometimes not set during unit tests. - if(prediction.transformId !== undefined) { - prediction.transform.id = prediction.transformId; - } - const probs = tuple.metadata.probabilities; if(!verbose) { @@ -1478,9 +1467,40 @@ export function toAnnotatedSuggestion( result.appendedTransform = suggestion.appendedTransform; } - if(suggestion.transformId !== undefined) { - result.transformId = suggestion.transformId; + if(suggestion.transform.id !== undefined) { + result.transform.id = suggestion.transform.id; } return result; +} + +/** + * For applicable scenarios, this mutates the passed-in suggestion array by + * prepending a predictive-text reversion that restores the context to a prior + * state. Otherwise, it leaves the suggestion array unaltered. + * @param suggestions + * @param transitionToRevert + * @returns + */ +export function prependReversion(suggestions: Suggestion[], transitionToRevert: TransitionReversionView) { + if(transitionToRevert) { + const reversion = transitionToRevert.reversion; + if(reversion) { + if(suggestions[0]?.tag == 'keep') { + const appliedId = -reversion.id; + const appliedSuggestion = transitionToRevert.final.suggestions.find((s) => s.id == appliedId); + // If the selected suggestion was itself a `keep`, we don't need a + // reversion. They'd do the same thing. + if(appliedSuggestion.tag != 'keep') { + const keep = suggestions.shift(); + suggestions.unshift(reversion); + suggestions.unshift(keep); + } + } else { + suggestions.unshift(reversion); + } + } + } + + return suggestions; } \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts b/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts index a1b9b920b7..c5959e919b 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts @@ -3,7 +3,7 @@ export * from './correction/context-state.js'; export * from './correction/context-token.js'; export * from './correction/context-tokenization.js'; export { ContextTracker } from './correction/context-tracker.js'; -export { ContextTransition } from './correction/context-transition.js'; +export * from './correction/context-transition.js'; export * from './correction/correction-searchable.js'; export * from './correction/correction-result-mapping.js'; export * from './correction/distance-modeler.js'; diff --git a/web/src/engine/src/interfaces/prediction/predictionContext.ts b/web/src/engine/src/interfaces/prediction/predictionContext.ts index 97182a9bf1..6ebcec0db9 100644 --- a/web/src/engine/src/interfaces/prediction/predictionContext.ts +++ b/web/src/engine/src/interfaces/prediction/predictionContext.ts @@ -266,14 +266,14 @@ export class PredictionContext extends EventEmitter { this._revertSuggestion = s as Reversion; } - if (this.langProcessor.mayAutoCorrect && s.autoAccept && !this.selected) { + if (this.langProcessor.mayAutoCorrect && s.autoAccept && !this.selected && s.tag != 'keep') { this.selected = s; } } // Verify that the transition IDs are still valid and remove special entries. this._currentSuggestions = suggestions.filter(s => { - return this.langProcessor.hasState(Math.abs(s.transformId)) && + return this.langProcessor.hasState(Math.abs(s.transform.id)) && s != this._keepSuggestion && s != this._revertSuggestion }); diff --git a/web/src/engine/src/main/headless/inputProcessor.ts b/web/src/engine/src/main/headless/inputProcessor.ts index 1099b09145..5018e4de02 100644 --- a/web/src/engine/src/main/headless/inputProcessor.ts +++ b/web/src/engine/src/main/headless/inputProcessor.ts @@ -386,7 +386,7 @@ export class InputProcessor { // If so, since it behaves the same in either case, and it's a known word-breaking mark, // let's apply the selected suggestion automatically. if(postApplyTransform.insert == ruleTransform.insert && transformMatchesPattern(postApplyTransform, breakingMarks)) { - const baseTransition = this.contextCache.get(predictionContext.selected.transformId); + const baseTransition = this.contextCache.get(predictionContext.selected.transform.id); // Somehow, the base state is out of context - abort! if(!baseTransition) { diff --git a/web/src/engine/src/main/headless/languageProcessor.ts b/web/src/engine/src/main/headless/languageProcessor.ts index b98d4246df..20515c2288 100644 --- a/web/src/engine/src/main/headless/languageProcessor.ts +++ b/web/src/engine/src/main/headless/languageProcessor.ts @@ -166,6 +166,12 @@ export class LanguageProcessor extends EventEmitter { return !!this.recentTranscriptions.get(transitionId); } + /** + * When called, this requests the worker to generate predictions for the transcribed + * context transition. + * @param transcription + * @param layerId + */ public predict(transcription: Transcription, layerId: string): Promise { if(!this.isActive) { return null; @@ -218,13 +224,13 @@ export class LanguageProcessor extends EventEmitter { // Find the state of the context at the time the suggestion was generated. // This may refer to the context before an input keystroke or before application // of a predictive suggestion. - const original = this.getPredictionState(suggestion.transformId); + const original = this.getPredictionState(suggestion.transform.id); if(!original) { console.warn("Could not apply the Suggestion!"); return null; } - this.recentTranscriptions.rewindTo(suggestion.transformId); + this.recentTranscriptions.rewindTo(suggestion.transform.id); // Apply the Suggestion! @@ -306,10 +312,7 @@ export class LanguageProcessor extends EventEmitter { // Find the state of the context at the time the suggestion was generated. // This may refer to the context before an input keystroke or before application // of a predictive suggestion. - // - // Reversions use the additive inverse of the id token of the Transcription being - // reverted to. - const reversionId = appendedOnly ? reversion.appendedTransform.id : -reversion.transformId; + const reversionId = appendedOnly ? reversion.appendedTransform.id : reversion.transform.id; const original = this.getPredictionState(reversionId); if(!original) { console.warn("Could not apply the Suggestion!"); @@ -373,6 +376,9 @@ export class LanguageProcessor extends EventEmitter { this.lmEngine.resetContext(context, transcription.token); } + // The "may correct" setting is enforced here, in the main Web engine - not + // in the worker. As the desired setting may vary language-to-language, + // we'd rather do this than reconfigure the worker constantly. let alternates = transcription.alternates; if(!this.mayCorrect || !alternates || alternates.length == 0) { alternates = [{ @@ -386,6 +392,12 @@ export class LanguageProcessor extends EventEmitter { return promise.then((suggestions: Suggestion[]) => { if(promise == this.currentPromise) { + if(!this.mayAutoCorrect) { + // We disable the auto-accept flag here, rather than in the worker - + // that way, we don't need to reconfigure the worker when settings + // change. + suggestions.forEach((s) => delete s.autoAccept); + } const result = new ReadySuggestions(suggestions, transform.id); this.emit("suggestionsready", result); this.currentPromise = null; diff --git a/web/src/engine/src/main/headless/transcriptionCache.ts b/web/src/engine/src/main/headless/transcriptionCache.ts index 4a3e6408a6..7ff3832b9b 100644 --- a/web/src/engine/src/main/headless/transcriptionCache.ts +++ b/web/src/engine/src/main/headless/transcriptionCache.ts @@ -1,7 +1,7 @@ import { Transcription } from "keyman/engine/keyboard"; import { RewindableCache } from "keyman/common/web-utils"; -const TRANSCRIPTION_BUFFER_SIZE = 10; +const TRANSCRIPTION_BUFFER_SIZE = 20; export class TranscriptionCache extends RewindableCache { constructor() { diff --git a/web/src/test/auto/headless/engine/main/inputProcessor.tests.ts b/web/src/test/auto/headless/engine/main/inputProcessor.tests.ts index 1d63669537..bb469c723e 100644 --- a/web/src/test/auto/headless/engine/main/inputProcessor.tests.ts +++ b/web/src/test/auto/headless/engine/main/inputProcessor.tests.ts @@ -283,9 +283,8 @@ describe('InputProcessor', function() { [], [ { - transform: { insert: 'testing', deleteLeft: 4 }, + transform: { insert: 'testing', deleteLeft: 4, id: 0 }, appendedTransform: { insert: ' ', deleteLeft: 0 }, - transformId: 0, // will be overwritten by the DummyModel to match the transition ID. id: 1, displayAs: 'testing' } diff --git a/web/src/test/auto/headless/engine/predictive-text/templates/common.tests.ts b/web/src/test/auto/headless/engine/predictive-text/templates/common.tests.ts index 5abd27cd75..95db19df8d 100644 --- a/web/src/test/auto/headless/engine/predictive-text/templates/common.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/templates/common.tests.ts @@ -191,7 +191,6 @@ describe('Common utility functions', function() { deleteLeft: 0, id: 0 }, - transformId: 0, displayAs: 'hello' }; @@ -205,7 +204,6 @@ describe('Common utility functions', function() { deleteLeft: 0, id: 0 }, - transformId: 0, displayAs: 'hello', p: 0 }; @@ -220,7 +218,6 @@ describe('Common utility functions', function() { deleteLeft: 0, id: 0 }, - transformId: 0, displayAs: 'hello', p: 0.5 }; @@ -228,14 +225,13 @@ describe('Common utility functions', function() { assert.deepEqual(models.transformToSuggestion(suggestion.transform, 0.5), suggestion); }); - it('properly handles the transformId', function() { + it('properly handles the transition ID', function() { let suggestion = { transform: { insert: 'hello', deleteLeft: 0, id: 3 }, - transformId: 3, // Ensures there isn't a separate ID seed in use. displayAs: 'hello' }; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index af1d90aa3a..e858b76e66 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -28,7 +28,8 @@ import { models, SearchQuotientSpur, traceInsertEdits, - LegacyQuotientSpur + LegacyQuotientSpur, + TransitionEdge } from '@keymanapp/lm-worker/test-index'; import Transform = LexicalModelTypes.Transform; @@ -45,8 +46,8 @@ function toToken(text: string) { } let TOKEN_TRANSFORM_SEED = 0; -function toTransformToken(text: string, transformId?: number) { - let idSeed = transformId === undefined ? TOKEN_TRANSFORM_SEED++ : transformId; +function toTransitionToken(text: string, transitionId?: number) { + let idSeed = transitionId === undefined ? TOKEN_TRANSFORM_SEED++ : transitionId; let isWhitespace = text == ' '; let token = ContextToken.fromRawText(plainModel, ''); const textAsTransform = { insert: text, deleteLeft: 0, id: idSeed }; @@ -105,7 +106,7 @@ describe('ContextTokenization', function() { it("constructs from a token array + alignment data", () => { const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; - const tokens = rawTextTokens.map((text => toTransformToken(text))); + const tokens = rawTextTokens.map((text => toTransitionToken(text))); let tokenization = new ContextTokenization(tokens); @@ -117,8 +118,28 @@ describe('ContextTokenization', function() { it('clones', () => { const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; - const tokens = rawTextTokens.map((text => toTransformToken(text))); - let baseTokenization = new ContextTokenization(tokens); + const tokens = rawTextTokens.map((text => toTransitionToken(text))); + const emptyTransform = { insert: '', deleteLeft: 0, deleteRight: 0 }; + + // We _could_ flesh this out a bit more... but it's not really needed for this test. + const edgeWindow = buildEdgeWindow(tokens, emptyTransform, false, testEdgeWindowSpec); + let transitionEdits: TransitionEdge = { + alignment: { + merges: [], + splits: [], + unmappedEdits: [], + edgeWindow: {...edgeWindow, retokenization: rawTextTokens.slice(edgeWindow.sliceIndex)}, + removedTokenCount: 0 + }, + inputs: [{sample: (() => { + const map = new Map(); + map.set(0, emptyTransform); + return map; + })(), p: 1}], + inputSubsetId: generateSubsetId() + }; + + let baseTokenization = new ContextTokenization(tokens, transitionEdits); let cloned = new ContextTokenization(baseTokenization); assert.sameOrderedMembers( @@ -353,6 +374,43 @@ describe('ContextTokenization', function() { ); }); + it('handles simple case - deletion of final context content via backspace', () => { + const baseTokens = ['a']; + const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t))); + + const targetTokens = [''].map((t) => ({text: t, isWhitespace: t == ' '})); + const inputTransform = { insert: '', deleteLeft: 1, deleteRight: 0, id: 42 }; + const inputTransformMap: Map = new Map(); + inputTransformMap.set(0, { insert: '', deleteLeft: 1, id: 42 }); + + const edgeWindow = buildEdgeWindow(baseTokenization.tokens, inputTransform, false, testEdgeWindowSpec); + const tokenization = baseTokenization.evaluateTransition({ + alignment: { + merges: [], + splits: [], + unmappedEdits: [], + edgeWindow: { + ...edgeWindow, + // The range within the window constructed by the prior call for its parameterization. + // Any adjustments on the boundary token itself are included here. + retokenization: [...targetTokens.slice(edgeWindow.sliceIndex).map(t => t.text)] + }, + removedTokenCount: 1 + }, + inputs: [{ sample: inputTransformMap, p: 1 }], + inputSubsetId: generateSubsetId() + }, + inputTransform.id, + 1 + ); + + assert.isOk(tokenization); + assert.equal(tokenization.tokens.length, targetTokens.length); + assert.deepEqual(tokenization.tokens.map((t) => ({text: t.exampleInput, isWhitespace: t.isWhitespace})), + targetTokens + ); + }); + it('handles simple case - new character added to last token', () => { const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'da']; const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t))); @@ -779,6 +837,198 @@ describe('ContextTokenization', function() { assert.equal(preTail.exampleInput, '\''); assert.equal(tail.exampleInput, '.'); }); + + describe('properly handles previously-applied transition IDs', () => { + it('does not preserve applied transition IDs on edited tokens', () => { + const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day', ' ', 'can']; + const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t))); + + const REVERTABLE_TRANSITION_ID = 31415; + baseTokenization.tail.appliedTransitionId = REVERTABLE_TRANSITION_ID; + const NEW_TRANSITION_ID = REVERTABLE_TRANSITION_ID + 1; + + const dist = [ + { + sample: { insert: 't', deleteLeft: 0, deleteRight: 0, id: NEW_TRANSITION_ID }, + p: 1 + } + ]; + + const resultTokenization = baseTokenization.evaluateTransition( + { + alignment: { + merges: [], + splits: [], + unmappedEdits: [], + edgeWindow: { + ...buildEdgeWindow(baseTokenization.tokens, dist[0].sample, false), + retokenization: baseTokenization.tokens.map((t) => t.exampleInput), + retokenizationText: baseTokenization.tokens.map((t) => t.exampleInput).reduce((accum, curr) => accum + curr, '') + }, + removedTokenCount: 0 + }, + inputs: (() => { + const map: Map = new Map(); + map.set(0, dist[0].sample); + + return [ + {sample: map, p: 1} + ]; + })(), + inputSubsetId: 0 + }, + NEW_TRANSITION_ID, + 1 + ) + + resultTokenization.tokens.forEach((t) => { + assert.isUndefined(t.appliedTransitionId); + }); + }); + + it('preserves applied transition IDs on applicable tokens', () => { + const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day', ' ', 'can']; + const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t))); + + const REVERTABLE_TRANSITION_ID = 31415; + baseTokenization.tail.appliedTransitionId = REVERTABLE_TRANSITION_ID; + const NEW_TRANSITION_ID = REVERTABLE_TRANSITION_ID + 1; + + + const dist = [ + { + sample: { insert: ' ', deleteLeft: 0, deleteRight: 0, id: NEW_TRANSITION_ID }, + p: 1 + } + ]; + + const resultTokenization = baseTokenization.evaluateTransition( + { + alignment: { + merges: [], + splits: [], + unmappedEdits: [], + edgeWindow: { + ...buildEdgeWindow(baseTokenization.tokens, dist[0].sample, false), + retokenization: baseTokenization.tokens.map((t) => t.exampleInput), + retokenizationText: baseTokenization.tokens.map((t) => t.exampleInput).reduce((accum, curr) => accum + curr, '') + }, + removedTokenCount: 0 + }, + inputs: (() => { + const map: Map = new Map(); + map.set(1, dist[0].sample); + + return [ + {sample: map, p: 1} + ]; + })(), + inputSubsetId: 0 + }, + NEW_TRANSITION_ID, + 1 + ) + + const resultTokenLength = resultTokenization.tokens.length; + + resultTokenization.tokens.forEach((t, i) => { + // The space will add TWO tokens. + if(i == resultTokenLength - 3) { + assert.equal(t.appliedTransitionId, REVERTABLE_TRANSITION_ID); + } else { + assert.isUndefined(t.appliedTransitionId); + } + }); + }); + + // Performs the two above in sequence in a manner that could cause cross-effects + // if implemented incorrectly. + it('does not conflate effects between different tokenization transitions', () => { + const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day', ' ', 'can']; + const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t))); + + const REVERTABLE_TRANSITION_ID = 31415; + baseTokenization.tail.appliedTransitionId = REVERTABLE_TRANSITION_ID; + const NEW_TRANSITION_ID = REVERTABLE_TRANSITION_ID + 1; + + const dist = [ + { + sample: { insert: ' ', deleteLeft: 0, deleteRight: 0, id: NEW_TRANSITION_ID }, + p: .8 + }, { + sample: { insert: ' ', deleteLeft: 0, deleteRight: 0, id: NEW_TRANSITION_ID }, + p: .2 + } + ]; + + const baseTransitionEdge: TransitionEdge = { + alignment: { + merges: [], + splits: [], + unmappedEdits: [], + edgeWindow: { + ...buildEdgeWindow(baseTokenization.tokens, dist[0].sample, false), + retokenization: baseTokenization.tokens.map((t) => t.exampleInput), + retokenizationText: baseTokenization.tokens.map((t) => t.exampleInput).reduce((accum, curr) => accum + curr, '') + }, + removedTokenCount: 0 + }, + inputs: (() => { + const map: Map = new Map(); + map.set(1, dist[0].sample); + + return [ + {sample: map, p: dist[0].p} + ]; + })(), + inputSubsetId: 0 + }; + + // We don't care about the results here. What we care about is that + // this call doesn't remove the appliedTransitionId from the source + // token, preventing it from being marked on later tokenization + // transitions. + baseTokenization.evaluateTransition( + { + alignment: { + ...baseTransitionEdge.alignment, + edgeWindow: { + ...baseTransitionEdge.alignment.edgeWindow, + ...buildEdgeWindow(baseTokenization.tokens, dist[1].sample, false) + } + }, + inputs: (() => { + const map: Map = new Map(); + map.set(0, dist[1].sample); + + return [ + {sample: map, p: dist[1].p} + ]; + })(), + inputSubsetId: 0 + }, + NEW_TRANSITION_ID, + dist[1].p + ) + + const resultTokenization = baseTokenization.evaluateTransition( + baseTransitionEdge, + NEW_TRANSITION_ID, + dist[0].p + ); + + const resultTokenLength = resultTokenization.tokens.length; + + resultTokenization.tokens.forEach((t, i) => { + // The space will add TWO tokens. + if(i == resultTokenLength - 3) { + assert.equal(t.appliedTransitionId, REVERTABLE_TRANSITION_ID); + } else { + assert.isUndefined(t.appliedTransitionId); + } + }); + }); + }); }); describe('buildEdgeWindow', () => { @@ -786,7 +1036,7 @@ describe('ContextTokenization', function() { it('handles empty contexts', () => { const baseTokens = ['']; const idSeed = TOKEN_TRANSFORM_SEED; - const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t))); const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 0 }, true, testEdgeWindowSpec); assert.deepEqual(results, { @@ -806,7 +1056,7 @@ describe('ContextTokenization', function() { it('handles empty contexts and invalid Transforms', () => { const baseTokens = ['']; const idSeed = TOKEN_TRANSFORM_SEED; - const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t))); const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 2 }, true, testEdgeWindowSpec); assert.deepEqual(results, { @@ -826,7 +1076,7 @@ describe('ContextTokenization', function() { it('builds edge windows for the start of context with no edits', () => { const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; const idSeed = TOKEN_TRANSFORM_SEED; - const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t))); const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 0 }, true, testEdgeWindowSpec); assert.deepEqual(results, { @@ -866,7 +1116,7 @@ describe('ContextTokenization', function() { it('builds edge windows for the start of context with deletion edits (1)', () => { const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; const idSeed = TOKEN_TRANSFORM_SEED; - const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t))); const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 2 }, true, testEdgeWindowSpec); assert.deepEqual(results, { @@ -906,7 +1156,7 @@ describe('ContextTokenization', function() { it('builds edge windows for the start of context with deletion edits (2)', () => { const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; const idSeed = TOKEN_TRANSFORM_SEED; - const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t))); const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 4 }, true, testEdgeWindowSpec); assert.deepEqual(results, { @@ -926,7 +1176,7 @@ describe('ContextTokenization', function() { it('builds edge windows for the end of context with no edits', () => { const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; const idSeed = TOKEN_TRANSFORM_SEED; - const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t))); baseTokenization.tail.isPartial = true; const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 0 }, false, testEdgeWindowSpec); @@ -947,7 +1197,7 @@ describe('ContextTokenization', function() { it('builds edge windows for the end of context with no edits, trailing whitespace', () => { const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day', ' ', '']; const idSeed = TOKEN_TRANSFORM_SEED; - const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t))); const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 0 }, false, testEdgeWindowSpec); assert.deepEqual(results, { @@ -1734,7 +1984,7 @@ describe('ContextTokenization', function() { it('returns the standard edge window for empty transform inputs', () => { const baseTokens = ['quick', ' ', 'brown', ' ', 'fox']; const idSeed = TOKEN_TRANSFORM_SEED; - const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t))); const editTransform = { insert: '', @@ -1768,7 +2018,7 @@ describe('ContextTokenization', function() { it('returns the standard edge window for empty transforms with context-final whitespace', () => { const baseTokens = ['quick', ' ', 'brown', ' ', 'fox', ' ']; const idSeed = TOKEN_TRANSFORM_SEED; - const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t))); const editTransform = { insert: '', @@ -1803,7 +2053,7 @@ describe('ContextTokenization', function() { it('returns the standard edge window for pure transform w insert inputs', () => { const baseTokens = ['quick', ' ', 'brown', ' ', 'fox']; const idSeed = TOKEN_TRANSFORM_SEED; - const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t))); const editTransform = { insert: ' jumped', @@ -1837,7 +2087,7 @@ describe('ContextTokenization', function() { it('returns the proper edge window for transforms w deleteLeft inputs (1)', () => { const baseTokens = ['quick', ' ', 'brown', ' ', 'fox']; const idSeed = TOKEN_TRANSFORM_SEED; - const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t))); const editTransform = { insert: 'rog', @@ -1872,7 +2122,7 @@ describe('ContextTokenization', function() { it('returns the proper edge window for transforms w deleteLeft inputs (2)', () => { const baseTokens = ['quick', ' ', 'brown', ' ', 'fox']; const idSeed = TOKEN_TRANSFORM_SEED; - const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t))); const editTransform = { insert: 'fox and brown fox', // => quick fox and brown fox diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts index 989e950b24..327613758d 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts @@ -34,7 +34,7 @@ describe('ContextTracker', function() { deleteLeft: 0, id: 15 }, - transformId: 2, + transitionId: 2, id: 1, displayAs: 'world' }; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-transition.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-transition.tests.ts index d3cfd91446..2ee2c1cf50 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-transition.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-transition.tests.ts @@ -118,9 +118,8 @@ describe('ContextTransition', () => { appendedTransform: { insert: ' ', deleteLeft: 0, - id: 2 + id: 3 }, - transformId: 2, id: 10, displayAs: 'world' }, { @@ -132,9 +131,8 @@ describe('ContextTransition', () => { appendedTransform: { insert: ' ', deleteLeft: 0, - id: 2 + id: 3 }, - transformId: 2, id: 11, displayAs: 'won' }]; @@ -155,7 +153,7 @@ describe('ContextTransition', () => { // 3 long, only last token was edited. appliedTransition.base.final.displayTokenization.tokens.forEach((token, index) => { if(index >= 2) { - assert.equal(token.appliedTransitionId, suggestions[0].transformId); + assert.equal(token.appliedTransitionId, suggestions[0].transform.id); } else { assert.isUndefined(token.appliedTransitionId); } @@ -163,7 +161,7 @@ describe('ContextTransition', () => { appliedTransition.appended.final.displayTokenization.tokens.forEach((token, index) => { if(index >= 2) { - assert.equal(token.appliedTransitionId, suggestions[0].transformId); + assert.equal(token.appliedTransitionId, suggestions[0].transform.id); } else { assert.isUndefined(token.appliedTransitionId); } @@ -202,9 +200,8 @@ describe('ContextTransition', () => { appendedTransform: { insert: ' ', deleteLeft: 0, - id: 2 + id: 3 }, - transformId: 2, id: 10, displayAs: 'the' }, { @@ -216,9 +213,8 @@ describe('ContextTransition', () => { appendedTransform: { insert: ' ', deleteLeft: 0, - id: 2 + id: 3 }, - transformId: 2, id: 11, displayAs: 'and' }]; @@ -239,7 +235,7 @@ describe('ContextTransition', () => { // 3 long, only last token was edited. appliedTransition.base.final.displayTokenization.tokens.forEach((token, index) => { if(index >= 4) { - assert.equal(token.appliedTransitionId, suggestions[0].transformId); + assert.equal(token.appliedTransitionId, suggestions[0].transform.id); } else { assert.isUndefined(token.appliedTransitionId); } @@ -247,7 +243,7 @@ describe('ContextTransition', () => { appliedTransition.appended.final.displayTokenization.tokens.forEach((token, index) => { if(index >= 4) { - assert.equal(token.appliedTransitionId, suggestions[0].transformId); + assert.equal(token.appliedTransitionId, suggestions[0].transform.id); } else { assert.isUndefined(token.appliedTransitionId); } @@ -287,9 +283,8 @@ describe('ContextTransition', () => { appendedTransform: { insert: ' ', deleteLeft: 0, - id: 2 + id: 3 }, - transformId: 2, id: 10, displayAs: 'world' }, { @@ -301,9 +296,8 @@ describe('ContextTransition', () => { appendedTransform: { insert: ' ', deleteLeft: 0, - id: 2 + id: 3 }, - transformId: 2, id: 11, displayAs: 'won' }]; @@ -337,9 +331,8 @@ describe('ContextTransition', () => { appendedTransform: { insert: ' ', deleteLeft: 0, - id: 2 + id: 3 }, - transformId: 2, id: 10, displayAs: 'world' }, { @@ -351,9 +344,8 @@ describe('ContextTransition', () => { appendedTransform: { insert: ' ', deleteLeft: 0, - id: 2 + id: 3 }, - transformId: 2, id: 11, displayAs: 'won' }]; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts index 8fc319eb64..44d55fde4c 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts @@ -21,6 +21,7 @@ import { ContextToken, ContextTokenization, LegacyQuotientSpur, + legacySubsetKeyer, models, precomputationSubsetKeyer, TokenizationTransitionEdits, @@ -31,7 +32,7 @@ import Distribution = LexicalModelTypes.Distribution; import Transform = LexicalModelTypes.Transform; import TrieModel = models.TrieModel; -var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'), +const plainModel = new TrieModel(jsonFixture('models/tries/english-1000'), {wordBreaker: defaultBreaker}); function toToken(text: string) { @@ -41,6 +42,54 @@ function toToken(text: string) { return token; } +describe('legacySubsetKeyer', () => { + it('does not map backspace inputs to same result as standard key inputs', () => { + const appleToken = ContextToken.fromRawText(plainModel, 'apple', true); + + const bksp = { insert: '', deleteLeft: 1, deleteRight: 0, id: 3 }; + const bkspKey = legacySubsetKeyer({ + alignment: { + merges: [], + splits: [], + unmappedEdits: [], + edgeWindow: { + ...buildEdgeWindow([appleToken], bksp, false), + retokenization: ['appl'], + retokenizationText: 'appl' + }, + removedTokenCount: 0 + }, + tokenizedTransform: (() => { + const map: Map = new Map(); + map.set(0, bksp); + return map; + })() + }); + + const input = { insert: 's', deleteLeft: 0, deleteRight: 0, id: bksp.id }; + const inputKey = legacySubsetKeyer({ + alignment: { + merges: [], + splits: [], + unmappedEdits: [], + edgeWindow: { + ...buildEdgeWindow([appleToken], input, false), + retokenization: ['apples'], + retokenizationText: 'apples' + }, + removedTokenCount: 0 + }, + tokenizedTransform: (() => { + const map: Map = new Map(); + map.set(0, input); + return map; + })() + }); + + assert.notEqual(bkspKey, inputKey); + }); +}); + describe('precomputationSubsetKeyer', function() { it("safely generates keys for empty transition + empty contexts", () => { const rawTextTokens = ['']; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts index ecb4b09357..afcb02af09 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts @@ -16,7 +16,7 @@ describe('predictionAutoSelect', () => { assert.sameDeepOrderedMembers(predictions, originalPredictions); }); - it(`selects solitary 'keep' suggestion that does match the model`, () => { + it(`selects nothing if solitary 'keep' suggestion does match the model`, () => { const predictions: CompositedIntermediatePrediction[] = [ { components: { @@ -47,7 +47,7 @@ describe('predictionAutoSelect', () => { assert.sameDeepOrderedMembers(predictions, originalPredictions); const autoselected = predictions.find((entry) => entry.components.prediction.autoAccept); - assert.isOk(autoselected); + assert.isNotOk(autoselected); }); it(`does not select suggestions if the root correction has no letters`, () => { @@ -139,7 +139,7 @@ describe('predictionAutoSelect', () => { assert.isNotOk(autoselected); }); - it(`selects 'keep' suggestion that does match the model over any alternatives`, () => { + it(`selects nothing for 'keep' suggestion that does match the model even with alternatives`, () => { const keepSuggestion: CompositedIntermediatePrediction = { components: { prediction: { @@ -234,7 +234,7 @@ describe('predictionAutoSelect', () => { assert.sameDeepMembers(predictions, originalPredictions); const autoselected = predictions.find((entry) => entry.components.prediction.autoAccept); - assert.equal(autoselected, keepSuggestion); + assert.isNotOk(autoselected); }); it(`selects solitary non-'keep' suggestion when 'keep' does not match model`, () => { diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/base-context-state.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/base-context-state.tests.ts index 6b36685093..a38b4e3012 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/base-context-state.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/base-context-state.tests.ts @@ -168,7 +168,6 @@ describe('matchBaseContextState', () => { const suggestion: Suggestion= { transform: { insert: 'dramatically', deleteLeft: 0, id: 3 }, displayAs: 'dramatically', - transformId: 3, id: 5 } @@ -234,7 +233,6 @@ describe('matchBaseContextState', () => { const suggestion: Suggestion= { transform: { insert: '', deleteLeft: 'dramatically'.length, id: 3 }, displayAs: '""', - transformId: 3, id: 5 } diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts index 6bf18e3a36..8bed658ed4 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts @@ -110,7 +110,6 @@ describe('createDefaultKeep', () => { deleteLeft: 4, id: transformId }, - transformId, displayAs: '', matchesModel: false, tag: 'keep' diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts index 8cdbed20b2..2412af3fcd 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts @@ -135,7 +135,6 @@ describe('determineContextTransition', () => { insert: ' ', deleteLeft: 0 }, - transformId: 0, displayAs: 'testing' }; baseTransition.final.suggestions = [pred_testing]; @@ -149,7 +148,7 @@ describe('determineContextTransition', () => { } }; - compositor.acceptSuggestion(applied_testing, baseContext, { insert: '', deleteLeft: 0 }); + compositor.acceptSuggestion(applied_testing, baseContext, { insert: '', deleteLeft: 0, id: applied_testing.transform.id }); const acceptingTransition = tracker.latest; const inputDistribution: Distribution = [{sample: applied_testing.appendedTransform, p: 1}]; @@ -190,7 +189,7 @@ describe('determineContextTransition', () => { transform: { insert: 'testing', deleteLeft: 4, - id: 1 + id: 0 }, appendedTransform: { insert: ' ', @@ -198,12 +197,11 @@ describe('determineContextTransition', () => { id: 2 }, id: 4, - transformId: 0, displayAs: 'testing' }; baseTransition.final.suggestions = [pred_testing]; - compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0 }); + compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0, id: pred_testing.transform.id }); const inputDistribution: Distribution = [{sample: { insert: 'a', deleteLeft: 0, id: 5 }, p: 1}]; @@ -224,8 +222,8 @@ describe('determineContextTransition', () => { assert.notEqual(extendingTransition, baseTransition); // These values support delayed reversions. - assert.equal(extendingTransition.final.displayTokenization.tokens[6].appliedTransitionId, pred_testing.transformId); - assert.equal(extendingTransition.final.displayTokenization.tokens[7].appliedTransitionId, pred_testing.transformId); + assert.equal(extendingTransition.final.displayTokenization.tokens[6].appliedTransitionId, pred_testing.transform.id); + assert.equal(extendingTransition.final.displayTokenization.tokens[7].appliedTransitionId, pred_testing.transform.id); // We start a new token here, rather than continue (and/or replace) an old one; // this shouldn't be set here yet. @@ -251,19 +249,18 @@ describe('determineContextTransition', () => { transform: { insert: 'testing', deleteLeft: 4, - id: 1 + id: 0 }, appendedTransform: { insert: ' ', deleteLeft: 0, id: 2 }, - transformId: 0, displayAs: 'testing' }; baseTransition.final.suggestions = [pred_testing]; - compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0 }); + compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0, id: pred_testing.transform.id }); const inputDistribution: Distribution = [{sample: { insert: 'a', deleteLeft: 0, id: 5 }, p: 1}]; @@ -291,7 +288,7 @@ describe('determineContextTransition', () => { [{sample: { insert: '', deleteLeft: 1 }, p: 1}] ); - assert.equal(extensionDeletingTransition.revertableTransitionId, pred_testing.transformId); + assert.equal(extensionDeletingTransition.revertableTransitionId, pred_testing.transform.id); } finally { warningEmitterSpy.restore(); } @@ -313,19 +310,18 @@ describe('determineContextTransition', () => { transform: { insert: 'testing', deleteLeft: 4, - id: 1 + id: 0 }, appendedTransform: { insert: ' ', deleteLeft: 0, id: 2 }, - transformId: 0, displayAs: 'testing' }; baseTransition.final.suggestions = [pred_testing]; - compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0 }); + compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0, id: pred_testing.transform.id }); const inputDistribution: Distribution = [{sample: { insert: 'a', deleteLeft: 0, id: 5 }, p: 1}]; @@ -365,7 +361,7 @@ describe('determineContextTransition', () => { [{sample: { insert: '', deleteLeft: 1 }, p: 1}] ); - assert.equal(appendDeletingTransition.revertableTransitionId, pred_testing.transformId); + assert.equal(appendDeletingTransition.revertableTransitionId, pred_testing.transform.id); } finally { warningEmitterSpy.restore(); } @@ -387,19 +383,18 @@ describe('determineContextTransition', () => { transform: { insert: 'testing', deleteLeft: 4, - id: 1 + id: 0 }, appendedTransform: { insert: ' ', deleteLeft: 0, id: 2 }, - transformId: 0, displayAs: 'testing' }; baseTransition.final.suggestions = [pred_testing]; - compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0 }); + compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0, id: pred_testing.transform.id }); const inputDistribution: Distribution = [{sample: { insert: 'a', deleteLeft: 0, id: 5 }, p: 1}]; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts index c1f9f63d07..6b755be5c0 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts @@ -137,7 +137,6 @@ describe('predictFromCorrectionSequence', () => { assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components[0].prediction), dummied_suggestions.map((s) => { delete s.p; - s.transformId = transitionID; s.transform.id = transitionID; return s; })); @@ -205,14 +204,13 @@ describe('predictFromCorrectionSequence', () => { assert.sameOrderedMembers(predictions.map((entry) => entry.components[0].prediction.displayAs), ["it's", "its"]); assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components[0].prediction), dummied_suggestions.map((entry) => { entry = deepCopy(entry); - entry.transformId = transitionID; entry.transform.id = transitionID; return entry; })); assert.approximately(predictions[0].metadata.probabilities.total, 0.18 * 0.6, 0.00001); assert.approximately(predictions[1].metadata.probabilities.total, 0.02 * 0.6, 0.00001); - predictions.forEach((prediction) => assert.equal(prediction.components[0].prediction.transformId, transitionID)); + predictions.forEach((prediction) => assert.equal(prediction.components[0].prediction.transform.id, transitionID)); }); it('constructs suggestions without input (as if after a context reset)', () => { @@ -266,7 +264,6 @@ describe('predictFromCorrectionSequence', () => { assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components.map((c) => c.prediction)), [dummied_suggestions.map((s) => { delete s.p; - s.transformId = transitionID; s.transform.id = transitionID; return s; })]); @@ -355,24 +352,21 @@ describe('predictFromCorrectionSequence', () => { deleteLeft: 0, id: transitionID }, - displayAs: 'g', - transformId: transitionID, + displayAs: 'g' }, { transform: { insert: ' ', deleteLeft: 0, id: transitionID }, - displayAs: ' ', - transformId: transitionID + displayAs: ' ' }, { transform: { insert: 'apple', deleteLeft: 0, id: transitionID }, - displayAs: 'apple', - transformId: transitionID + displayAs: 'apple' } ]; @@ -565,24 +559,21 @@ describe('predictFromCorrectionSequence', () => { deleteLeft: 0, id: transitionID }, - displayAs: 'g', - transformId: transitionID + displayAs: 'g' }, { transform: { insert: ' ', deleteLeft: 0, id: transitionID }, - displayAs: ' ', - transformId: transitionID + displayAs: ' ' }, { transform: { insert: 'apple', deleteLeft: 0, id: transitionID }, - displayAs: 'apple', - transformId: transitionID + displayAs: 'apple' } ]; @@ -710,16 +701,14 @@ describe('predictFromCorrectionSequence', () => { deleteLeft: 0, id: transitionID }, - displayAs: 'golden', - transformId: transitionID + displayAs: 'golden' }, { transform: { insert: ' ', deleteLeft: 0, id: transitionID }, - displayAs: ' ', - transformId: transitionID + displayAs: ' ' } ]; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/prepend-reversion.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/prepend-reversion.tests.ts new file mode 100644 index 0000000000..5215705ed6 --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/prepend-reversion.tests.ts @@ -0,0 +1,101 @@ + +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2026-07-27 + * + * This file contains tests designed to ensure predictive text does not + * provide matching 'keep' and 'revert' suggestions in any context. + */ + + +import { assert } from 'chai'; + +import { LexicalModelTypes } from '@keymanapp/common-types'; +import { prependReversion, type TransitionReversionView } from "@keymanapp/lm-worker/test-index"; + +import Suggestion = LexicalModelTypes.Suggestion; + +describe('prependReversion', () => { + it(`prepends reversions when reverting a non-'keep' suggestion`, () => { + // context: Original was appl+u, corrected to apply. Reached via bksp. + const suggestions: Suggestion[] = [{ + tag: 'keep', + transform: { insert: 'apply', deleteLeft: 6, id: 3 }, + displayAs: '"apply"', + id: 5, + matchesModel: false + } as Suggestion]; + + const revertable: TransitionReversionView = { + reversion: { + tag: 'revert', + transform: { insert: 'u', deleteLeft: 0, id: 3 }, + id: -3, + displayAs: '"applu"' + }, + final: { + suggestions: [{ + tag: 'keep', + transform: { insert: 'applu', deleteLeft: 4, id: 3 }, + displayAs: '"applu"', + id: 2, + matchesModel: false + } as Suggestion, { + transform: { insert: 'apply', deleteLeft: 4, id: 3 }, + displayAs: 'apply', + id: 3 + }, { + transform: { insert: 'applied', deleteLeft: 4, id: 3 }, + displayAs: 'applied', + id: 4 + }] + } + }; + + prependReversion(suggestions, revertable); + + assert.includeMembers(suggestions, [revertable.reversion]); + }); + + it(`does not prepend reversions when reverting a 'keep' suggestion`, () => { + // context: Original was appl+u, corrected to apply + const suggestions: Suggestion[] = [{ + tag: 'keep', + transform: { insert: 'applu', deleteLeft: 5, id: 3 }, + displayAs: '"applu"', + id: 5, + matchesModel: false + } as Suggestion]; + + const revertable: TransitionReversionView = { + reversion: { + tag: 'revert', + transform: { insert: 'u', deleteLeft: 0, id: 3 }, + id: -2, + displayAs: '"applu"' + }, + final: { + suggestions: [{ + tag: 'keep', + transform: { insert: 'applu', deleteLeft: 4, id: 3 }, + displayAs: '"applu"', + id: 2, + matchesModel: false + } as Suggestion, { + transform: { insert: 'apply', deleteLeft: 4, id: 3 }, + displayAs: 'apply', + id: 3 + }, { + transform: { insert: 'applied', deleteLeft: 4, id: 3 }, + displayAs: 'applied', + id: 4 + }] + } + }; + + prependReversion(suggestions, revertable); + + assert.notIncludeMembers(suggestions, [revertable.reversion]); + }); +}); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts index d20bc7c976..5c28e2a486 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts @@ -296,7 +296,6 @@ describe('processSimilarity', () => { deleteLeft: 4, id: transformId }, - transformId, displayAs: 'apple' }, correction: 'appl' diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts index 8c20df9626..5320784777 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts @@ -588,7 +588,6 @@ describe('ModelCompositor', function() { deleteLeft: 0, id: 0 }, - transformId: 0, displayAs: 'hello' }; @@ -668,13 +667,12 @@ describe('ModelCompositor', function() { } it('first word of context, postTransform provided, .deleteLeft = 0', function() { - let baseSuggestion = { + let baseSuggestion: Suggestion = { transform: { insert: 'hello ', deleteLeft: 2, id: 0 }, - transformId: 0, id: 1, displayAs: 'hello' }; @@ -687,11 +685,12 @@ describe('ModelCompositor', function() { // of the Context when the suggestion is built. let postTransform = { insert: 'l', - deleteLeft: 0 + deleteLeft: 0, + id: baseSuggestion.transform.id } let reversion = acceptanceTest(englishPunctuation, baseSuggestion, baseContext, postTransform); - assert.equal(reversion.transformId, baseSuggestion.transformId); + assert.equal(reversion.transform.id, baseSuggestion.transform.id); assert.equal(reversion.id, -baseSuggestion.id); // Check #1: Does the returned reversion properly revert the context to its pre-application state? @@ -711,13 +710,12 @@ describe('ModelCompositor', function() { }); it('second word of context, postTransform provided, .deleteLeft = 0', function() { - let baseSuggestion = { + let baseSuggestion: Suggestion = { transform: { insert: 'world ', deleteLeft: 3, id: 0 }, - transformId: 0, id: 0, displayAs: 'world' }; @@ -730,11 +728,12 @@ describe('ModelCompositor', function() { // of the Context when the suggestion is built. let postTransform = { insert: 'l', - deleteLeft: 0 + deleteLeft: 0, + id: baseSuggestion.id } let reversion = acceptanceTest(englishPunctuation, baseSuggestion, baseContext, postTransform); - assert.equal(reversion.transformId, baseSuggestion.transformId); + assert.equal(reversion.transform.id, baseSuggestion.transform.id); assert.equal(reversion.id, -baseSuggestion.id); // Check #1: Does the returned reversion properly revert the context to its pre-application state? @@ -754,13 +753,12 @@ describe('ModelCompositor', function() { }); it('second word of context, postTransform undefined', function() { - let baseSuggestion = { + let baseSuggestion: Suggestion = { transform: { insert: 'world ', deleteLeft: 3, id: 0 }, - transformId: 0, id: 0, displayAs: 'world' }; @@ -770,7 +768,7 @@ describe('ModelCompositor', function() { } let reversion = acceptanceTest(englishPunctuation, baseSuggestion, baseContext); - assert.equal(reversion.transformId, baseSuggestion.transformId); + assert.equal(reversion.transform.id, baseSuggestion.transform.id); assert.equal(reversion.id, -baseSuggestion.id); // Check #1: Does the returned reversion properly revert the context to its pre-application state? @@ -790,13 +788,12 @@ describe('ModelCompositor', function() { }); it('first word of context + postTransform provided, .deleteLeft > 0', function() { - let baseSuggestion = { + let baseSuggestion: Suggestion = { transform: { insert: 'hello ', deleteLeft: 2, id: 0 }, - transformId: 0, id: 0, displayAs: 'hello' }; @@ -809,11 +806,12 @@ describe('ModelCompositor', function() { // of the Context when the suggestion is built. let postTransform = { insert: 'i', - deleteLeft: 1 + deleteLeft: 1, + id: baseSuggestion.transform.id } let reversion = acceptanceTest(englishPunctuation, baseSuggestion, baseContext, postTransform); - assert.equal(reversion.transformId, baseSuggestion.transformId); + assert.equal(reversion.transform.id, baseSuggestion.transform.id); assert.equal(reversion.id, -baseSuggestion.id); // Check #1: Does the returned reversion properly revert the context to its pre-application state? @@ -847,13 +845,12 @@ describe('ModelCompositor', function() { // it('first word of context + postTransform provided, .deleteLeft > 0') // seen earlier in the file. - let baseSuggestion = { + let baseSuggestion: Suggestion = { transform: { insert: 'hello ', deleteLeft: 2, id: 0 }, - transformId: 0, id: 0, displayAs: 'hello' }; @@ -866,7 +863,8 @@ describe('ModelCompositor', function() { // of the Context when the suggestion is built. let postTransform = { insert: 'i', - deleteLeft: 1 + deleteLeft: 1, + id: baseSuggestion.transform.id } // Future adjustment: add the 'baseSuggestion' to DummyModel so that it actually @@ -876,7 +874,7 @@ describe('ModelCompositor', function() { let compositor = new ModelCompositor(model, true); let reversion = compositor.acceptSuggestion(baseSuggestion, baseContext, postTransform); - assert.equal(reversion.transformId, baseSuggestion.transformId); + assert.equal(reversion.transform.id, baseSuggestion.transform.id); assert.equal(reversion.id, -baseSuggestion.id); let appliedContext = models.applyTransform(baseSuggestion.transform, baseContext); @@ -891,7 +889,8 @@ describe('ModelCompositor', function() { let expectedTransform = { insert: 'hi', // Keeps current context the same, though it adds a wordbreak. - deleteLeft: 2 + deleteLeft: 2, + id: 0 } assert.deepEqual(suggestions[0].transform, expectedTransform); assert.deepEqual(suggestions[0].appendedTransform, { @@ -927,7 +926,7 @@ describe('ModelCompositor', function() { let baseSuggestion = initialSuggestions[1]; baseSuggestion.appendedTransform.id = 15; // set an id for the applied transform. let reversion = compositor.acceptSuggestion(baseSuggestion, baseContext, postTransform); - assert.equal(reversion.transformId, -baseSuggestion.transformId); + assert.equal(reversion.transform.id, baseSuggestion.transform.id); assert.equal(reversion.id, -baseSuggestion.id); let appliedContext = models.applyTransform(baseSuggestion.transform, baseContext); @@ -974,7 +973,7 @@ describe('ModelCompositor', function() { assert.equal(compositor.contextTracker.unitTestEndPoints.cache().size, 2); let contextIds = compositor.contextTracker.unitTestEndPoints.cache().keys(); - assert.equal(reversion.transformId, -baseSuggestion.transformId); + assert.equal(reversion.transform.id, baseSuggestion.transform.id); assert.equal(reversion.id, -baseSuggestion.id); const appliedContextState = compositor.contextTracker.unitTestEndPoints.cache().get(15);