erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-08-05 00:15:32 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

Merge branch 'epic/autocorrect' into change/merge-autocorrect-into-boundarycorrect

Browse source
This commit is contained in:
Joshua Horton 2026-07-30 14:14:21 -05:00
commit 6896981bf6
35 changed files with 785 additions and 286 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
common/web/types/src/lexical-model-types.ts
View file

@ -253,9 +253,14 @@ export interface LexicalModel {
*/
export interface Transform {
/**
* Facilitates use of unique identifiers for tracking the Transform and
* any related data from its original source, as the reference cannot be
* preserved across WebWorker boundaries.
* Facilitates use of unique identifiers for tracking data about the context
* transition to which the Transform belongs. More than one Transform may
* hold the same `id` if they are alternate interpretations of the same
* transition event - say, the resulting effects of neighbor keys that may
* have been missed due to "fat fingering".
*
* Also note that the Transform reference cannot be preserved across WebWorker
* boundaries, but this ID may.
*
* This is *separate* from any LMLayer-internal identification values.
*/
@ -287,13 +292,6 @@ export interface Transform {
* A concrete suggestion
*/
export interface Suggestion {
/**
* Indicates the externally-supplied id of the Transform that prompted
* the Suggestion. Automatically handled by the LMLayer; models should
* not handle this field.
*/
transformId?: number;
/**
* A unique identifier for the Suggestion itself, not shared with any others -
* even for Suggestions sourced from the same Transform.

1
ios/engine/KMEI/KeymanEngine/Classes/Constants.swift
View file

@ -33,6 +33,7 @@ public enum Key {
/// Dictionary of prediction/correction toggle settings keyed by language id in UserDefaults
static let userPredictSettings = "UserPredictionEnablementSettings"
static let userCorrectSettings = "UserCorrectionEnablementSettings"
static let userAutocorrectSettings = "UserAutocorrectionEnablementSettings"
// Internal user defaults keys
static let engineVersion = "KeymanEngineVersion"

29
ios/engine/KMEI/KeymanEngine/Classes/Extension/UserDefaults+Types.swift
View file

@ -331,6 +331,17 @@ public extension UserDefaults {
set(prefs, forKey: Key.userCorrectSettings)
}
}
// stores a dictionary of autocorrection-enablement settings keyed to language ids, i.e., [langID: Bool]
var autocorrectionEnablements: [String: Bool]? {
get {
return dictionary(forKey: Key.userAutocorrectSettings) as? [String : Bool]
}
set(prefs) {
set(prefs, forKey: Key.userAutocorrectSettings)
}
}
var portraitKeyboardHeight: Double {
get {
@ -387,4 +398,22 @@ public extension UserDefaults {
prefs?[forLanguageID] = correctSetting
correctionEnablements = prefs
}
func autocorrectSettingForLanguage(languageID: String) -> Bool {
if let dict = autocorrectionEnablements {
return dict[languageID] ?? true
} else {
return true
}
}
func set(autocorrectSetting: Bool, forLanguageID: String) {
var prefs: [String: Bool]?
prefs = autocorrectionEnablements
if prefs == nil {
prefs = [String: Bool]()
}
prefs?[forLanguageID] = autocorrectSetting
autocorrectionEnablements = prefs
}
}

3
ios/engine/KMEI/KeymanEngine/Classes/Keyboard/KeymanWebViewController.swift
View file

@ -433,10 +433,11 @@ extension KeymanWebViewController {
let predict = userDefaults.predictSettingForLanguage(languageID: lexicalModel.languageID)
let correct = userDefaults.correctSettingForLanguage(languageID: lexicalModel.languageID)
let autocorrect = userDefaults.autocorrectSettingForLanguage(languageID: lexicalModel.languageID)
// Pass these off to KMW!
// We do these first so that they're automatically set for the to-be-registered model in advance.
webView!.evaluateJavaScript("enableSuggestions(\(stubString), \(predict), \(correct))")
webView!.evaluateJavaScript("enableSuggestions(\(stubString), \(predict), \(correct), \(autocorrect))")
self.activeModel = predict
} else { // We're registering a model in the background - don't change settings.
webView!.evaluateJavaScript("keyman.addModel(\(stubString));", completionHandler: nil)

134
ios/engine/KMEI/KeymanEngine/Classes/Settings/LanguageSettingsViewController.swift
View file

@ -18,8 +18,11 @@ class LanguageSettingsViewController: UITableViewController {
private var doPredictionsSwitch: UISwitch?
private var doCorrectionsSwitch: UISwitch?
private var doAutocorrectionsSwitch: UISwitch?
private var doCorrectionsLabel: UILabel?
private var doAutocorrectionsLabel: UILabel?
private var correctionsCell: UITableViewCell?
private var autocorrectionsCell: UITableViewCell?
public init(_ inLanguage: Language) {
language = inLanguage
@ -87,17 +90,8 @@ class LanguageSettingsViewController: UITableViewController {
return switchFrame
}
@objc
func predictionSwitchValueChanged(source: UISwitch) {
let value = source.isOn;
func refreshModelIfNeeded() {
let userDefaults = Storage.active.userDefaults
userDefaults.set(predictSetting: value, forLanguageID: self.language.id)
// Reactively set the corrections switch interactivity state.
self.doCorrectionsSwitch?.isHidden = !value
self.doCorrectionsLabel?.isEnabled = value
self.correctionsCell?.isUserInteractionEnabled = value
if let lm = Manager.shared.preferredLexicalModel(userDefaults, forLanguage: self.language.id) {
if Manager.shared.currentKeyboardID?.languageID == self.language.id {
// re-register the model - that'll enact the settings.
@ -111,23 +105,63 @@ class LanguageSettingsViewController: UITableViewController {
}
}
func refreshSwitchVisibility() {
let userDefaults = Storage.active.userDefaults
let mayPredict = userDefaults.predictSettingForLanguage(languageID: self.language.id)
self.doCorrectionsSwitch?.isHidden = !mayPredict
self.doCorrectionsLabel?.isEnabled = mayPredict
self.correctionsCell?.isUserInteractionEnabled = mayPredict
let mayCorrect = userDefaults.correctSettingForLanguage(languageID: self.language.id)
self.doAutocorrectionsSwitch?.isHidden = !(mayPredict && mayCorrect)
self.doAutocorrectionsLabel?.isEnabled = mayPredict && mayCorrect
self.autocorrectionsCell?.isUserInteractionEnabled = mayPredict && mayCorrect
}
@objc
func predictionSwitchValueChanged(source: UISwitch) {
let value = source.isOn;
let userDefaults = Storage.active.userDefaults
userDefaults.set(predictSetting: value, forLanguageID: self.language.id)
refreshSwitchVisibility()
refreshModelIfNeeded()
}
@objc
func correctionSwitchValueChanged(source: UISwitch) {
let value = source.isOn;
let userDefaults = Storage.active.userDefaults
userDefaults.set(correctSetting: value, forLanguageID: self.language.id)
if let lm = Manager.shared.preferredLexicalModel(userDefaults, forLanguage: self.language.id) {
if Manager.shared.currentKeyboardID?.languageID == self.language.id {
// re-register the model - that'll enact the settings.
_ = Manager.shared.registerLexicalModel(lm)
}
// Based on how Manager chooses the model in setKeyboard.
} else if let lm = userDefaults.userLexicalModels?.first(where: { $0.languageID == self.language.id }) {
if Manager.shared.currentKeyboardID?.languageID == self.language.id {
_ = Manager.shared.registerLexicalModel(lm)
}
}
refreshSwitchVisibility()
refreshModelIfNeeded()
}
@objc
func autocorrectionSwitchValueChanged(source: UISwitch) {
let value = source.isOn;
let userDefaults = Storage.active.userDefaults
userDefaults.set(autocorrectSetting: value, forLanguageID: self.language.id)
refreshModelIfNeeded()
}
func addSwitchToTableCell(_ toggle: UISwitch, cell: UITableViewCell, isOn: Bool, selector: Selector) {
cell.accessoryType = .none
toggle.translatesAutoresizingMaskIntoConstraints = false
let switchFrame = frameAtRightOfCell(cell: cell.frame, controlSize: toggle.frame.size)
toggle.frame = switchFrame
toggle.isOn = isOn
toggle.addTarget(self, action: selector, for: .valueChanged)
cell.addSubview(toggle)
cell.contentView.isUserInteractionEnabled = false
toggle.rightAnchor.constraint(equalTo: cell.layoutMarginsGuide.rightAnchor).isActive = true
toggle.centerYAnchor.constraint(equalTo: cell.layoutMarginsGuide.centerYAnchor).isActive = true
}
override func tableView(_ tableView: UITableView, cellForRowAt indexPath: IndexPath) -> UITableViewCell {
@ -143,40 +177,38 @@ class LanguageSettingsViewController: UITableViewController {
let cell = UITableViewCell(style: .subtitle, reuseIdentifier: cellIdentifier)
if 1 == indexPath.section {
if 0 == indexPath.row {
cell.accessoryType = .none
doPredictionsSwitch = UISwitch()
doPredictionsSwitch!.translatesAutoresizingMaskIntoConstraints = false
let switchFrame = frameAtRightOfCell(cell: cell.frame, controlSize: doPredictionsSwitch!.frame.size)
doPredictionsSwitch!.frame = switchFrame
doPredictionsSwitch!.isOn = userDefaults.predictSettingForLanguage(languageID: self.language.id)
doPredictionsSwitch!.addTarget(self, action: #selector(self.predictionSwitchValueChanged), for: .valueChanged)
cell.addSubview(doPredictionsSwitch!)
cell.contentView.isUserInteractionEnabled = false
doPredictionsSwitch!.rightAnchor.constraint(equalTo: cell.layoutMarginsGuide.rightAnchor).isActive = true
doPredictionsSwitch!.centerYAnchor.constraint(equalTo: cell.layoutMarginsGuide.centerYAnchor).isActive = true
self.addSwitchToTableCell(
doPredictionsSwitch!,
cell: cell,
isOn: userDefaults.predictSettingForLanguage(languageID: self.language.id),
selector: #selector(self.predictionSwitchValueChanged)
)
} else if 1 == indexPath.row {
correctionsCell = cell
cell.accessoryType = .none
doCorrectionsSwitch = UISwitch()
doCorrectionsSwitch!.translatesAutoresizingMaskIntoConstraints = false
let switchFrame = frameAtRightOfCell(cell: cell.frame, controlSize: doCorrectionsSwitch!.frame.size)
doCorrectionsSwitch!.frame = switchFrame
self.addSwitchToTableCell(
doCorrectionsSwitch!,
cell: cell,
isOn: userDefaults.correctSettingForLanguage(languageID: self.language.id),
selector: #selector(self.correctionSwitchValueChanged)
)
refreshSwitchVisibility()
} else if 2 == indexPath.row {
autocorrectionsCell = cell
doAutocorrectionsSwitch = UISwitch()
doCorrectionsSwitch!.isOn = userDefaults.correctSettingForLanguage(languageID: self.language.id)
doCorrectionsSwitch!.addTarget(self, action: #selector(self.correctionSwitchValueChanged), for: .valueChanged)
cell.addSubview(doCorrectionsSwitch!)
cell.contentView.isUserInteractionEnabled = false
self.addSwitchToTableCell(
doAutocorrectionsSwitch!,
cell: cell,
isOn: userDefaults.autocorrectSettingForLanguage(languageID: self.language.id),
selector: #selector(self.autocorrectionSwitchValueChanged)
)
doCorrectionsSwitch!.rightAnchor.constraint(equalTo: cell.layoutMarginsGuide.rightAnchor).isActive = true
doCorrectionsSwitch!.centerYAnchor.constraint(equalTo: cell.layoutMarginsGuide.centerYAnchor).isActive = true
// Disable interactivity if the prediction toggle is set to 'off'.
doCorrectionsSwitch!.isHidden = !userDefaults.predictSettingForLanguage(languageID: self.language.id)
cell.isUserInteractionEnabled = userDefaults.predictSettingForLanguage(languageID: self.language.id)
refreshSwitchVisibility()
} else { // rows 3 and 4
cell.accessoryType = .disclosureIndicator
}
@ -245,6 +277,10 @@ class LanguageSettingsViewController: UITableViewController {
cell.textLabel?.text = NSLocalizedString("menu-langsettings-toggle-correct", bundle: engineBundle, comment: "")
cell.textLabel?.isEnabled = !(doCorrectionsSwitch?.isHidden ?? false)
case 2:
doAutocorrectionsLabel = cell.textLabel
cell.textLabel?.text = NSLocalizedString("menu-langsettings-toggle-autocorrect", bundle: engineBundle, comment: "")
cell.textLabel?.isEnabled = !(doAutocorrectionsSwitch?.isHidden ?? false)
case 3:
cell.textLabel?.text = NSLocalizedString("menu-langsettings-label-lexical-models", bundle: engineBundle, comment: "")
cell.accessoryType = .disclosureIndicator
let modelCt = language.lexicalModels?.count ?? 0
@ -332,8 +368,8 @@ class LanguageSettingsViewController: UITableViewController {
showKeyboardInfoView(kb: (language.keyboards?[safe: indexPath.row])!)
case 1:
switch indexPath.row {
// case 0, 1: the toggles - but a general 'click' not on the toggle itself.
case 2:
// case 0, 1, 2: the toggles - but a general 'click' not on the toggle itself.
case 3:
showLexicalModelsView()
default:
break

7
ios/engine/KMEI/KeymanEngine/en.lproj/Localizable.strings
View file

@ -152,10 +152,13 @@
"menu-langsettings-title" = "%@ Settings";
/* Label for the toggle that enables corrections that is displayed within a language-specific settings menu */
"menu-langsettings-toggle-correct" = "Enable corrections";
"menu-langsettings-toggle-correct" = "Offer corrections";
/* Label for the toggle that enables auto-corrections that is displayed within a language-specific settings menu */
"menu-langsettings-toggle-autocorrect" = "Apply corrections automatically";
/* Label for the toggle that enables predictions that is displayed within a language-specific settings menu */
"menu-langsettings-toggle-predict" = "Enable predictions";
"menu-langsettings-toggle-predict" = "Offer word completions";
/* Help message for a prompt that appears for confirming a lexical model download: language (1): lexical model (dictionary) name (2) */
"menu-lexical-model-install-message" = "Would you like to install this dictionary?";

3
ios/engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/ios-host.js
View file

@ -326,11 +326,12 @@ function toHex(theString) {
return hexString.substr(0, hexString.length-1);
}
function enableSuggestions(model, mayPredict, mayCorrect) {
function enableSuggestions(model, mayPredict, mayCorrect, mayAutocorrect) {
// Set the options first so that KMW's ModelCache can properly handle model enablement states
// the moment we actually register the new model.
keyman.core.languageProcessor.mayPredict = mayPredict;
keyman.core.languageProcessor.mayCorrect = mayCorrect;
keyman.core.languageProcessor.mayAutoCorrect = mayAutocorrect;
keyman.addModel(model);
}

4
web/src/engine/predictive-text/templates/src/common.ts
View file

@ -132,10 +132,6 @@ export function transformToSuggestion(transform: Transform, p?: number): Outcome
displayAs: transform.insert
};
if(transform.id !== undefined) {
suggestion.transformId = transform.id;
}
if(p === 0 || p) {
suggestion.p = p;
}

2
web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts
View file

@ -95,7 +95,7 @@ export class ContextState {
return undefined;
}
return this.suggestions.find(s => s.id == this.appliedSuggestionId)?.transformId;
return this.suggestions.find(s => s.id == this.appliedSuggestionId)?.transform.id;
}
/**

14
web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts
View file

@ -10,7 +10,7 @@
import { LexicalModelTypes } from '@keymanapp/common-types';
import { SearchQuotientNode, PathInputProperties } from "./search-quotient-node.js";
import { TokenSplitMap } from "./context-tokenization.js";
import { TokenSplitMapping } from "./context-tokenization.js";
import { LegacyQuotientSpur } from "./legacy-quotient-spur.js";
import { LegacyQuotientRoot } from "./legacy-quotient-root.js";
import { generateSubsetId } from './tokenization-subsets.js';
@ -26,12 +26,12 @@ import Transform = LexicalModelTypes.Transform;
* any prior cached data or for rewriting its probabilities after
* receiving backspace input.
* @param text
* @param transformId
* @param transitionId
* @returns
*/
function textToCharTransforms(text: string, transformId?: number): Transform[] {
return transformId ?
[...text].map(insert => ({insert, deleteLeft: 0, id: transformId})) :
function textToCharTransforms(text: string, transitionId?: number): Transform[] {
return transitionId ?
[...text].map(insert => ({insert, deleteLeft: 0, id: transitionId})) :
[...text].map(insert => ({insert, deleteLeft: 0}));
}
@ -134,7 +134,7 @@ export class ContextToken implements ContextTokenLike {
static fromRawText(model: LexicalModel, rawText: string, isPartial?: boolean) {
rawText ||= '';
// Supports the old pathway for: updateWithBackspace(tokenText: string, transformId: number)
// Supports the old pathway for: updateWithBackspace(tokenText: string, transitionId: number)
// Build a token that represents the current text with no ambiguity - probability at max (1.0)
let searchModule: SearchQuotientNode = new LegacyQuotientRoot(model);
const BASE_PROBABILITY = 1;
@ -233,7 +233,7 @@ export class ContextToken implements ContextTokenLike {
* @param lexicalModel
* @returns
*/
split(split: TokenSplitMap): ContextToken[] {
split(split: TokenSplitMapping): ContextToken[] {
// Split from tail to head - leave as much 'head' intact as possible at each
// step, rather than needing to reconstruct the tail multiple times.
const splitSpecs = split.matches.slice();

70
web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
View file

@ -32,7 +32,7 @@ const MIN_CHARS_TO_RECONSIDER_FOR_TOKENIZATION = 8;
* This type is used to indicate properties of tokens affected by merges and
* splits during a context transition.
*/
interface EditTokenMap {
interface EditTokenMappingHalf {
/**
* The index of the affected token.
*/
@ -43,10 +43,22 @@ interface EditTokenMap {
text: string
};
/**
* This type is used to indicate properties of tokens affected by splits during
* a context transition.
*/
interface SplitTokenMappingHalf extends EditTokenMappingHalf {
/**
* The codepoint index within the original token at which the split-off token
* begins.
*/
textOffset: number
};
/**
* This type represents mappings for tokens affected by merge edit operations.
*/
interface TokenMergeMap {
interface TokenMergeMapping {
/**
* Entries here represent source-context tokens that are combined as part of
* the merge edit operation.
@ -54,7 +66,7 @@ interface TokenMergeMap {
* Entries will appear in the same ordering as the codepoints they represent
* appear in the underlying context.
*/
inputs: EditTokenMap[],
inputs: EditTokenMappingHalf[],
/**
* This entry represents the post-transition context token produced from the
@ -63,14 +75,14 @@ interface TokenMergeMap {
* Note that it is possible for extra codepoints to exist here that were not
* represented in the original source-context tokens.
*/
match: EditTokenMap
match: EditTokenMappingHalf
};
/**
* This type represents mappings for tokens affected by split edit operations.
*/
export interface TokenSplitMap {
export interface TokenSplitMapping {
/**
* This entry represents the source-context token that was split as part of
* the split edit operation.
@ -78,7 +90,7 @@ export interface TokenSplitMap {
* Note that it is possible for codepoints to exist here but not be
* represented in the post-transition context tokens resulting from the split.
*/
input: EditTokenMap,
input: EditTokenMappingHalf,
/**
* Entries here represent post-transition tokens that represent pieces of the
@ -87,7 +99,7 @@ export interface TokenSplitMap {
* Entries will appear in the same ordering as the codepoints they represent
* appear in the underlying context.
*/
matches: (EditTokenMap & { textOffset: number })[]
matches: SplitTokenMappingHalf[]
};
/**
@ -100,11 +112,11 @@ export interface TransitionEdgeAlignment {
/**
* Denotes any token merge edits needed after applying the Transform.
*/
merges: TokenMergeMap[];
merges: TokenMergeMapping[];
/**
* Denotes any token split edits needed after applying the Transform.
*/
splits: TokenSplitMap[];
splits: TokenSplitMapping[];
/**
* Denotes any further token edits needed that cannot be attributed to
* 'merge's, 'split's, or edits from the input `Transform`.
@ -431,8 +443,8 @@ export class ContextTokenization {
* @param transitionEdge Batched results from one or more
* `precomputeTokenizationAfterInput` calls on this instance, all with the
* same alignment values.
* @param transitionId The id of the Transform associated with the keystroke
* triggering the transition.
* @param transitionId The id of the context transition associated with the
* Transition
* @param bestProbFromSet The probability of the single most likely input
* transform in the overall transformDistribution associated with the
* keystroke triggering the transition. It need not be represented by the
@ -479,9 +491,6 @@ export class ContextTokenization {
tailTokenization.splice(tokenIndex, 1, affectedToken);
}
affectedToken.isPartial = true;
delete affectedToken.appliedTransitionId;
// If we are completely replacing a token via delete left, erase the deleteLeft;
// that part applied to a _previous_ token that no longer exists.
// We start at index 0 in the insert string for the "new" token.
@ -507,6 +516,11 @@ export class ContextTokenization {
affectedToken.isPartial
);
// Do not adjust the original token, as it may be used by other transitions.
// Only adjust the new, extended token.
affectedToken.isPartial = true;
delete affectedToken.appliedTransitionId;
const tokenize = determineModelTokenizer(lexicalModel);
affectedToken.isWhitespace = tokenize({left: affectedToken.exampleInput, startOfBuffer: false, endOfBuffer: false}).left[0]?.isWhitespace ?? false;
// Do not re-use the previous token; the mutation may have unexpected
@ -516,7 +530,21 @@ export class ContextTokenization {
affectedToken = null;
}
return new ContextTokenization(this.tokens.slice(0, sliceIndex).concat(tailTokenization));
// Backspace handling - emptying context via backspace or erasing _part_ of
// a whitespace token can erase the tokenization-final empty token usually
// used for word-initial suggestions.
//
// We re-add it here so that suggestions can be presented to the user as
// normal.
const tokenSequence = this.tokens.slice(0, sliceIndex).concat(tailTokenization);
if(tokenSequence.length == 0 || tokenSequence[tokenSequence.length - 1]?.isWhitespace) {
tokenSequence.push(new ContextToken(new LegacyQuotientRoot(lexicalModel)));
}
return new ContextTokenization(
tokenSequence,
null
);
}
}
@ -1030,13 +1058,13 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo
* generality, if two separate groups of tokens are merged, two groups will be
* defined - one for each token resulting from a merge.
*/
merges: TokenMergeMap[],
merges: TokenMergeMapping[],
/**
* Indicates groupings of directly related splits. Without loss of
* generality, if two separate tokens are split, two groups will be defined -
* one for each source token split.
*/
splits: TokenSplitMap[]
splits: TokenSplitMapping[]
} {
// We've found the root token to which changes may apply.
// We've found the last post-application token to which transform changes contributed.
@ -1074,8 +1102,8 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo
const mappedPath: EditTuple<EditOperation>[] = [];
let mergeOffset = 0;
let splitOffset = 0;
const merges: TokenMergeMap[] = [];
const splits: TokenSplitMap[] = [];
const merges: TokenMergeMapping[] = [];
const splits: TokenSplitMapping[] = [];
while(queueIndex < editPath.length) {
const edit = editPath[queueIndex];
const { input, match } = edit;
@ -1085,7 +1113,7 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo
let matchOffset: number = 0;
if(op == 'merge') {
const mergeTarget = resultTokenization[match];
const merge: TokenMergeMap = {
const merge: TokenMergeMapping = {
match: {
index: match,
text: mergeTarget
@ -1116,7 +1144,7 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo
merges.push(merge);
} else if(op == 'split') {
const splitTarget = preTokenization[input];
const split: TokenSplitMap = {
const split: TokenSplitMapping = {
input: {
index: input,
text: splitTarget

21
web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts
View file

@ -19,6 +19,15 @@ import Reversion = LexicalModelTypes.Reversion;
import Suggestion = LexicalModelTypes.Suggestion;
import Transform = LexicalModelTypes.Transform;
export interface TransitionReversionView extends Pick<ContextTransition, 'reversion'> {
/**
* Gets the context state resulting from the context transition event,
* including any generated suggestions and data regarding potential
* application thereof.
*/
final: Pick<ContextState, 'suggestions'>
}
/**
* Represents the transition between two context states as triggered
* by input keystrokes or applied suggestions.
@ -36,7 +45,7 @@ export class ContextTransition {
*/
inputDistribution?: Distribution<Transform>;
// The transform ID in play.
// The transition ID in play.
private _transitionId?: number;
/**
@ -177,7 +186,7 @@ export class ContextTransition {
// We won't try to partially revert a multi-word suggestion; reversions
// are only supported at the end of the last word of the main suggestion
// body and after any appended whitespace.
resultingTokenization.tail.appliedTransitionId = suggestion.transformId;
resultingTokenization.tail.appliedTransitionId = suggestion.transform.id;
const resultingState = new ContextState(
applyTransform(transformToApply, baseState.context),
@ -189,12 +198,12 @@ export class ContextTransition {
resultingState.appliedSuggestionId = suggestion.id;
resultingState.suggestions = this.final.suggestions;
// Use the transform's ID for the transition. Note that when applying the
// Use the transition ID tracked on the Transform. Note that when applying the
// `appendedTransform` component of a suggestion, this will differ from
// suggestion.transformId.
// suggestion.transitionId.
const resultingTransition = new ContextTransition(baseState, transformToApply.id);
resultingTransition.finalize(resultingState, inputDistribution);
resultingTransition.revertableTransitionId = suggestion.transformId;
resultingTransition.revertableTransitionId = suggestion.transform.id;
// .finalize unsets _.transitionId; re-assign it.
resultingTransition._transitionId = transformToApply.id;
@ -236,7 +245,7 @@ export class ContextTransition {
const baseTokenizationLength = results.transition.final.displayTokenization.tokens.length;
const appliedTokenization = appendingTransition.final.displayTokenization;
for(let i = baseTokenizationLength; i < appliedTokenization.tokens.length; i++) {
appliedTokenization.tokens[i].appliedTransitionId = suggestion.transformId;
appliedTokenization.tokens[i].appliedTransitionId = suggestion.transform.id;
}
return {

7
web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-subsets.ts
View file

@ -135,7 +135,8 @@ export function legacySubsetKeyer(tokenizationEdits: TokenizationTransitionEdits
// Now, based on the transform tokenization. We want to force uniqueness for
// all variations of result length on each tokenized transform resulting from
// the precomputation's represented keystroke.
for(const {0: relativeIndex} of tokenizedTransform.entries()) {
for(const {0: relativeIndex, 1: transform} of tokenizedTransform.entries()) {
const insertLen = KMWString.length(transform.insert);
if(relativeIndex > 0) {
// The true boundary lie before the insert if the value is non-zero;
// don't differentiate here!
@ -148,10 +149,10 @@ export function legacySubsetKeyer(tokenizationEdits: TokenizationTransitionEdits
//
// IMPORTANT: update unit tests manually if the BI marker here changes
// or the use of SENTINEL_CODE_UNIT as a key component separator changes.
components.push(`BI@${relativeIndex}`);
components.push(`BI@${relativeIndex}-${boundaryTextLen + insertLen}`);
boundaryTextLen = 0;
} else {
components.push(`I@${relativeIndex}`);
components.push(`I@${relativeIndex}-${boundaryTextLen + insertLen}`);
}
}

63
web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts
View file

@ -1,7 +1,19 @@
import * as models from '@keymanapp/models-templates';
import { LexicalModelTypes } from '@keymanapp/common-types';
import { applySuggestionCasing, composeIntermediatePredictions, correctAndEnumerate, createDefaultKeep, dedupeSuggestions, finalizeSuggestions, predictionAutoSelect, processSimilarity, toAnnotatedSuggestion, tupleDisplayOrderSort } from './predict-helpers.js';
import {
applySuggestionCasing,
composeIntermediatePredictions,
correctAndEnumerate,
createDefaultKeep,
dedupeSuggestions,
finalizeSuggestions,
predictionAutoSelect,
prependReversion,
processSimilarity,
toAnnotatedSuggestion,
tupleDisplayOrderSort
} from './predict-helpers.js';
import { determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js';
import { ContextTracker } from './correction/context-tracker.js';
@ -79,12 +91,12 @@ export class ModelCompositor {
this.configuration = config;
}
initContextTracker(context: Context, transformId: number) {
initContextTracker(context: Context, transitionId: number) {
if(this.contextTracker || !this.lexicalModel.traverseFromRoot) {
return;
}
this._contextTracker = new ContextTracker(this.lexicalModel, context, transformId, this.configuration);
this._contextTracker = new ContextTracker(this.lexicalModel, context, transitionId, this.configuration);
}
async predict(transformDistribution: Transform | Distribution<Transform>, context: Context): Promise<Outcome<Suggestion|Keep>[]> {
@ -120,8 +132,8 @@ export class ModelCompositor {
// Only allow new-word suggestions if space was the most likely keypress.
// const allowSpace = TransformUtils.isWhitespace(inputTransform);
const inputTransform = transformDistribution[0].sample;
const transformId = inputTransform.id;
this.initContextTracker(context, transformId);
const transitionId = inputTransform.id;
this.initContextTracker(context, transitionId);
// Section 1: determine 'prediction roots' - enumerate corrections from most to least likely,
// searching for results that yield viable predictions from the model.
@ -190,18 +202,8 @@ export class ModelCompositor {
this.SUGGESTION_ID_SEED++;
});
if(revertableTransitionId) {
const reversion = this.contextTracker.peek(revertableTransitionId)?.reversion;
if(reversion) {
if(suggestions[0]?.tag == 'keep') {
const keep = suggestions.shift();
suggestions.unshift(reversion);
suggestions.unshift(keep);
} else {
suggestions.unshift(reversion)
}
}
}
const transitionToRevert = this.contextTracker?.peek(revertableTransitionId);
prependReversion(suggestions, transitionToRevert);
if(suggestions.filter((s) => s.tag == 'keep').length > 1) {
throw new Error(`Unexpected state: multiple keep suggestions exist: ${JSON.stringify(suggestions.filter((s) => s.tag == 'keep'))}`);
@ -229,7 +231,9 @@ export class ModelCompositor {
// Step 1: re-use the original input Transform as the reversion's Transform.
// The Web engine will restore the original state of the context before accepting
// and before reverting; all we need to do is put the original keystroke back in place.
let reversionTransform: Transform = originalInput ?? { insert: '', deleteLeft: 0 };
let reversionTransform: Transform = originalInput
? { ...originalInput }
: { insert: '', deleteLeft: 0, id: suggestion.transform.id };
// Step 2: building the proper 'displayAs' string for the Reversion
const postContext = originalInput ? models.applyTransform(originalInput, context) : context;
@ -253,9 +257,6 @@ export class ModelCompositor {
// Since we're outside of the standard `predict` control path, we'll need to
// set the Reversion's ID directly.
let reversion = toAnnotatedSuggestion(this.lexicalModel, firstConversion, 'revert');
if(suggestion.transformId != null) {
reversion.transformId = -suggestion.transformId;
}
if(suggestion.id != null) {
// Since a reversion inverts its source suggestion, we set its ID to be the
// additive inverse of the source suggestion's ID. Makes easy mapping /
@ -277,11 +278,11 @@ export class ModelCompositor {
}
} else {
let originalTransition = this.contextTracker.latest;
if(originalTransition.transitionId != suggestion.transformId) {
originalTransition = this.contextTracker.findAndRevert(suggestion.transformId);
if(originalTransition.transitionId != suggestion.transform.id) {
originalTransition = this.contextTracker.findAndRevert(suggestion.transform.id);
}
if(!originalTransition) {
this.contextTracker.reset(context, suggestion.transformId);
this.contextTracker.reset(context, suggestion.transform.id);
originalTransition = this.contextTracker.latest;
}
@ -319,11 +320,8 @@ export class ModelCompositor {
let compositor = this;
let suggestions: Promise<Suggestion[]>;
let fallbackSuggestions = async function() {
const suggestions = await compositor.predict({insert: '', deleteLeft: 0}, context);
const suggestions = await compositor.predict({insert: '', deleteLeft: 0, id: reversion.transform.id}, context);
suggestions.forEach(function(suggestion) {
// A reversion's transform ID is the additive inverse of its original suggestion;
// we revert to the state of said original suggestion.
suggestion.transformId = -reversion.transformId;
// Prevent auto-selection of any suggestion immediately after a reversion.
// It's fine after at least one keystroke, but not before.
suggestion.autoAccept = false;
@ -337,18 +335,19 @@ export class ModelCompositor {
}
// When the context is tracked, we prefer the tracked information.
// Note that the base reversion's .transformId will predate the appendedTransform id
// Note that the base reversion's .transform.id will predate the appendedTransform id
// used to add whitespace (if one existed), so reverting to the base ID's associated
// context also reverts the appendedTransform.
let originalTransition = this.contextTracker.findAndRevert(-reversion.transformId);
const baseTransitionId = reversion.transform.id;
let originalTransition = this.contextTracker.findAndRevert(baseTransitionId);
if(appendedOnly) {
this.contextTracker.latest = originalTransition;
return Promise.resolve([]);
}
if(!originalTransition) {
this.contextTracker.reset(context, -reversion.transformId);
if(!originalTransition || originalTransition.transitionId != baseTransitionId) {
this.contextTracker.reset(context, baseTransitionId);
originalTransition = this.contextTracker.latest;
suggestions = fallbackSuggestions();

3
web/src/engine/predictive-text/worker-thread/src/main/models/dummy-model.ts
View file

@ -98,12 +98,11 @@ export class DummyModel implements LexicalModel {
predict(transform: Transform, context: Context, injectedSuggestions?: Outcome<Suggestion>[]): Distribution<Suggestion> {
let makeUniformDistribution = function(suggestions: Outcome<Suggestion>[]): Distribution<Suggestion> {
let distribution: Distribution<Suggestion> = [];
const transitionId = transform.id;
for(let s of suggestions) {
const transitionId = transform.id;
if(transitionId !== undefined) {
// Set the transform ID to match the incoming transform ID if one exists.
s.transformId = transitionId;
if(s.transform) {
s.transform.id = transitionId;
}

60
web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts
View file

@ -9,7 +9,7 @@ import { ContextTokenLike } from './correction/context-token.js';
import { ContextTokenization } from './correction/context-tokenization.js';
import { ContextTracker } from './correction/context-tracker.js';
import { ContextState, determineContextSlideTransform } from './correction/context-state.js';
import { ContextTransition } from './correction/context-transition.js';
import { ContextTransition, TransitionReversionView } from './correction/context-transition.js';
import { ExecutionTimer } from './correction/execution-timer.js';
import { ModelCompositor } from './model-compositor.js';
import { EDIT_DISTANCE_COST_SCALE, getBestTokenMatches } from './correction/distance-modeler.js';
@ -690,11 +690,6 @@ export async function correctAndEnumerate(
// Corrections obtained: now to predict from them!
const tokenization = tokenizations.find(t => t.spaceId == match.spaceId);
// If our 'match' results in fully deleting the new token, reject it and try again.
if(match.matchSequence.length == 0 && match.inputSequence.length != 0) {
continue;
}
// If our 'match' fully replaces the token, reject it and try again.
if(match.matchSequence.length != 0 && match.matchSequence.length == match.knownCost) {
continue;
@ -822,7 +817,6 @@ export function predictFromCorrectionSequence(
entry.sample.transform.deleteLeft = correctionTransform.deleteLeft;
if(transitionId !== undefined) {
entry.sample.transformId = transitionId;
entry.sample.transform.id = transitionId;
}
});
@ -946,7 +940,7 @@ export function composeIntermediatePredictions(predictions: TokenizedIntermediat
insert: '',
deleteLeft: 0
}
const transformId = predictionData.components[0].prediction.transformId;
const transformId = predictionData.components[0].prediction.transform.id;
if(transformId !== undefined) {
reduceBaseTransform.id = transformId;
}
@ -1140,7 +1134,6 @@ export function createDefaultKeep(
let keepOption = toAnnotatedSuggestion(lexicalModel, keepSuggestion, 'keep');
if(inputTransform.id !== undefined) {
keepOption.transformId = inputTransform.id;
keepOption.transform.id = inputTransform.id;
}
keepOption.matchesModel = false;
@ -1199,9 +1192,11 @@ export function predictionAutoSelect(suggestionDistribution: CompositedIntermedi
const keepOption = suggestionDistribution[0].components.prediction as Outcome<Keep>;
if(keepOption.tag == 'keep' && keepOption.matchesModel) {
// Auto-select it for auto-acceptance; we don't correct away from perfectly-valid
// lexical entries, even if they are comparatively low-frequency.
keepOption.autoAccept = true;
// Do not auto-select 'keep' suggestions'; there's no need to apply them.
//
// Do, however, block auto-selection of any other suggestions if we would
// have auto-selected the 'keep'; even if it is comparatively unlikely /
// low-frequency, we 'keep' the current context.
return;
} else if(suggestionDistribution.length == 1) {
return;
@ -1296,12 +1291,6 @@ export function finalizeSuggestions(
const suggestions = deduplicatedSuggestionTuples.map((tuple) => {
const prediction = tuple.components.prediction;
// Is sometimes not set during unit tests.
if(prediction.transformId !== undefined) {
prediction.transform.id = prediction.transformId;
}
const probs = tuple.metadata.probabilities;
if(!verbose) {
@ -1404,9 +1393,40 @@ export function toAnnotatedSuggestion(
result.appendedTransform = suggestion.appendedTransform;
}
if(suggestion.transformId !== undefined) {
result.transformId = suggestion.transformId;
if(suggestion.transform.id !== undefined) {
result.transform.id = suggestion.transform.id;
}
return result;
}
/**
* For applicable scenarios, this mutates the passed-in suggestion array by
* prepending a predictive-text reversion that restores the context to a prior
* state. Otherwise, it leaves the suggestion array unaltered.
* @param suggestions
* @param transitionToRevert
* @returns
*/
export function prependReversion(suggestions: Suggestion[], transitionToRevert: TransitionReversionView) {
if(transitionToRevert) {
const reversion = transitionToRevert.reversion;
if(reversion) {
if(suggestions[0]?.tag == 'keep') {
const appliedId = -reversion.id;
const appliedSuggestion = transitionToRevert.final.suggestions.find((s) => s.id == appliedId);
// If the selected suggestion was itself a `keep`, we don't need a
// reversion. They'd do the same thing.
if(appliedSuggestion.tag != 'keep') {
const keep = suggestions.shift();
suggestions.unshift(reversion);
suggestions.unshift(keep);
}
} else {
suggestions.unshift(reversion);
}
}
}
return suggestions;
}

2
web/src/engine/predictive-text/worker-thread/src/main/test-index.ts
View file

@ -3,7 +3,7 @@ export * from './correction/context-state.js';
export * from './correction/context-token.js';
export * from './correction/context-tokenization.js';
export { ContextTracker } from './correction/context-tracker.js';
export { ContextTransition } from './correction/context-transition.js';
export * from './correction/context-transition.js';
export * from './correction/correction-searchable.js';
export * from './correction/correction-result-mapping.js';
export * from './correction/distance-modeler.js';

4
web/src/engine/src/interfaces/prediction/predictionContext.ts
View file

@ -266,14 +266,14 @@ export class PredictionContext extends EventEmitter<PredictionContextEventMap> {
this._revertSuggestion = s as Reversion;
}
if (this.langProcessor.mayAutoCorrect && s.autoAccept && !this.selected) {
if (this.langProcessor.mayAutoCorrect && s.autoAccept && !this.selected && s.tag != 'keep') {
this.selected = s;
}
}
// Verify that the transition IDs are still valid and remove special entries.
this._currentSuggestions = suggestions.filter(s => {
return this.langProcessor.hasState(Math.abs(s.transformId)) &&
return this.langProcessor.hasState(Math.abs(s.transform.id)) &&
s != this._keepSuggestion &&
s != this._revertSuggestion
});

2
web/src/engine/src/main/headless/inputProcessor.ts
View file

@ -386,7 +386,7 @@ export class InputProcessor {
// If so, since it behaves the same in either case, and it's a known word-breaking mark,
// let's apply the selected suggestion automatically.
if(postApplyTransform.insert == ruleTransform.insert && transformMatchesPattern(postApplyTransform, breakingMarks)) {
const baseTransition = this.contextCache.get(predictionContext.selected.transformId);
const baseTransition = this.contextCache.get(predictionContext.selected.transform.id);
// Somehow, the base state is out of context - abort!
if(!baseTransition) {

24
web/src/engine/src/main/headless/languageProcessor.ts
View file

@ -166,6 +166,12 @@ export class LanguageProcessor extends EventEmitter<LanguageProcessorEventMap> {
return !!this.recentTranscriptions.get(transitionId);
}
/**
* When called, this requests the worker to generate predictions for the transcribed
* context transition.
* @param transcription
* @param layerId
*/
public predict(transcription: Transcription, layerId: string): Promise<Suggestion[]> {
if(!this.isActive) {
return null;
@ -218,13 +224,13 @@ export class LanguageProcessor extends EventEmitter<LanguageProcessorEventMap> {
// Find the state of the context at the time the suggestion was generated.
// This may refer to the context before an input keystroke or before application
// of a predictive suggestion.
const original = this.getPredictionState(suggestion.transformId);
const original = this.getPredictionState(suggestion.transform.id);
if(!original) {
console.warn("Could not apply the Suggestion!");
return null;
}
this.recentTranscriptions.rewindTo(suggestion.transformId);
this.recentTranscriptions.rewindTo(suggestion.transform.id);
// Apply the Suggestion!
@ -306,10 +312,7 @@ export class LanguageProcessor extends EventEmitter<LanguageProcessorEventMap> {
// Find the state of the context at the time the suggestion was generated.
// This may refer to the context before an input keystroke or before application
// of a predictive suggestion.
//
// Reversions use the additive inverse of the id token of the Transcription being
// reverted to.
const reversionId = appendedOnly ? reversion.appendedTransform.id : -reversion.transformId;
const reversionId = appendedOnly ? reversion.appendedTransform.id : reversion.transform.id;
const original = this.getPredictionState(reversionId);
if(!original) {
console.warn("Could not apply the Suggestion!");
@ -373,6 +376,9 @@ export class LanguageProcessor extends EventEmitter<LanguageProcessorEventMap> {
this.lmEngine.resetContext(context, transcription.token);
}
// The "may correct" setting is enforced here, in the main Web engine - not
// in the worker. As the desired setting may vary language-to-language,
// we'd rather do this than reconfigure the worker constantly.
let alternates = transcription.alternates;
if(!this.mayCorrect || !alternates || alternates.length == 0) {
alternates = [{
@ -386,6 +392,12 @@ export class LanguageProcessor extends EventEmitter<LanguageProcessorEventMap> {
return promise.then((suggestions: Suggestion[]) => {
if(promise == this.currentPromise) {
if(!this.mayAutoCorrect) {
// We disable the auto-accept flag here, rather than in the worker -
// that way, we don't need to reconfigure the worker when settings
// change.
suggestions.forEach((s) => delete s.autoAccept);
}
const result = new ReadySuggestions(suggestions, transform.id);
this.emit("suggestionsready", result);
this.currentPromise = null;

2
web/src/engine/src/main/headless/transcriptionCache.ts
View file

@ -1,7 +1,7 @@
import { Transcription } from "keyman/engine/keyboard";
import { RewindableCache } from "keyman/common/web-utils";
const TRANSCRIPTION_BUFFER_SIZE = 10;
const TRANSCRIPTION_BUFFER_SIZE = 20;
export class TranscriptionCache extends RewindableCache<Transcription> {
constructor() {

3
web/src/test/auto/headless/engine/main/inputProcessor.tests.ts
View file

@ -283,9 +283,8 @@ describe('InputProcessor', function() {
[],
[
{
transform: { insert: 'testing', deleteLeft: 4 },
transform: { insert: 'testing', deleteLeft: 4, id: 0 },
appendedTransform: { insert: ' ', deleteLeft: 0 },
transformId: 0, // will be overwritten by the DummyModel to match the transition ID.
id: 1,
displayAs: 'testing'
}

6
web/src/test/auto/headless/engine/predictive-text/templates/common.tests.ts
View file

@ -191,7 +191,6 @@ describe('Common utility functions', function() {
deleteLeft: 0,
id: 0
},
transformId: 0,
displayAs: 'hello'
};
@ -205,7 +204,6 @@ describe('Common utility functions', function() {
deleteLeft: 0,
id: 0
},
transformId: 0,
displayAs: 'hello',
p: 0
};
@ -220,7 +218,6 @@ describe('Common utility functions', function() {
deleteLeft: 0,
id: 0
},
transformId: 0,
displayAs: 'hello',
p: 0.5
};
@ -228,14 +225,13 @@ describe('Common utility functions', function() {
assert.deepEqual(models.transformToSuggestion(suggestion.transform, 0.5), suggestion);
});
it('properly handles the transformId', function() {
it('properly handles the transition ID', function() {
let suggestion = {
transform: {
insert: 'hello',
deleteLeft: 0,
id: 3
},
transformId: 3, // Ensures there isn't a separate ID seed in use.
displayAs: 'hello'
};

286
web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts
View file

@ -28,7 +28,8 @@ import {
models,
SearchQuotientSpur,
traceInsertEdits,
LegacyQuotientSpur
LegacyQuotientSpur,
TransitionEdge
} from '@keymanapp/lm-worker/test-index';
import Transform = LexicalModelTypes.Transform;
@ -45,8 +46,8 @@ function toToken(text: string) {
}
let TOKEN_TRANSFORM_SEED = 0;
function toTransformToken(text: string, transformId?: number) {
let idSeed = transformId === undefined ? TOKEN_TRANSFORM_SEED++ : transformId;
function toTransitionToken(text: string, transitionId?: number) {
let idSeed = transitionId === undefined ? TOKEN_TRANSFORM_SEED++ : transitionId;
let isWhitespace = text == ' ';
let token = ContextToken.fromRawText(plainModel, '');
const textAsTransform = { insert: text, deleteLeft: 0, id: idSeed };
@ -105,7 +106,7 @@ describe('ContextTokenization', function() {
it("constructs from a token array + alignment data", () => {
const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
const tokens = rawTextTokens.map((text => toTransformToken(text)));
const tokens = rawTextTokens.map((text => toTransitionToken(text)));
let tokenization = new ContextTokenization(tokens);
@ -117,8 +118,28 @@ describe('ContextTokenization', function() {
it('clones', () => {
const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
const tokens = rawTextTokens.map((text => toTransformToken(text)));
let baseTokenization = new ContextTokenization(tokens);
const tokens = rawTextTokens.map((text => toTransitionToken(text)));
const emptyTransform = { insert: '', deleteLeft: 0, deleteRight: 0 };
// We _could_ flesh this out a bit more... but it's not really needed for this test.
const edgeWindow = buildEdgeWindow(tokens, emptyTransform, false, testEdgeWindowSpec);
let transitionEdits: TransitionEdge = {
alignment: {
merges: [],
splits: [],
unmappedEdits: [],
edgeWindow: {...edgeWindow, retokenization: rawTextTokens.slice(edgeWindow.sliceIndex)},
removedTokenCount: 0
},
inputs: [{sample: (() => {
const map = new Map<number, Transform>();
map.set(0, emptyTransform);
return map;
})(), p: 1}],
inputSubsetId: generateSubsetId()
};
let baseTokenization = new ContextTokenization(tokens, transitionEdits);
let cloned = new ContextTokenization(baseTokenization);
assert.sameOrderedMembers(
@ -353,6 +374,43 @@ describe('ContextTokenization', function() {
);
});
it('handles simple case - deletion of final context content via backspace', () => {
const baseTokens = ['a'];
const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t)));
const targetTokens = [''].map((t) => ({text: t, isWhitespace: t == ' '}));
const inputTransform = { insert: '', deleteLeft: 1, deleteRight: 0, id: 42 };
const inputTransformMap: Map<number, Transform> = new Map();
inputTransformMap.set(0, { insert: '', deleteLeft: 1, id: 42 });
const edgeWindow = buildEdgeWindow(baseTokenization.tokens, inputTransform, false, testEdgeWindowSpec);
const tokenization = baseTokenization.evaluateTransition({
alignment: {
merges: [],
splits: [],
unmappedEdits: [],
edgeWindow: {
...edgeWindow,
// The range within the window constructed by the prior call for its parameterization.
// Any adjustments on the boundary token itself are included here.
retokenization: [...targetTokens.slice(edgeWindow.sliceIndex).map(t => t.text)]
},
removedTokenCount: 1
},
inputs: [{ sample: inputTransformMap, p: 1 }],
inputSubsetId: generateSubsetId()
},
inputTransform.id,
1
);
assert.isOk(tokenization);
assert.equal(tokenization.tokens.length, targetTokens.length);
assert.deepEqual(tokenization.tokens.map((t) => ({text: t.exampleInput, isWhitespace: t.isWhitespace})),
targetTokens
);
});
it('handles simple case - new character added to last token', () => {
const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'da'];
const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t)));
@ -779,6 +837,198 @@ describe('ContextTokenization', function() {
assert.equal(preTail.exampleInput, '\'');
assert.equal(tail.exampleInput, '.');
});
describe('properly handles previously-applied transition IDs', () => {
it('does not preserve applied transition IDs on edited tokens', () => {
const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day', ' ', 'can'];
const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t)));
const REVERTABLE_TRANSITION_ID = 31415;
baseTokenization.tail.appliedTransitionId = REVERTABLE_TRANSITION_ID;
const NEW_TRANSITION_ID = REVERTABLE_TRANSITION_ID + 1;
const dist = [
{
sample: { insert: 't', deleteLeft: 0, deleteRight: 0, id: NEW_TRANSITION_ID },
p: 1
}
];
const resultTokenization = baseTokenization.evaluateTransition(
{
alignment: {
merges: [],
splits: [],
unmappedEdits: [],
edgeWindow: {
...buildEdgeWindow(baseTokenization.tokens, dist[0].sample, false),
retokenization: baseTokenization.tokens.map((t) => t.exampleInput),
retokenizationText: baseTokenization.tokens.map((t) => t.exampleInput).reduce((accum, curr) => accum + curr, '')
},
removedTokenCount: 0
},
inputs: (() => {
const map: Map<number, Transform> = new Map();
map.set(0, dist[0].sample);
return [
{sample: map, p: 1}
];
})(),
inputSubsetId: 0
},
NEW_TRANSITION_ID,
1
)
resultTokenization.tokens.forEach((t) => {
assert.isUndefined(t.appliedTransitionId);
});
});
it('preserves applied transition IDs on applicable tokens', () => {
const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day', ' ', 'can'];
const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t)));
const REVERTABLE_TRANSITION_ID = 31415;
baseTokenization.tail.appliedTransitionId = REVERTABLE_TRANSITION_ID;
const NEW_TRANSITION_ID = REVERTABLE_TRANSITION_ID + 1;
const dist = [
{
sample: { insert: ' ', deleteLeft: 0, deleteRight: 0, id: NEW_TRANSITION_ID },
p: 1
}
];
const resultTokenization = baseTokenization.evaluateTransition(
{
alignment: {
merges: [],
splits: [],
unmappedEdits: [],
edgeWindow: {
...buildEdgeWindow(baseTokenization.tokens, dist[0].sample, false),
retokenization: baseTokenization.tokens.map((t) => t.exampleInput),
retokenizationText: baseTokenization.tokens.map((t) => t.exampleInput).reduce((accum, curr) => accum + curr, '')
},
removedTokenCount: 0
},
inputs: (() => {
const map: Map<number, Transform> = new Map();
map.set(1, dist[0].sample);
return [
{sample: map, p: 1}
];
})(),
inputSubsetId: 0
},
NEW_TRANSITION_ID,
1
)
const resultTokenLength = resultTokenization.tokens.length;
resultTokenization.tokens.forEach((t, i) => {
// The space will add TWO tokens.
if(i == resultTokenLength - 3) {
assert.equal(t.appliedTransitionId, REVERTABLE_TRANSITION_ID);
} else {
assert.isUndefined(t.appliedTransitionId);
}
});
});
// Performs the two above in sequence in a manner that could cause cross-effects
// if implemented incorrectly.
it('does not conflate effects between different tokenization transitions', () => {
const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day', ' ', 'can'];
const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t)));
const REVERTABLE_TRANSITION_ID = 31415;
baseTokenization.tail.appliedTransitionId = REVERTABLE_TRANSITION_ID;
const NEW_TRANSITION_ID = REVERTABLE_TRANSITION_ID + 1;
const dist = [
{
sample: { insert: ' ', deleteLeft: 0, deleteRight: 0, id: NEW_TRANSITION_ID },
p: .8
}, {
sample: { insert: ' ', deleteLeft: 0, deleteRight: 0, id: NEW_TRANSITION_ID },
p: .2
}
];
const baseTransitionEdge: TransitionEdge = {
alignment: {
merges: [],
splits: [],
unmappedEdits: [],
edgeWindow: {
...buildEdgeWindow(baseTokenization.tokens, dist[0].sample, false),
retokenization: baseTokenization.tokens.map((t) => t.exampleInput),
retokenizationText: baseTokenization.tokens.map((t) => t.exampleInput).reduce((accum, curr) => accum + curr, '')
},
removedTokenCount: 0
},
inputs: (() => {
const map: Map<number, Transform> = new Map();
map.set(1, dist[0].sample);
return [
{sample: map, p: dist[0].p}
];
})(),
inputSubsetId: 0
};
// We don't care about the results here. What we care about is that
// this call doesn't remove the appliedTransitionId from the source
// token, preventing it from being marked on later tokenization
// transitions.
baseTokenization.evaluateTransition(
{
alignment: {
...baseTransitionEdge.alignment,
edgeWindow: {
...baseTransitionEdge.alignment.edgeWindow,
...buildEdgeWindow(baseTokenization.tokens, dist[1].sample, false)
}
},
inputs: (() => {
const map: Map<number, Transform> = new Map();
map.set(0, dist[1].sample);
return [
{sample: map, p: dist[1].p}
];
})(),
inputSubsetId: 0
},
NEW_TRANSITION_ID,
dist[1].p
)
const resultTokenization = baseTokenization.evaluateTransition(
baseTransitionEdge,
NEW_TRANSITION_ID,
dist[0].p
);
const resultTokenLength = resultTokenization.tokens.length;
resultTokenization.tokens.forEach((t, i) => {
// The space will add TWO tokens.
if(i == resultTokenLength - 3) {
assert.equal(t.appliedTransitionId, REVERTABLE_TRANSITION_ID);
} else {
assert.isUndefined(t.appliedTransitionId);
}
});
});
});
});
describe('buildEdgeWindow', () => {
@ -786,7 +1036,7 @@ describe('ContextTokenization', function() {
it('handles empty contexts', () => {
const baseTokens = [''];
const idSeed = TOKEN_TRANSFORM_SEED;
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t)));
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t)));
const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 0 }, true, testEdgeWindowSpec);
assert.deepEqual(results, {
@ -806,7 +1056,7 @@ describe('ContextTokenization', function() {
it('handles empty contexts and invalid Transforms', () => {
const baseTokens = [''];
const idSeed = TOKEN_TRANSFORM_SEED;
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t)));
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t)));
const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 2 }, true, testEdgeWindowSpec);
assert.deepEqual(results, {
@ -826,7 +1076,7 @@ describe('ContextTokenization', function() {
it('builds edge windows for the start of context with no edits', () => {
const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
const idSeed = TOKEN_TRANSFORM_SEED;
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t)));
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t)));
const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 0 }, true, testEdgeWindowSpec);
assert.deepEqual(results, {
@ -866,7 +1116,7 @@ describe('ContextTokenization', function() {
it('builds edge windows for the start of context with deletion edits (1)', () => {
const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
const idSeed = TOKEN_TRANSFORM_SEED;
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t)));
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t)));
const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 2 }, true, testEdgeWindowSpec);
assert.deepEqual(results, {
@ -906,7 +1156,7 @@ describe('ContextTokenization', function() {
it('builds edge windows for the start of context with deletion edits (2)', () => {
const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
const idSeed = TOKEN_TRANSFORM_SEED;
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t)));
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t)));
const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 4 }, true, testEdgeWindowSpec);
assert.deepEqual(results, {
@ -926,7 +1176,7 @@ describe('ContextTokenization', function() {
it('builds edge windows for the end of context with no edits', () => {
const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
const idSeed = TOKEN_TRANSFORM_SEED;
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t)));
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t)));
baseTokenization.tail.isPartial = true;
const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 0 }, false, testEdgeWindowSpec);
@ -947,7 +1197,7 @@ describe('ContextTokenization', function() {
it('builds edge windows for the end of context with no edits, trailing whitespace', () => {
const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day', ' ', ''];
const idSeed = TOKEN_TRANSFORM_SEED;
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t)));
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t)));
const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 0 }, false, testEdgeWindowSpec);
assert.deepEqual(results, {
@ -1734,7 +1984,7 @@ describe('ContextTokenization', function() {
it('returns the standard edge window for empty transform inputs', () => {
const baseTokens = ['quick', ' ', 'brown', ' ', 'fox'];
const idSeed = TOKEN_TRANSFORM_SEED;
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t)));
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t)));
const editTransform = {
insert: '',
@ -1768,7 +2018,7 @@ describe('ContextTokenization', function() {
it('returns the standard edge window for empty transforms with context-final whitespace', () => {
const baseTokens = ['quick', ' ', 'brown', ' ', 'fox', ' '];
const idSeed = TOKEN_TRANSFORM_SEED;
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t)));
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t)));
const editTransform = {
insert: '',
@ -1803,7 +2053,7 @@ describe('ContextTokenization', function() {
it('returns the standard edge window for pure transform w insert inputs', () => {
const baseTokens = ['quick', ' ', 'brown', ' ', 'fox'];
const idSeed = TOKEN_TRANSFORM_SEED;
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t)));
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t)));
const editTransform = {
insert: ' jumped',
@ -1837,7 +2087,7 @@ describe('ContextTokenization', function() {
it('returns the proper edge window for transforms w deleteLeft inputs (1)', () => {
const baseTokens = ['quick', ' ', 'brown', ' ', 'fox'];
const idSeed = TOKEN_TRANSFORM_SEED;
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t)));
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t)));
const editTransform = {
insert: 'rog',
@ -1872,7 +2122,7 @@ describe('ContextTokenization', function() {
it('returns the proper edge window for transforms w deleteLeft inputs (2)', () => {
const baseTokens = ['quick', ' ', 'brown', ' ', 'fox'];
const idSeed = TOKEN_TRANSFORM_SEED;
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t)));
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t)));
const editTransform = {
insert: 'fox and brown fox', // => quick fox and brown fox

2
web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts
View file

@ -34,7 +34,7 @@ describe('ContextTracker', function() {
deleteLeft: 0,
id: 15
},
transformId: 2,
transitionId: 2,
id: 1,
displayAs: 'world'
};

32
web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-transition.tests.ts
View file

@ -118,9 +118,8 @@ describe('ContextTransition', () => {
appendedTransform: {
insert: ' ',
deleteLeft: 0,
id: 2
id: 3
},
transformId: 2,
id: 10,
displayAs: 'world'
}, {
@ -132,9 +131,8 @@ describe('ContextTransition', () => {
appendedTransform: {
insert: ' ',
deleteLeft: 0,
id: 2
id: 3
},
transformId: 2,
id: 11,
displayAs: 'won'
}];
@ -155,7 +153,7 @@ describe('ContextTransition', () => {
// 3 long, only last token was edited.
appliedTransition.base.final.displayTokenization.tokens.forEach((token, index) => {
if(index >= 2) {
assert.equal(token.appliedTransitionId, suggestions[0].transformId);
assert.equal(token.appliedTransitionId, suggestions[0].transform.id);
} else {
assert.isUndefined(token.appliedTransitionId);
}
@ -163,7 +161,7 @@ describe('ContextTransition', () => {
appliedTransition.appended.final.displayTokenization.tokens.forEach((token, index) => {
if(index >= 2) {
assert.equal(token.appliedTransitionId, suggestions[0].transformId);
assert.equal(token.appliedTransitionId, suggestions[0].transform.id);
} else {
assert.isUndefined(token.appliedTransitionId);
}
@ -202,9 +200,8 @@ describe('ContextTransition', () => {
appendedTransform: {
insert: ' ',
deleteLeft: 0,
id: 2
id: 3
},
transformId: 2,
id: 10,
displayAs: 'the'
}, {
@ -216,9 +213,8 @@ describe('ContextTransition', () => {
appendedTransform: {
insert: ' ',
deleteLeft: 0,
id: 2
id: 3
},
transformId: 2,
id: 11,
displayAs: 'and'
}];
@ -239,7 +235,7 @@ describe('ContextTransition', () => {
// 3 long, only last token was edited.
appliedTransition.base.final.displayTokenization.tokens.forEach((token, index) => {
if(index >= 4) {
assert.equal(token.appliedTransitionId, suggestions[0].transformId);
assert.equal(token.appliedTransitionId, suggestions[0].transform.id);
} else {
assert.isUndefined(token.appliedTransitionId);
}
@ -247,7 +243,7 @@ describe('ContextTransition', () => {
appliedTransition.appended.final.displayTokenization.tokens.forEach((token, index) => {
if(index >= 4) {
assert.equal(token.appliedTransitionId, suggestions[0].transformId);
assert.equal(token.appliedTransitionId, suggestions[0].transform.id);
} else {
assert.isUndefined(token.appliedTransitionId);
}
@ -287,9 +283,8 @@ describe('ContextTransition', () => {
appendedTransform: {
insert: ' ',
deleteLeft: 0,
id: 2
id: 3
},
transformId: 2,
id: 10,
displayAs: 'world'
}, {
@ -301,9 +296,8 @@ describe('ContextTransition', () => {
appendedTransform: {
insert: ' ',
deleteLeft: 0,
id: 2
id: 3
},
transformId: 2,
id: 11,
displayAs: 'won'
}];
@ -337,9 +331,8 @@ describe('ContextTransition', () => {
appendedTransform: {
insert: ' ',
deleteLeft: 0,
id: 2
id: 3
},
transformId: 2,
id: 10,
displayAs: 'world'
}, {
@ -351,9 +344,8 @@ describe('ContextTransition', () => {
appendedTransform: {
insert: ' ',
deleteLeft: 0,
id: 2
id: 3
},
transformId: 2,
id: 11,
displayAs: 'won'
}];

51
web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts
View file

@ -21,6 +21,7 @@ import {
ContextToken,
ContextTokenization,
LegacyQuotientSpur,
legacySubsetKeyer,
models,
precomputationSubsetKeyer,
TokenizationTransitionEdits,
@ -31,7 +32,7 @@ import Distribution = LexicalModelTypes.Distribution;
import Transform = LexicalModelTypes.Transform;
import TrieModel = models.TrieModel;
var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'),
const plainModel = new TrieModel(jsonFixture('models/tries/english-1000'),
{wordBreaker: defaultBreaker});
function toToken(text: string) {
@ -41,6 +42,54 @@ function toToken(text: string) {
return token;
}
describe('legacySubsetKeyer', () => {
it('does not map backspace inputs to same result as standard key inputs', () => {
const appleToken = ContextToken.fromRawText(plainModel, 'apple', true);
const bksp = { insert: '', deleteLeft: 1, deleteRight: 0, id: 3 };
const bkspKey = legacySubsetKeyer({
alignment: {
merges: [],
splits: [],
unmappedEdits: [],
edgeWindow: {
...buildEdgeWindow([appleToken], bksp, false),
retokenization: ['appl'],
retokenizationText: 'appl'
},
removedTokenCount: 0
},
tokenizedTransform: (() => {
const map: Map<number, Transform> = new Map();
map.set(0, bksp);
return map;
})()
});
const input = { insert: 's', deleteLeft: 0, deleteRight: 0, id: bksp.id };
const inputKey = legacySubsetKeyer({
alignment: {
merges: [],
splits: [],
unmappedEdits: [],
edgeWindow: {
...buildEdgeWindow([appleToken], input, false),
retokenization: ['apples'],
retokenizationText: 'apples'
},
removedTokenCount: 0
},
tokenizedTransform: (() => {
const map: Map<number, Transform> = new Map();
map.set(0, input);
return map;
})()
});
assert.notEqual(bkspKey, inputKey);
});
});
describe('precomputationSubsetKeyer', function() {
it("safely generates keys for empty transition + empty contexts", () => {
const rawTextTokens = [''];

8
web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts
View file

@ -16,7 +16,7 @@ describe('predictionAutoSelect', () => {
assert.sameDeepOrderedMembers(predictions, originalPredictions);
});
it(`selects solitary 'keep' suggestion that does match the model`, () => {
it(`selects nothing if solitary 'keep' suggestion does match the model`, () => {
const predictions: CompositedIntermediatePrediction[] = [
{
components: {
@ -47,7 +47,7 @@ describe('predictionAutoSelect', () => {
assert.sameDeepOrderedMembers(predictions, originalPredictions);
const autoselected = predictions.find((entry) => entry.components.prediction.autoAccept);
assert.isOk(autoselected);
assert.isNotOk(autoselected);
});
it(`does not select suggestions if the root correction has no letters`, () => {
@ -139,7 +139,7 @@ describe('predictionAutoSelect', () => {
assert.isNotOk(autoselected);
});
it(`selects 'keep' suggestion that does match the model over any alternatives`, () => {
it(`selects nothing for 'keep' suggestion that does match the model even with alternatives`, () => {
const keepSuggestion: CompositedIntermediatePrediction = {
components: {
prediction: {
@ -234,7 +234,7 @@ describe('predictionAutoSelect', () => {
assert.sameDeepMembers(predictions, originalPredictions);
const autoselected = predictions.find((entry) => entry.components.prediction.autoAccept);
assert.equal(autoselected, keepSuggestion);
assert.isNotOk(autoselected);
});
it(`selects solitary non-'keep' suggestion when 'keep' does not match model`, () => {

2
web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/base-context-state.tests.ts
View file

@ -168,7 +168,6 @@ describe('matchBaseContextState', () => {
const suggestion: Suggestion= {
transform: { insert: 'dramatically', deleteLeft: 0, id: 3 },
displayAs: 'dramatically',
transformId: 3,
id: 5
}
@ -234,7 +233,6 @@ describe('matchBaseContextState', () => {
const suggestion: Suggestion= {
transform: { insert: '', deleteLeft: 'dramatically'.length, id: 3 },
displayAs: '""',
transformId: 3,
id: 5
}

1
web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts
View file

@ -110,7 +110,6 @@ describe('createDefaultKeep', () => {
deleteLeft: 4,
id: transformId
},
transformId,
displayAs: '<appl>',
matchesModel: false,
tag: 'keep'

31
web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts
View file

@ -135,7 +135,6 @@ describe('determineContextTransition', () => {
insert: ' ',
deleteLeft: 0
},
transformId: 0,
displayAs: 'testing'
};
baseTransition.final.suggestions = [pred_testing];
@ -149,7 +148,7 @@ describe('determineContextTransition', () => {
}
};
compositor.acceptSuggestion(applied_testing, baseContext, { insert: '', deleteLeft: 0 });
compositor.acceptSuggestion(applied_testing, baseContext, { insert: '', deleteLeft: 0, id: applied_testing.transform.id });
const acceptingTransition = tracker.latest;
const inputDistribution: Distribution<Transform> = [{sample: applied_testing.appendedTransform, p: 1}];
@ -190,7 +189,7 @@ describe('determineContextTransition', () => {
transform: {
insert: 'testing',
deleteLeft: 4,
id: 1
id: 0
},
appendedTransform: {
insert: ' ',
@ -198,12 +197,11 @@ describe('determineContextTransition', () => {
id: 2
},
id: 4,
transformId: 0,
displayAs: 'testing'
};
baseTransition.final.suggestions = [pred_testing];
compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0 });
compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0, id: pred_testing.transform.id });
const inputDistribution: Distribution<Transform> = [{sample: { insert: 'a', deleteLeft: 0, id: 5 }, p: 1}];
@ -224,8 +222,8 @@ describe('determineContextTransition', () => {
assert.notEqual(extendingTransition, baseTransition);
// These values support delayed reversions.
assert.equal(extendingTransition.final.displayTokenization.tokens[6].appliedTransitionId, pred_testing.transformId);
assert.equal(extendingTransition.final.displayTokenization.tokens[7].appliedTransitionId, pred_testing.transformId);
assert.equal(extendingTransition.final.displayTokenization.tokens[6].appliedTransitionId, pred_testing.transform.id);
assert.equal(extendingTransition.final.displayTokenization.tokens[7].appliedTransitionId, pred_testing.transform.id);
// We start a new token here, rather than continue (and/or replace) an old one;
// this shouldn't be set here yet.
@ -251,19 +249,18 @@ describe('determineContextTransition', () => {
transform: {
insert: 'testing',
deleteLeft: 4,
id: 1
id: 0
},
appendedTransform: {
insert: ' ',
deleteLeft: 0,
id: 2
},
transformId: 0,
displayAs: 'testing'
};
baseTransition.final.suggestions = [pred_testing];
compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0 });
compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0, id: pred_testing.transform.id });
const inputDistribution: Distribution<Transform> = [{sample: { insert: 'a', deleteLeft: 0, id: 5 }, p: 1}];
@ -291,7 +288,7 @@ describe('determineContextTransition', () => {
[{sample: { insert: '', deleteLeft: 1 }, p: 1}]
);
assert.equal(extensionDeletingTransition.revertableTransitionId, pred_testing.transformId);
assert.equal(extensionDeletingTransition.revertableTransitionId, pred_testing.transform.id);
} finally {
warningEmitterSpy.restore();
}
@ -313,19 +310,18 @@ describe('determineContextTransition', () => {
transform: {
insert: 'testing',
deleteLeft: 4,
id: 1
id: 0
},
appendedTransform: {
insert: ' ',
deleteLeft: 0,
id: 2
},
transformId: 0,
displayAs: 'testing'
};
baseTransition.final.suggestions = [pred_testing];
compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0 });
compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0, id: pred_testing.transform.id });
const inputDistribution: Distribution<Transform> = [{sample: { insert: 'a', deleteLeft: 0, id: 5 }, p: 1}];
@ -365,7 +361,7 @@ describe('determineContextTransition', () => {
[{sample: { insert: '', deleteLeft: 1 }, p: 1}]
);
assert.equal(appendDeletingTransition.revertableTransitionId, pred_testing.transformId);
assert.equal(appendDeletingTransition.revertableTransitionId, pred_testing.transform.id);
} finally {
warningEmitterSpy.restore();
}
@ -387,19 +383,18 @@ describe('determineContextTransition', () => {
transform: {
insert: 'testing',
deleteLeft: 4,
id: 1
id: 0
},
appendedTransform: {
insert: ' ',
deleteLeft: 0,
id: 2
},
transformId: 0,
displayAs: 'testing'
};
baseTransition.final.suggestions = [pred_testing];
compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0 });
compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0, id: pred_testing.transform.id });
const inputDistribution: Distribution<Transform> = [{sample: { insert: 'a', deleteLeft: 0, id: 5 }, p: 1}];

29
web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/predict-from-correction-sequence.tests.ts
View file

@ -137,7 +137,6 @@ describe('predictFromCorrectionSequence', () => {
assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components[0].prediction), dummied_suggestions.map((s) => {
delete s.p;
s.transformId = transitionID;
s.transform.id = transitionID;
return s;
}));
@ -205,14 +204,13 @@ describe('predictFromCorrectionSequence', () => {
assert.sameOrderedMembers(predictions.map((entry) => entry.components[0].prediction.displayAs), ["it's", "its"]);
assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components[0].prediction), dummied_suggestions.map((entry) => {
entry = deepCopy(entry);
entry.transformId = transitionID;
entry.transform.id = transitionID;
return entry;
}));
assert.approximately(predictions[0].metadata.probabilities.total, 0.18 * 0.6, 0.00001);
assert.approximately(predictions[1].metadata.probabilities.total, 0.02 * 0.6, 0.00001);
predictions.forEach((prediction) => assert.equal(prediction.components[0].prediction.transformId, transitionID));
predictions.forEach((prediction) => assert.equal(prediction.components[0].prediction.transform.id, transitionID));
});
it('constructs suggestions without input (as if after a context reset)', () => {
@ -266,7 +264,6 @@ describe('predictFromCorrectionSequence', () => {
assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components.map((c) => c.prediction)), [dummied_suggestions.map((s) => {
delete s.p;
s.transformId = transitionID;
s.transform.id = transitionID;
return s;
})]);
@ -355,24 +352,21 @@ describe('predictFromCorrectionSequence', () => {
deleteLeft: 0,
id: transitionID
},
displayAs: 'g',
transformId: transitionID,
displayAs: 'g'
}, {
transform: {
insert: ' ',
deleteLeft: 0,
id: transitionID
},
displayAs: ' ',
transformId: transitionID
displayAs: ' '
}, {
transform: {
insert: 'apple',
deleteLeft: 0,
id: transitionID
},
displayAs: 'apple',
transformId: transitionID
displayAs: 'apple'
}
];
@ -565,24 +559,21 @@ describe('predictFromCorrectionSequence', () => {
deleteLeft: 0,
id: transitionID
},
displayAs: 'g',
transformId: transitionID
displayAs: 'g'
}, {
transform: {
insert: ' ',
deleteLeft: 0,
id: transitionID
},
displayAs: ' ',
transformId: transitionID
displayAs: ' '
}, {
transform: {
insert: 'apple',
deleteLeft: 0,
id: transitionID
},
displayAs: 'apple',
transformId: transitionID
displayAs: 'apple'
}
];
@ -710,16 +701,14 @@ describe('predictFromCorrectionSequence', () => {
deleteLeft: 0,
id: transitionID
},
displayAs: 'golden',
transformId: transitionID
displayAs: 'golden'
}, {
transform: {
insert: ' ',
deleteLeft: 0,
id: transitionID
},
displayAs: ' ',
transformId: transitionID
displayAs: ' '
}
];

101
web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/prepend-reversion.tests.ts Normal file
View file

@ -0,0 +1,101 @@
/*
* Keyman is copyright (C) SIL Global. MIT License.
*
* Created by jahorton on 2026-07-27
*
* This file contains tests designed to ensure predictive text does not
* provide matching 'keep' and 'revert' suggestions in any context.
*/
import { assert } from 'chai';
import { LexicalModelTypes } from '@keymanapp/common-types';
import { prependReversion, type TransitionReversionView } from "@keymanapp/lm-worker/test-index";
import Suggestion = LexicalModelTypes.Suggestion;
describe('prependReversion', () => {
it(`prepends reversions when reverting a non-'keep' suggestion`, () => {
// context: Original was appl+u, corrected to apply. Reached via bksp.
const suggestions: Suggestion[] = [{
tag: 'keep',
transform: { insert: 'apply', deleteLeft: 6, id: 3 },
displayAs: '"apply"',
id: 5,
matchesModel: false
} as Suggestion];
const revertable: TransitionReversionView = {
reversion: {
tag: 'revert',
transform: { insert: 'u', deleteLeft: 0, id: 3 },
id: -3,
displayAs: '"applu"'
},
final: {
suggestions: [{
tag: 'keep',
transform: { insert: 'applu', deleteLeft: 4, id: 3 },
displayAs: '"applu"',
id: 2,
matchesModel: false
} as Suggestion, {
transform: { insert: 'apply', deleteLeft: 4, id: 3 },
displayAs: 'apply',
id: 3
}, {
transform: { insert: 'applied', deleteLeft: 4, id: 3 },
displayAs: 'applied',
id: 4
}]
}
};
prependReversion(suggestions, revertable);
assert.includeMembers(suggestions, [revertable.reversion]);
});
it(`does not prepend reversions when reverting a 'keep' suggestion`, () => {
// context: Original was appl+u, corrected to apply
const suggestions: Suggestion[] = [{
tag: 'keep',
transform: { insert: 'applu', deleteLeft: 5, id: 3 },
displayAs: '"applu"',
id: 5,
matchesModel: false
} as Suggestion];
const revertable: TransitionReversionView = {
reversion: {
tag: 'revert',
transform: { insert: 'u', deleteLeft: 0, id: 3 },
id: -2,
displayAs: '"applu"'
},
final: {
suggestions: [{
tag: 'keep',
transform: { insert: 'applu', deleteLeft: 4, id: 3 },
displayAs: '"applu"',
id: 2,
matchesModel: false
} as Suggestion, {
transform: { insert: 'apply', deleteLeft: 4, id: 3 },
displayAs: 'apply',
id: 3
}, {
transform: { insert: 'applied', deleteLeft: 4, id: 3 },
displayAs: 'applied',
id: 4
}]
}
};
prependReversion(suggestions, revertable);
assert.notIncludeMembers(suggestions, [revertable.reversion]);
});
});

1
web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts
View file

@ -296,7 +296,6 @@ describe('processSimilarity', () => {
deleteLeft: 4,
id: transformId
},
transformId,
displayAs: 'apple'
},
correction: 'appl'

45
web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts
View file

@ -588,7 +588,6 @@ describe('ModelCompositor', function() {
deleteLeft: 0,
id: 0
},
transformId: 0,
displayAs: 'hello'
};
@ -668,13 +667,12 @@ describe('ModelCompositor', function() {
}
it('first word of context, postTransform provided, .deleteLeft = 0', function() {
let baseSuggestion = {
let baseSuggestion: Suggestion = {
transform: {
insert: 'hello ',
deleteLeft: 2,
id: 0
},
transformId: 0,
id: 1,
displayAs: 'hello'
};
@ -687,11 +685,12 @@ describe('ModelCompositor', function() {
// of the Context when the suggestion is built.
let postTransform = {
insert: 'l',
deleteLeft: 0
deleteLeft: 0,
id: baseSuggestion.transform.id
}
let reversion = acceptanceTest(englishPunctuation, baseSuggestion, baseContext, postTransform);
assert.equal(reversion.transformId, baseSuggestion.transformId);
assert.equal(reversion.transform.id, baseSuggestion.transform.id);
assert.equal(reversion.id, -baseSuggestion.id);
// Check #1: Does the returned reversion properly revert the context to its pre-application state?
@ -711,13 +710,12 @@ describe('ModelCompositor', function() {
});
it('second word of context, postTransform provided, .deleteLeft = 0', function() {
let baseSuggestion = {
let baseSuggestion: Suggestion = {
transform: {
insert: 'world ',
deleteLeft: 3,
id: 0
},
transformId: 0,
id: 0,
displayAs: 'world'
};
@ -730,11 +728,12 @@ describe('ModelCompositor', function() {
// of the Context when the suggestion is built.
let postTransform = {
insert: 'l',
deleteLeft: 0
deleteLeft: 0,
id: baseSuggestion.id
}
let reversion = acceptanceTest(englishPunctuation, baseSuggestion, baseContext, postTransform);
assert.equal(reversion.transformId, baseSuggestion.transformId);
assert.equal(reversion.transform.id, baseSuggestion.transform.id);
assert.equal(reversion.id, -baseSuggestion.id);
// Check #1: Does the returned reversion properly revert the context to its pre-application state?
@ -754,13 +753,12 @@ describe('ModelCompositor', function() {
});
it('second word of context, postTransform undefined', function() {
let baseSuggestion = {
let baseSuggestion: Suggestion = {
transform: {
insert: 'world ',
deleteLeft: 3,
id: 0
},
transformId: 0,
id: 0,
displayAs: 'world'
};
@ -770,7 +768,7 @@ describe('ModelCompositor', function() {
}
let reversion = acceptanceTest(englishPunctuation, baseSuggestion, baseContext);
assert.equal(reversion.transformId, baseSuggestion.transformId);
assert.equal(reversion.transform.id, baseSuggestion.transform.id);
assert.equal(reversion.id, -baseSuggestion.id);
// Check #1: Does the returned reversion properly revert the context to its pre-application state?
@ -790,13 +788,12 @@ describe('ModelCompositor', function() {
});
it('first word of context + postTransform provided, .deleteLeft > 0', function() {
let baseSuggestion = {
let baseSuggestion: Suggestion = {
transform: {
insert: 'hello ',
deleteLeft: 2,
id: 0
},
transformId: 0,
id: 0,
displayAs: 'hello'
};
@ -809,11 +806,12 @@ describe('ModelCompositor', function() {
// of the Context when the suggestion is built.
let postTransform = {
insert: 'i',
deleteLeft: 1
deleteLeft: 1,
id: baseSuggestion.transform.id
}
let reversion = acceptanceTest(englishPunctuation, baseSuggestion, baseContext, postTransform);
assert.equal(reversion.transformId, baseSuggestion.transformId);
assert.equal(reversion.transform.id, baseSuggestion.transform.id);
assert.equal(reversion.id, -baseSuggestion.id);
// Check #1: Does the returned reversion properly revert the context to its pre-application state?
@ -847,13 +845,12 @@ describe('ModelCompositor', function() {
// it('first word of context + postTransform provided, .deleteLeft > 0')
// seen earlier in the file.
let baseSuggestion = {
let baseSuggestion: Suggestion = {
transform: {
insert: 'hello ',
deleteLeft: 2,
id: 0
},
transformId: 0,
id: 0,
displayAs: 'hello'
};
@ -866,7 +863,8 @@ describe('ModelCompositor', function() {
// of the Context when the suggestion is built.
let postTransform = {
insert: 'i',
deleteLeft: 1
deleteLeft: 1,
id: baseSuggestion.transform.id
}
// Future adjustment: add the 'baseSuggestion' to DummyModel so that it actually
@ -876,7 +874,7 @@ describe('ModelCompositor', function() {
let compositor = new ModelCompositor(model, true);
let reversion = compositor.acceptSuggestion(baseSuggestion, baseContext, postTransform);
assert.equal(reversion.transformId, baseSuggestion.transformId);
assert.equal(reversion.transform.id, baseSuggestion.transform.id);
assert.equal(reversion.id, -baseSuggestion.id);
let appliedContext = models.applyTransform(baseSuggestion.transform, baseContext);
@ -891,7 +889,8 @@ describe('ModelCompositor', function() {
let expectedTransform = {
insert: 'hi', // Keeps current context the same, though it adds a wordbreak.
deleteLeft: 2
deleteLeft: 2,
id: 0
}
assert.deepEqual(suggestions[0].transform, expectedTransform);
assert.deepEqual(suggestions[0].appendedTransform, {
@ -927,7 +926,7 @@ describe('ModelCompositor', function() {
let baseSuggestion = initialSuggestions[1];
baseSuggestion.appendedTransform.id = 15; // set an id for the applied transform.
let reversion = compositor.acceptSuggestion(baseSuggestion, baseContext, postTransform);
assert.equal(reversion.transformId, -baseSuggestion.transformId);
assert.equal(reversion.transform.id, baseSuggestion.transform.id);
assert.equal(reversion.id, -baseSuggestion.id);
let appliedContext = models.applyTransform(baseSuggestion.transform, baseContext);
@ -974,7 +973,7 @@ describe('ModelCompositor', function() {
assert.equal(compositor.contextTracker.unitTestEndPoints.cache().size, 2);
let contextIds = compositor.contextTracker.unitTestEndPoints.cache().keys();
assert.equal(reversion.transformId, -baseSuggestion.transformId);
assert.equal(reversion.transform.id, baseSuggestion.transform.id);
assert.equal(reversion.id, -baseSuggestion.id);
const appliedContextState = compositor.contextTracker.unitTestEndPoints.cache().get(15);