mirror of
https://github.com/keymanapp/keyman.git
synced 2026-08-05 00:15:32 +00:00
Merge branch 'epic/boundary-correction' into feat/web/prep-tokenization-search
Some checks failed
Keyman Build Summary / Summarize build status checks (push) Has been cancelled
Some checks failed
Keyman Build Summary / Summarize build status checks (push) Has been cancelled
This commit is contained in:
Joshua Horton
commit
d5c3cc2e06
35 changed files with 786 additions and 287 deletions
|
|
@ -253,9 +253,14 @@ export interface LexicalModel {
|
|||
*/
|
||||
export interface Transform {
|
||||
/**
|
||||
* Facilitates use of unique identifiers for tracking the Transform and
|
||||
* any related data from its original source, as the reference cannot be
|
||||
* preserved across WebWorker boundaries.
|
||||
* Facilitates use of unique identifiers for tracking data about the context
|
||||
* transition to which the Transform belongs. More than one Transform may
|
||||
* hold the same `id` if they are alternate interpretations of the same
|
||||
* transition event - say, the resulting effects of neighbor keys that may
|
||||
* have been missed due to "fat fingering".
|
||||
*
|
||||
* Also note that the Transform reference cannot be preserved across WebWorker
|
||||
* boundaries, but this ID may.
|
||||
*
|
||||
* This is *separate* from any LMLayer-internal identification values.
|
||||
*/
|
||||
|
|
@ -287,13 +292,6 @@ export interface Transform {
|
|||
* A concrete suggestion
|
||||
*/
|
||||
export interface Suggestion {
|
||||
/**
|
||||
* Indicates the externally-supplied id of the Transform that prompted
|
||||
* the Suggestion. Automatically handled by the LMLayer; models should
|
||||
* not handle this field.
|
||||
*/
|
||||
transformId?: number;
|
||||
|
||||
/**
|
||||
* A unique identifier for the Suggestion itself, not shared with any others -
|
||||
* even for Suggestions sourced from the same Transform.
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ public enum Key {
|
|||
/// Dictionary of prediction/correction toggle settings keyed by language id in UserDefaults
|
||||
static let userPredictSettings = "UserPredictionEnablementSettings"
|
||||
static let userCorrectSettings = "UserCorrectionEnablementSettings"
|
||||
static let userAutocorrectSettings = "UserAutocorrectionEnablementSettings"
|
||||
|
||||
// Internal user defaults keys
|
||||
static let engineVersion = "KeymanEngineVersion"
|
||||
|
|
|
|||
|
|
@ -331,6 +331,17 @@ public extension UserDefaults {
|
|||
set(prefs, forKey: Key.userCorrectSettings)
|
||||
}
|
||||
}
|
||||
|
||||
// stores a dictionary of autocorrection-enablement settings keyed to language ids, i.e., [langID: Bool]
|
||||
var autocorrectionEnablements: [String: Bool]? {
|
||||
get {
|
||||
return dictionary(forKey: Key.userAutocorrectSettings) as? [String : Bool]
|
||||
}
|
||||
|
||||
set(prefs) {
|
||||
set(prefs, forKey: Key.userAutocorrectSettings)
|
||||
}
|
||||
}
|
||||
|
||||
var portraitKeyboardHeight: Double {
|
||||
get {
|
||||
|
|
@ -387,4 +398,22 @@ public extension UserDefaults {
|
|||
prefs?[forLanguageID] = correctSetting
|
||||
correctionEnablements = prefs
|
||||
}
|
||||
|
||||
func autocorrectSettingForLanguage(languageID: String) -> Bool {
|
||||
if let dict = autocorrectionEnablements {
|
||||
return dict[languageID] ?? true
|
||||
} else {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func set(autocorrectSetting: Bool, forLanguageID: String) {
|
||||
var prefs: [String: Bool]?
|
||||
prefs = autocorrectionEnablements
|
||||
if prefs == nil {
|
||||
prefs = [String: Bool]()
|
||||
}
|
||||
prefs?[forLanguageID] = autocorrectSetting
|
||||
autocorrectionEnablements = prefs
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -433,10 +433,11 @@ extension KeymanWebViewController {
|
|||
|
||||
let predict = userDefaults.predictSettingForLanguage(languageID: lexicalModel.languageID)
|
||||
let correct = userDefaults.correctSettingForLanguage(languageID: lexicalModel.languageID)
|
||||
let autocorrect = userDefaults.autocorrectSettingForLanguage(languageID: lexicalModel.languageID)
|
||||
|
||||
// Pass these off to KMW!
|
||||
// We do these first so that they're automatically set for the to-be-registered model in advance.
|
||||
webView!.evaluateJavaScript("enableSuggestions(\(stubString), \(predict), \(correct))")
|
||||
webView!.evaluateJavaScript("enableSuggestions(\(stubString), \(predict), \(correct), \(autocorrect))")
|
||||
self.activeModel = predict
|
||||
} else { // We're registering a model in the background - don't change settings.
|
||||
webView!.evaluateJavaScript("keyman.addModel(\(stubString));", completionHandler: nil)
|
||||
|
|
|
|||
|
|
@ -18,8 +18,11 @@ class LanguageSettingsViewController: UITableViewController {
|
|||
|
||||
private var doPredictionsSwitch: UISwitch?
|
||||
private var doCorrectionsSwitch: UISwitch?
|
||||
private var doAutocorrectionsSwitch: UISwitch?
|
||||
private var doCorrectionsLabel: UILabel?
|
||||
private var doAutocorrectionsLabel: UILabel?
|
||||
private var correctionsCell: UITableViewCell?
|
||||
private var autocorrectionsCell: UITableViewCell?
|
||||
|
||||
public init(_ inLanguage: Language) {
|
||||
language = inLanguage
|
||||
|
|
@ -87,17 +90,8 @@ class LanguageSettingsViewController: UITableViewController {
|
|||
return switchFrame
|
||||
}
|
||||
|
||||
@objc
|
||||
func predictionSwitchValueChanged(source: UISwitch) {
|
||||
let value = source.isOn;
|
||||
func refreshModelIfNeeded() {
|
||||
let userDefaults = Storage.active.userDefaults
|
||||
userDefaults.set(predictSetting: value, forLanguageID: self.language.id)
|
||||
|
||||
// Reactively set the corrections switch interactivity state.
|
||||
self.doCorrectionsSwitch?.isHidden = !value
|
||||
self.doCorrectionsLabel?.isEnabled = value
|
||||
self.correctionsCell?.isUserInteractionEnabled = value
|
||||
|
||||
if let lm = Manager.shared.preferredLexicalModel(userDefaults, forLanguage: self.language.id) {
|
||||
if Manager.shared.currentKeyboardID?.languageID == self.language.id {
|
||||
// re-register the model - that'll enact the settings.
|
||||
|
|
@ -111,23 +105,63 @@ class LanguageSettingsViewController: UITableViewController {
|
|||
}
|
||||
}
|
||||
|
||||
func refreshSwitchVisibility() {
|
||||
let userDefaults = Storage.active.userDefaults
|
||||
|
||||
let mayPredict = userDefaults.predictSettingForLanguage(languageID: self.language.id)
|
||||
self.doCorrectionsSwitch?.isHidden = !mayPredict
|
||||
self.doCorrectionsLabel?.isEnabled = mayPredict
|
||||
self.correctionsCell?.isUserInteractionEnabled = mayPredict
|
||||
|
||||
let mayCorrect = userDefaults.correctSettingForLanguage(languageID: self.language.id)
|
||||
self.doAutocorrectionsSwitch?.isHidden = !(mayPredict && mayCorrect)
|
||||
self.doAutocorrectionsLabel?.isEnabled = mayPredict && mayCorrect
|
||||
self.autocorrectionsCell?.isUserInteractionEnabled = mayPredict && mayCorrect
|
||||
}
|
||||
|
||||
@objc
|
||||
func predictionSwitchValueChanged(source: UISwitch) {
|
||||
let value = source.isOn;
|
||||
let userDefaults = Storage.active.userDefaults
|
||||
userDefaults.set(predictSetting: value, forLanguageID: self.language.id)
|
||||
|
||||
refreshSwitchVisibility()
|
||||
refreshModelIfNeeded()
|
||||
}
|
||||
|
||||
@objc
|
||||
func correctionSwitchValueChanged(source: UISwitch) {
|
||||
let value = source.isOn;
|
||||
let userDefaults = Storage.active.userDefaults
|
||||
userDefaults.set(correctSetting: value, forLanguageID: self.language.id)
|
||||
|
||||
if let lm = Manager.shared.preferredLexicalModel(userDefaults, forLanguage: self.language.id) {
|
||||
if Manager.shared.currentKeyboardID?.languageID == self.language.id {
|
||||
// re-register the model - that'll enact the settings.
|
||||
_ = Manager.shared.registerLexicalModel(lm)
|
||||
}
|
||||
// Based on how Manager chooses the model in setKeyboard.
|
||||
} else if let lm = userDefaults.userLexicalModels?.first(where: { $0.languageID == self.language.id }) {
|
||||
if Manager.shared.currentKeyboardID?.languageID == self.language.id {
|
||||
_ = Manager.shared.registerLexicalModel(lm)
|
||||
}
|
||||
}
|
||||
refreshSwitchVisibility()
|
||||
refreshModelIfNeeded()
|
||||
}
|
||||
|
||||
@objc
|
||||
func autocorrectionSwitchValueChanged(source: UISwitch) {
|
||||
let value = source.isOn;
|
||||
let userDefaults = Storage.active.userDefaults
|
||||
userDefaults.set(autocorrectSetting: value, forLanguageID: self.language.id)
|
||||
|
||||
refreshModelIfNeeded()
|
||||
}
|
||||
|
||||
func addSwitchToTableCell(_ toggle: UISwitch, cell: UITableViewCell, isOn: Bool, selector: Selector) {
|
||||
cell.accessoryType = .none
|
||||
toggle.translatesAutoresizingMaskIntoConstraints = false
|
||||
|
||||
let switchFrame = frameAtRightOfCell(cell: cell.frame, controlSize: toggle.frame.size)
|
||||
toggle.frame = switchFrame
|
||||
|
||||
toggle.isOn = isOn
|
||||
toggle.addTarget(self, action: selector, for: .valueChanged)
|
||||
cell.addSubview(toggle)
|
||||
cell.contentView.isUserInteractionEnabled = false
|
||||
|
||||
toggle.rightAnchor.constraint(equalTo: cell.layoutMarginsGuide.rightAnchor).isActive = true
|
||||
toggle.centerYAnchor.constraint(equalTo: cell.layoutMarginsGuide.centerYAnchor).isActive = true
|
||||
}
|
||||
|
||||
override func tableView(_ tableView: UITableView, cellForRowAt indexPath: IndexPath) -> UITableViewCell {
|
||||
|
|
@ -143,40 +177,38 @@ class LanguageSettingsViewController: UITableViewController {
|
|||
let cell = UITableViewCell(style: .subtitle, reuseIdentifier: cellIdentifier)
|
||||
if 1 == indexPath.section {
|
||||
if 0 == indexPath.row {
|
||||
cell.accessoryType = .none
|
||||
doPredictionsSwitch = UISwitch()
|
||||
doPredictionsSwitch!.translatesAutoresizingMaskIntoConstraints = false
|
||||
|
||||
let switchFrame = frameAtRightOfCell(cell: cell.frame, controlSize: doPredictionsSwitch!.frame.size)
|
||||
doPredictionsSwitch!.frame = switchFrame
|
||||
|
||||
doPredictionsSwitch!.isOn = userDefaults.predictSettingForLanguage(languageID: self.language.id)
|
||||
doPredictionsSwitch!.addTarget(self, action: #selector(self.predictionSwitchValueChanged), for: .valueChanged)
|
||||
cell.addSubview(doPredictionsSwitch!)
|
||||
cell.contentView.isUserInteractionEnabled = false
|
||||
|
||||
doPredictionsSwitch!.rightAnchor.constraint(equalTo: cell.layoutMarginsGuide.rightAnchor).isActive = true
|
||||
doPredictionsSwitch!.centerYAnchor.constraint(equalTo: cell.layoutMarginsGuide.centerYAnchor).isActive = true
|
||||
self.addSwitchToTableCell(
|
||||
doPredictionsSwitch!,
|
||||
cell: cell,
|
||||
isOn: userDefaults.predictSettingForLanguage(languageID: self.language.id),
|
||||
selector: #selector(self.predictionSwitchValueChanged)
|
||||
)
|
||||
} else if 1 == indexPath.row {
|
||||
correctionsCell = cell
|
||||
cell.accessoryType = .none
|
||||
doCorrectionsSwitch = UISwitch()
|
||||
doCorrectionsSwitch!.translatesAutoresizingMaskIntoConstraints = false
|
||||
|
||||
let switchFrame = frameAtRightOfCell(cell: cell.frame, controlSize: doCorrectionsSwitch!.frame.size)
|
||||
doCorrectionsSwitch!.frame = switchFrame
|
||||
self.addSwitchToTableCell(
|
||||
doCorrectionsSwitch!,
|
||||
cell: cell,
|
||||
isOn: userDefaults.correctSettingForLanguage(languageID: self.language.id),
|
||||
selector: #selector(self.correctionSwitchValueChanged)
|
||||
)
|
||||
|
||||
refreshSwitchVisibility()
|
||||
} else if 2 == indexPath.row {
|
||||
autocorrectionsCell = cell
|
||||
doAutocorrectionsSwitch = UISwitch()
|
||||
|
||||
doCorrectionsSwitch!.isOn = userDefaults.correctSettingForLanguage(languageID: self.language.id)
|
||||
doCorrectionsSwitch!.addTarget(self, action: #selector(self.correctionSwitchValueChanged), for: .valueChanged)
|
||||
cell.addSubview(doCorrectionsSwitch!)
|
||||
cell.contentView.isUserInteractionEnabled = false
|
||||
self.addSwitchToTableCell(
|
||||
doAutocorrectionsSwitch!,
|
||||
cell: cell,
|
||||
isOn: userDefaults.autocorrectSettingForLanguage(languageID: self.language.id),
|
||||
selector: #selector(self.autocorrectionSwitchValueChanged)
|
||||
)
|
||||
|
||||
doCorrectionsSwitch!.rightAnchor.constraint(equalTo: cell.layoutMarginsGuide.rightAnchor).isActive = true
|
||||
doCorrectionsSwitch!.centerYAnchor.constraint(equalTo: cell.layoutMarginsGuide.centerYAnchor).isActive = true
|
||||
|
||||
// Disable interactivity if the prediction toggle is set to 'off'.
|
||||
doCorrectionsSwitch!.isHidden = !userDefaults.predictSettingForLanguage(languageID: self.language.id)
|
||||
cell.isUserInteractionEnabled = userDefaults.predictSettingForLanguage(languageID: self.language.id)
|
||||
refreshSwitchVisibility()
|
||||
} else { // rows 3 and 4
|
||||
cell.accessoryType = .disclosureIndicator
|
||||
}
|
||||
|
|
@ -245,6 +277,10 @@ class LanguageSettingsViewController: UITableViewController {
|
|||
cell.textLabel?.text = NSLocalizedString("menu-langsettings-toggle-correct", bundle: engineBundle, comment: "")
|
||||
cell.textLabel?.isEnabled = !(doCorrectionsSwitch?.isHidden ?? false)
|
||||
case 2:
|
||||
doAutocorrectionsLabel = cell.textLabel
|
||||
cell.textLabel?.text = NSLocalizedString("menu-langsettings-toggle-autocorrect", bundle: engineBundle, comment: "")
|
||||
cell.textLabel?.isEnabled = !(doAutocorrectionsSwitch?.isHidden ?? false)
|
||||
case 3:
|
||||
cell.textLabel?.text = NSLocalizedString("menu-langsettings-label-lexical-models", bundle: engineBundle, comment: "")
|
||||
cell.accessoryType = .disclosureIndicator
|
||||
let modelCt = language.lexicalModels?.count ?? 0
|
||||
|
|
@ -332,8 +368,8 @@ class LanguageSettingsViewController: UITableViewController {
|
|||
showKeyboardInfoView(kb: (language.keyboards?[safe: indexPath.row])!)
|
||||
case 1:
|
||||
switch indexPath.row {
|
||||
// case 0, 1: the toggles - but a general 'click' not on the toggle itself.
|
||||
case 2:
|
||||
// case 0, 1, 2: the toggles - but a general 'click' not on the toggle itself.
|
||||
case 3:
|
||||
showLexicalModelsView()
|
||||
default:
|
||||
break
|
||||
|
|
|
|||
|
|
@ -152,10 +152,13 @@
|
|||
"menu-langsettings-title" = "%@ Settings";
|
||||
|
||||
/* Label for the toggle that enables corrections that is displayed within a language-specific settings menu */
|
||||
"menu-langsettings-toggle-correct" = "Enable corrections";
|
||||
"menu-langsettings-toggle-correct" = "Offer corrections";
|
||||
|
||||
/* Label for the toggle that enables auto-corrections that is displayed within a language-specific settings menu */
|
||||
"menu-langsettings-toggle-autocorrect" = "Apply corrections automatically";
|
||||
|
||||
/* Label for the toggle that enables predictions that is displayed within a language-specific settings menu */
|
||||
"menu-langsettings-toggle-predict" = "Enable predictions";
|
||||
"menu-langsettings-toggle-predict" = "Offer word completions";
|
||||
|
||||
/* Help message for a prompt that appears for confirming a lexical model download: language (1): lexical model (dictionary) name (2) */
|
||||
"menu-lexical-model-install-message" = "Would you like to install this dictionary?";
|
||||
|
|
|
|||
|
|
@ -326,11 +326,12 @@ function toHex(theString) {
|
|||
return hexString.substr(0, hexString.length-1);
|
||||
}
|
||||
|
||||
function enableSuggestions(model, mayPredict, mayCorrect) {
|
||||
function enableSuggestions(model, mayPredict, mayCorrect, mayAutocorrect) {
|
||||
// Set the options first so that KMW's ModelCache can properly handle model enablement states
|
||||
// the moment we actually register the new model.
|
||||
keyman.core.languageProcessor.mayPredict = mayPredict;
|
||||
keyman.core.languageProcessor.mayCorrect = mayCorrect;
|
||||
keyman.core.languageProcessor.mayAutoCorrect = mayAutocorrect;
|
||||
|
||||
keyman.addModel(model);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ export function buildMergedTransform(first: Transform, second: Transform): Trans
|
|||
deleteLeft: first.deleteLeft + mergedSecondDelete
|
||||
}
|
||||
|
||||
if(first.id && first.id == second.id) {
|
||||
if(first.id !== undefined && first.id == second.id) {
|
||||
returnedObj.id = first.id;
|
||||
}
|
||||
|
||||
|
|
@ -132,10 +132,6 @@ export function transformToSuggestion(transform: Transform, p?: number): Outcome
|
|||
displayAs: transform.insert
|
||||
};
|
||||
|
||||
if(transform.id !== undefined) {
|
||||
suggestion.transformId = transform.id;
|
||||
}
|
||||
|
||||
if(p === 0 || p) {
|
||||
suggestion.p = p;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -95,7 +95,7 @@ export class ContextState {
|
|||
return undefined;
|
||||
}
|
||||
|
||||
return this.suggestions.find(s => s.id == this.appliedSuggestionId)?.transformId;
|
||||
return this.suggestions.find(s => s.id == this.appliedSuggestionId)?.transform.id;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@
|
|||
import { LexicalModelTypes } from '@keymanapp/common-types';
|
||||
|
||||
import { SearchQuotientNode, PathInputProperties } from "./search-quotient-node.js";
|
||||
import { TokenSplitMap } from "./context-tokenization.js";
|
||||
import { TokenSplitMapping } from "./context-tokenization.js";
|
||||
import { LegacyQuotientSpur } from "./legacy-quotient-spur.js";
|
||||
import { LegacyQuotientRoot } from "./legacy-quotient-root.js";
|
||||
import { generateSubsetId } from './tokenization-subsets.js';
|
||||
|
|
@ -26,12 +26,12 @@ import Transform = LexicalModelTypes.Transform;
|
|||
* any prior cached data or for rewriting its probabilities after
|
||||
* receiving backspace input.
|
||||
* @param text
|
||||
* @param transformId
|
||||
* @param transitionId
|
||||
* @returns
|
||||
*/
|
||||
function textToCharTransforms(text: string, transformId?: number): Transform[] {
|
||||
return transformId ?
|
||||
[...text].map(insert => ({insert, deleteLeft: 0, id: transformId})) :
|
||||
function textToCharTransforms(text: string, transitionId?: number): Transform[] {
|
||||
return transitionId ?
|
||||
[...text].map(insert => ({insert, deleteLeft: 0, id: transitionId})) :
|
||||
[...text].map(insert => ({insert, deleteLeft: 0}));
|
||||
}
|
||||
|
||||
|
|
@ -134,7 +134,7 @@ export class ContextToken implements ContextTokenLike {
|
|||
static fromRawText(model: LexicalModel, rawText: string, isPartial?: boolean) {
|
||||
rawText ||= '';
|
||||
|
||||
// Supports the old pathway for: updateWithBackspace(tokenText: string, transformId: number)
|
||||
// Supports the old pathway for: updateWithBackspace(tokenText: string, transitionId: number)
|
||||
// Build a token that represents the current text with no ambiguity - probability at max (1.0)
|
||||
let searchModule: SearchQuotientNode = new LegacyQuotientRoot(model);
|
||||
const BASE_PROBABILITY = 1;
|
||||
|
|
@ -233,7 +233,7 @@ export class ContextToken implements ContextTokenLike {
|
|||
* @param lexicalModel
|
||||
* @returns
|
||||
*/
|
||||
split(split: TokenSplitMap): ContextToken[] {
|
||||
split(split: TokenSplitMapping): ContextToken[] {
|
||||
// Split from tail to head - leave as much 'head' intact as possible at each
|
||||
// step, rather than needing to reconstruct the tail multiple times.
|
||||
const splitSpecs = split.matches.slice();
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ const MIN_CHARS_TO_RECONSIDER_FOR_TOKENIZATION = 8;
|
|||
* This type is used to indicate properties of tokens affected by merges and
|
||||
* splits during a context transition.
|
||||
*/
|
||||
interface EditTokenMap {
|
||||
interface EditTokenMappingHalf {
|
||||
/**
|
||||
* The index of the affected token.
|
||||
*/
|
||||
|
|
@ -43,10 +43,22 @@ interface EditTokenMap {
|
|||
text: string
|
||||
};
|
||||
|
||||
/**
|
||||
* This type is used to indicate properties of tokens affected by splits during
|
||||
* a context transition.
|
||||
*/
|
||||
interface SplitTokenMappingHalf extends EditTokenMappingHalf {
|
||||
/**
|
||||
* The codepoint index within the original token at which the split-off token
|
||||
* begins.
|
||||
*/
|
||||
textOffset: number
|
||||
};
|
||||
|
||||
/**
|
||||
* This type represents mappings for tokens affected by merge edit operations.
|
||||
*/
|
||||
interface TokenMergeMap {
|
||||
interface TokenMergeMapping {
|
||||
/**
|
||||
* Entries here represent source-context tokens that are combined as part of
|
||||
* the merge edit operation.
|
||||
|
|
@ -54,7 +66,7 @@ interface TokenMergeMap {
|
|||
* Entries will appear in the same ordering as the codepoints they represent
|
||||
* appear in the underlying context.
|
||||
*/
|
||||
inputs: EditTokenMap[],
|
||||
inputs: EditTokenMappingHalf[],
|
||||
|
||||
/**
|
||||
* This entry represents the post-transition context token produced from the
|
||||
|
|
@ -63,14 +75,14 @@ interface TokenMergeMap {
|
|||
* Note that it is possible for extra codepoints to exist here that were not
|
||||
* represented in the original source-context tokens.
|
||||
*/
|
||||
match: EditTokenMap
|
||||
match: EditTokenMappingHalf
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* This type represents mappings for tokens affected by split edit operations.
|
||||
*/
|
||||
export interface TokenSplitMap {
|
||||
export interface TokenSplitMapping {
|
||||
/**
|
||||
* This entry represents the source-context token that was split as part of
|
||||
* the split edit operation.
|
||||
|
|
@ -78,7 +90,7 @@ export interface TokenSplitMap {
|
|||
* Note that it is possible for codepoints to exist here but not be
|
||||
* represented in the post-transition context tokens resulting from the split.
|
||||
*/
|
||||
input: EditTokenMap,
|
||||
input: EditTokenMappingHalf,
|
||||
|
||||
/**
|
||||
* Entries here represent post-transition tokens that represent pieces of the
|
||||
|
|
@ -87,7 +99,7 @@ export interface TokenSplitMap {
|
|||
* Entries will appear in the same ordering as the codepoints they represent
|
||||
* appear in the underlying context.
|
||||
*/
|
||||
matches: (EditTokenMap & { textOffset: number })[]
|
||||
matches: SplitTokenMappingHalf[]
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
@ -100,11 +112,11 @@ export interface TransitionEdgeAlignment {
|
|||
/**
|
||||
* Denotes any token merge edits needed after applying the Transform.
|
||||
*/
|
||||
merges: TokenMergeMap[];
|
||||
merges: TokenMergeMapping[];
|
||||
/**
|
||||
* Denotes any token split edits needed after applying the Transform.
|
||||
*/
|
||||
splits: TokenSplitMap[];
|
||||
splits: TokenSplitMapping[];
|
||||
/**
|
||||
* Denotes any further token edits needed that cannot be attributed to
|
||||
* 'merge's, 'split's, or edits from the input `Transform`.
|
||||
|
|
@ -431,8 +443,8 @@ export class ContextTokenization {
|
|||
* @param transitionEdge Batched results from one or more
|
||||
* `precomputeTokenizationAfterInput` calls on this instance, all with the
|
||||
* same alignment values.
|
||||
* @param transitionId The id of the Transform associated with the keystroke
|
||||
* triggering the transition.
|
||||
* @param transitionId The id of the context transition associated with the
|
||||
* Transition
|
||||
* @param bestProbFromSet The probability of the single most likely input
|
||||
* transform in the overall transformDistribution associated with the
|
||||
* keystroke triggering the transition. It need not be represented by the
|
||||
|
|
@ -479,9 +491,6 @@ export class ContextTokenization {
|
|||
tailTokenization.splice(tokenIndex, 1, affectedToken);
|
||||
}
|
||||
|
||||
affectedToken.isPartial = true;
|
||||
delete affectedToken.appliedTransitionId;
|
||||
|
||||
// If we are completely replacing a token via delete left, erase the deleteLeft;
|
||||
// that part applied to a _previous_ token that no longer exists.
|
||||
// We start at index 0 in the insert string for the "new" token.
|
||||
|
|
@ -507,6 +516,11 @@ export class ContextTokenization {
|
|||
affectedToken.isPartial
|
||||
);
|
||||
|
||||
// Do not adjust the original token, as it may be used by other transitions.
|
||||
// Only adjust the new, extended token.
|
||||
affectedToken.isPartial = true;
|
||||
delete affectedToken.appliedTransitionId;
|
||||
|
||||
const tokenize = determineModelTokenizer(lexicalModel);
|
||||
affectedToken.isWhitespace = tokenize({left: affectedToken.exampleInput, startOfBuffer: false, endOfBuffer: false}).left[0]?.isWhitespace ?? false;
|
||||
// Do not re-use the previous token; the mutation may have unexpected
|
||||
|
|
@ -516,7 +530,21 @@ export class ContextTokenization {
|
|||
affectedToken = null;
|
||||
}
|
||||
|
||||
return new ContextTokenization(this.tokens.slice(0, sliceIndex).concat(tailTokenization));
|
||||
// Backspace handling - emptying context via backspace or erasing _part_ of
|
||||
// a whitespace token can erase the tokenization-final empty token usually
|
||||
// used for word-initial suggestions.
|
||||
//
|
||||
// We re-add it here so that suggestions can be presented to the user as
|
||||
// normal.
|
||||
const tokenSequence = this.tokens.slice(0, sliceIndex).concat(tailTokenization);
|
||||
if(tokenSequence.length == 0 || tokenSequence[tokenSequence.length - 1]?.isWhitespace) {
|
||||
tokenSequence.push(new ContextToken(new LegacyQuotientRoot(lexicalModel)));
|
||||
}
|
||||
|
||||
return new ContextTokenization(
|
||||
tokenSequence,
|
||||
null
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1030,13 +1058,13 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo
|
|||
* generality, if two separate groups of tokens are merged, two groups will be
|
||||
* defined - one for each token resulting from a merge.
|
||||
*/
|
||||
merges: TokenMergeMap[],
|
||||
merges: TokenMergeMapping[],
|
||||
/**
|
||||
* Indicates groupings of directly related splits. Without loss of
|
||||
* generality, if two separate tokens are split, two groups will be defined -
|
||||
* one for each source token split.
|
||||
*/
|
||||
splits: TokenSplitMap[]
|
||||
splits: TokenSplitMapping[]
|
||||
} {
|
||||
// We've found the root token to which changes may apply.
|
||||
// We've found the last post-application token to which transform changes contributed.
|
||||
|
|
@ -1074,8 +1102,8 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo
|
|||
const mappedPath: EditTuple<EditOperation>[] = [];
|
||||
let mergeOffset = 0;
|
||||
let splitOffset = 0;
|
||||
const merges: TokenMergeMap[] = [];
|
||||
const splits: TokenSplitMap[] = [];
|
||||
const merges: TokenMergeMapping[] = [];
|
||||
const splits: TokenSplitMapping[] = [];
|
||||
while(queueIndex < editPath.length) {
|
||||
const edit = editPath[queueIndex];
|
||||
const { input, match } = edit;
|
||||
|
|
@ -1085,7 +1113,7 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo
|
|||
let matchOffset: number = 0;
|
||||
if(op == 'merge') {
|
||||
const mergeTarget = resultTokenization[match];
|
||||
const merge: TokenMergeMap = {
|
||||
const merge: TokenMergeMapping = {
|
||||
match: {
|
||||
index: match,
|
||||
text: mergeTarget
|
||||
|
|
@ -1116,7 +1144,7 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo
|
|||
merges.push(merge);
|
||||
} else if(op == 'split') {
|
||||
const splitTarget = preTokenization[input];
|
||||
const split: TokenSplitMap = {
|
||||
const split: TokenSplitMapping = {
|
||||
input: {
|
||||
index: input,
|
||||
text: splitTarget
|
||||
|
|
|
|||
|
|
@ -19,6 +19,15 @@ import Reversion = LexicalModelTypes.Reversion;
|
|||
import Suggestion = LexicalModelTypes.Suggestion;
|
||||
import Transform = LexicalModelTypes.Transform;
|
||||
|
||||
export interface TransitionReversionView extends Pick<ContextTransition, 'reversion'> {
|
||||
/**
|
||||
* Gets the context state resulting from the context transition event,
|
||||
* including any generated suggestions and data regarding potential
|
||||
* application thereof.
|
||||
*/
|
||||
final: Pick<ContextState, 'suggestions'>
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents the transition between two context states as triggered
|
||||
* by input keystrokes or applied suggestions.
|
||||
|
|
@ -36,7 +45,7 @@ export class ContextTransition {
|
|||
*/
|
||||
inputDistribution?: Distribution<Transform>;
|
||||
|
||||
// The transform ID in play.
|
||||
// The transition ID in play.
|
||||
private _transitionId?: number;
|
||||
|
||||
/**
|
||||
|
|
@ -177,7 +186,7 @@ export class ContextTransition {
|
|||
// We won't try to partially revert a multi-word suggestion; reversions
|
||||
// are only supported at the end of the last word of the main suggestion
|
||||
// body and after any appended whitespace.
|
||||
resultingTokenization.tail.appliedTransitionId = suggestion.transformId;
|
||||
resultingTokenization.tail.appliedTransitionId = suggestion.transform.id;
|
||||
|
||||
const resultingState = new ContextState(
|
||||
applyTransform(transformToApply, baseState.context),
|
||||
|
|
@ -189,12 +198,12 @@ export class ContextTransition {
|
|||
resultingState.appliedSuggestionId = suggestion.id;
|
||||
resultingState.suggestions = this.final.suggestions;
|
||||
|
||||
// Use the transform's ID for the transition. Note that when applying the
|
||||
// Use the transition ID tracked on the Transform. Note that when applying the
|
||||
// `appendedTransform` component of a suggestion, this will differ from
|
||||
// suggestion.transformId.
|
||||
// suggestion.transitionId.
|
||||
const resultingTransition = new ContextTransition(baseState, transformToApply.id);
|
||||
resultingTransition.finalize(resultingState, inputDistribution);
|
||||
resultingTransition.revertableTransitionId = suggestion.transformId;
|
||||
resultingTransition.revertableTransitionId = suggestion.transform.id;
|
||||
// .finalize unsets _.transitionId; re-assign it.
|
||||
resultingTransition._transitionId = transformToApply.id;
|
||||
|
||||
|
|
@ -236,7 +245,7 @@ export class ContextTransition {
|
|||
const baseTokenizationLength = results.transition.final.displayTokenization.tokens.length;
|
||||
const appliedTokenization = appendingTransition.final.displayTokenization;
|
||||
for(let i = baseTokenizationLength; i < appliedTokenization.tokens.length; i++) {
|
||||
appliedTokenization.tokens[i].appliedTransitionId = suggestion.transformId;
|
||||
appliedTokenization.tokens[i].appliedTransitionId = suggestion.transform.id;
|
||||
}
|
||||
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -135,7 +135,8 @@ export function legacySubsetKeyer(tokenizationEdits: TokenizationTransitionEdits
|
|||
// Now, based on the transform tokenization. We want to force uniqueness for
|
||||
// all variations of result length on each tokenized transform resulting from
|
||||
// the precomputation's represented keystroke.
|
||||
for(const {0: relativeIndex} of tokenizedTransform.entries()) {
|
||||
for(const {0: relativeIndex, 1: transform} of tokenizedTransform.entries()) {
|
||||
const insertLen = KMWString.length(transform.insert);
|
||||
if(relativeIndex > 0) {
|
||||
// The true boundary lie before the insert if the value is non-zero;
|
||||
// don't differentiate here!
|
||||
|
|
@ -148,10 +149,10 @@ export function legacySubsetKeyer(tokenizationEdits: TokenizationTransitionEdits
|
|||
//
|
||||
// IMPORTANT: update unit tests manually if the BI marker here changes
|
||||
// or the use of SENTINEL_CODE_UNIT as a key component separator changes.
|
||||
components.push(`BI@${relativeIndex}`);
|
||||
components.push(`BI@${relativeIndex}-${boundaryTextLen + insertLen}`);
|
||||
boundaryTextLen = 0;
|
||||
} else {
|
||||
components.push(`I@${relativeIndex}`);
|
||||
components.push(`I@${relativeIndex}-${boundaryTextLen + insertLen}`);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,19 @@
|
|||
import * as models from '@keymanapp/models-templates';
|
||||
import { LexicalModelTypes } from '@keymanapp/common-types';
|
||||
|
||||
import { applySuggestionCasing, composeIntermediatePredictions, correctAndEnumerate, createDefaultKeep, dedupeSuggestions, finalizeSuggestions, predictionAutoSelect, processSimilarity, toAnnotatedSuggestion, tupleDisplayOrderSort } from './predict-helpers.js';
|
||||
import {
|
||||
applySuggestionCasing,
|
||||
composeIntermediatePredictions,
|
||||
correctAndEnumerate,
|
||||
createDefaultKeep,
|
||||
dedupeSuggestions,
|
||||
finalizeSuggestions,
|
||||
predictionAutoSelect,
|
||||
prependReversion,
|
||||
processSimilarity,
|
||||
toAnnotatedSuggestion,
|
||||
tupleDisplayOrderSort
|
||||
} from './predict-helpers.js';
|
||||
import { determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js';
|
||||
|
||||
import { ContextTracker } from './correction/context-tracker.js';
|
||||
|
|
@ -79,12 +91,12 @@ export class ModelCompositor {
|
|||
this.configuration = config;
|
||||
}
|
||||
|
||||
initContextTracker(context: Context, transformId: number) {
|
||||
initContextTracker(context: Context, transitionId: number) {
|
||||
if(this.contextTracker || !this.lexicalModel.traverseFromRoot) {
|
||||
return;
|
||||
}
|
||||
|
||||
this._contextTracker = new ContextTracker(this.lexicalModel, context, transformId, this.configuration);
|
||||
this._contextTracker = new ContextTracker(this.lexicalModel, context, transitionId, this.configuration);
|
||||
}
|
||||
|
||||
async predict(transformDistribution: Transform | Distribution<Transform>, context: Context): Promise<Outcome<Suggestion|Keep>[]> {
|
||||
|
|
@ -120,8 +132,8 @@ export class ModelCompositor {
|
|||
// Only allow new-word suggestions if space was the most likely keypress.
|
||||
// const allowSpace = TransformUtils.isWhitespace(inputTransform);
|
||||
const inputTransform = transformDistribution[0].sample;
|
||||
const transformId = inputTransform.id;
|
||||
this.initContextTracker(context, transformId);
|
||||
const transitionId = inputTransform.id;
|
||||
this.initContextTracker(context, transitionId);
|
||||
|
||||
// Section 1: determine 'prediction roots' - enumerate corrections from most to least likely,
|
||||
// searching for results that yield viable predictions from the model.
|
||||
|
|
@ -190,18 +202,8 @@ export class ModelCompositor {
|
|||
this.SUGGESTION_ID_SEED++;
|
||||
});
|
||||
|
||||
if(revertableTransitionId) {
|
||||
const reversion = this.contextTracker.peek(revertableTransitionId)?.reversion;
|
||||
if(reversion) {
|
||||
if(suggestions[0]?.tag == 'keep') {
|
||||
const keep = suggestions.shift();
|
||||
suggestions.unshift(reversion);
|
||||
suggestions.unshift(keep);
|
||||
} else {
|
||||
suggestions.unshift(reversion)
|
||||
}
|
||||
}
|
||||
}
|
||||
const transitionToRevert = this.contextTracker?.peek(revertableTransitionId);
|
||||
prependReversion(suggestions, transitionToRevert);
|
||||
|
||||
if(suggestions.filter((s) => s.tag == 'keep').length > 1) {
|
||||
throw new Error(`Unexpected state: multiple keep suggestions exist: ${JSON.stringify(suggestions.filter((s) => s.tag == 'keep'))}`);
|
||||
|
|
@ -229,7 +231,9 @@ export class ModelCompositor {
|
|||
// Step 1: re-use the original input Transform as the reversion's Transform.
|
||||
// The Web engine will restore the original state of the context before accepting
|
||||
// and before reverting; all we need to do is put the original keystroke back in place.
|
||||
let reversionTransform: Transform = originalInput ?? { insert: '', deleteLeft: 0 };
|
||||
let reversionTransform: Transform = originalInput
|
||||
? { ...originalInput }
|
||||
: { insert: '', deleteLeft: 0, id: suggestion.transform.id };
|
||||
|
||||
// Step 2: building the proper 'displayAs' string for the Reversion
|
||||
const postContext = originalInput ? models.applyTransform(originalInput, context) : context;
|
||||
|
|
@ -253,9 +257,6 @@ export class ModelCompositor {
|
|||
// Since we're outside of the standard `predict` control path, we'll need to
|
||||
// set the Reversion's ID directly.
|
||||
let reversion = toAnnotatedSuggestion(this.lexicalModel, firstConversion, 'revert');
|
||||
if(suggestion.transformId != null) {
|
||||
reversion.transformId = -suggestion.transformId;
|
||||
}
|
||||
if(suggestion.id != null) {
|
||||
// Since a reversion inverts its source suggestion, we set its ID to be the
|
||||
// additive inverse of the source suggestion's ID. Makes easy mapping /
|
||||
|
|
@ -277,11 +278,11 @@ export class ModelCompositor {
|
|||
}
|
||||
} else {
|
||||
let originalTransition = this.contextTracker.latest;
|
||||
if(originalTransition.transitionId != suggestion.transformId) {
|
||||
originalTransition = this.contextTracker.findAndRevert(suggestion.transformId);
|
||||
if(originalTransition.transitionId != suggestion.transform.id) {
|
||||
originalTransition = this.contextTracker.findAndRevert(suggestion.transform.id);
|
||||
}
|
||||
if(!originalTransition) {
|
||||
this.contextTracker.reset(context, suggestion.transformId);
|
||||
this.contextTracker.reset(context, suggestion.transform.id);
|
||||
originalTransition = this.contextTracker.latest;
|
||||
}
|
||||
|
||||
|
|
@ -319,11 +320,8 @@ export class ModelCompositor {
|
|||
let compositor = this;
|
||||
let suggestions: Promise<Suggestion[]>;
|
||||
let fallbackSuggestions = async function() {
|
||||
const suggestions = await compositor.predict({insert: '', deleteLeft: 0}, context);
|
||||
const suggestions = await compositor.predict({insert: '', deleteLeft: 0, id: reversion.transform.id}, context);
|
||||
suggestions.forEach(function(suggestion) {
|
||||
// A reversion's transform ID is the additive inverse of its original suggestion;
|
||||
// we revert to the state of said original suggestion.
|
||||
suggestion.transformId = -reversion.transformId;
|
||||
// Prevent auto-selection of any suggestion immediately after a reversion.
|
||||
// It's fine after at least one keystroke, but not before.
|
||||
suggestion.autoAccept = false;
|
||||
|
|
@ -337,18 +335,19 @@ export class ModelCompositor {
|
|||
}
|
||||
|
||||
// When the context is tracked, we prefer the tracked information.
|
||||
// Note that the base reversion's .transformId will predate the appendedTransform id
|
||||
// Note that the base reversion's .transform.id will predate the appendedTransform id
|
||||
// used to add whitespace (if one existed), so reverting to the base ID's associated
|
||||
// context also reverts the appendedTransform.
|
||||
let originalTransition = this.contextTracker.findAndRevert(-reversion.transformId);
|
||||
const baseTransitionId = reversion.transform.id;
|
||||
let originalTransition = this.contextTracker.findAndRevert(baseTransitionId);
|
||||
|
||||
if(appendedOnly) {
|
||||
this.contextTracker.latest = originalTransition;
|
||||
return Promise.resolve([]);
|
||||
}
|
||||
|
||||
if(!originalTransition) {
|
||||
this.contextTracker.reset(context, -reversion.transformId);
|
||||
if(!originalTransition || originalTransition.transitionId != baseTransitionId) {
|
||||
this.contextTracker.reset(context, baseTransitionId);
|
||||
originalTransition = this.contextTracker.latest;
|
||||
|
||||
suggestions = fallbackSuggestions();
|
||||
|
|
|
|||
|
|
@ -98,12 +98,11 @@ export class DummyModel implements LexicalModel {
|
|||
predict(transform: Transform, context: Context, injectedSuggestions?: Outcome<Suggestion>[]): Distribution<Suggestion> {
|
||||
let makeUniformDistribution = function(suggestions: Outcome<Suggestion>[]): Distribution<Suggestion> {
|
||||
let distribution: Distribution<Suggestion> = [];
|
||||
const transitionId = transform.id;
|
||||
|
||||
for(let s of suggestions) {
|
||||
const transitionId = transform.id;
|
||||
if(transitionId !== undefined) {
|
||||
// Set the transform ID to match the incoming transform ID if one exists.
|
||||
s.transformId = transitionId;
|
||||
if(s.transform) {
|
||||
s.transform.id = transitionId;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import { ContextToken, ContextTokenLike } from './correction/context-token.js';
|
|||
import { ContextTokenization } from './correction/context-tokenization.js';
|
||||
import { ContextTracker } from './correction/context-tracker.js';
|
||||
import { ContextState, determineContextSlideTransform } from './correction/context-state.js';
|
||||
import { ContextTransition } from './correction/context-transition.js';
|
||||
import { ContextTransition, TransitionReversionView } from './correction/context-transition.js';
|
||||
import { ExecutionTimer } from './correction/execution-timer.js';
|
||||
import { ModelCompositor } from './model-compositor.js';
|
||||
import { EDIT_DISTANCE_COST_SCALE, getBestTokenMatches } from './correction/distance-modeler.js';
|
||||
|
|
@ -764,11 +764,6 @@ export async function correctAndEnumerate(
|
|||
// Corrections obtained: now to predict from them!
|
||||
const tokenization = tokenizations.find(t => t.spaceId == match.spaceId);
|
||||
|
||||
// If our 'match' results in fully deleting the new token, reject it and try again.
|
||||
if(match.matchSequence.length == 0 && match.inputSequence.length != 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// If our 'match' fully replaces the token, reject it and try again.
|
||||
if(match.matchSequence.length != 0 && match.matchSequence.length == match.knownCost) {
|
||||
continue;
|
||||
|
|
@ -896,7 +891,6 @@ export function predictFromCorrectionSequence(
|
|||
|
||||
entry.sample.transform.deleteLeft = correctionTransform.deleteLeft;
|
||||
if(transitionId !== undefined) {
|
||||
entry.sample.transformId = transitionId;
|
||||
entry.sample.transform.id = transitionId;
|
||||
}
|
||||
});
|
||||
|
|
@ -1020,7 +1014,7 @@ export function composeIntermediatePredictions(predictions: TokenizedIntermediat
|
|||
insert: '',
|
||||
deleteLeft: 0
|
||||
}
|
||||
const transformId = predictionData.components[0].prediction.transformId;
|
||||
const transformId = predictionData.components[0].prediction.transform.id;
|
||||
if(transformId !== undefined) {
|
||||
reduceBaseTransform.id = transformId;
|
||||
}
|
||||
|
|
@ -1214,7 +1208,6 @@ export function createDefaultKeep(
|
|||
|
||||
let keepOption = toAnnotatedSuggestion(lexicalModel, keepSuggestion, 'keep');
|
||||
if(inputTransform.id !== undefined) {
|
||||
keepOption.transformId = inputTransform.id;
|
||||
keepOption.transform.id = inputTransform.id;
|
||||
}
|
||||
keepOption.matchesModel = false;
|
||||
|
|
@ -1273,9 +1266,11 @@ export function predictionAutoSelect(suggestionDistribution: CompositedIntermedi
|
|||
|
||||
const keepOption = suggestionDistribution[0].components.prediction as Outcome<Keep>;
|
||||
if(keepOption.tag == 'keep' && keepOption.matchesModel) {
|
||||
// Auto-select it for auto-acceptance; we don't correct away from perfectly-valid
|
||||
// lexical entries, even if they are comparatively low-frequency.
|
||||
keepOption.autoAccept = true;
|
||||
// Do not auto-select 'keep' suggestions'; there's no need to apply them.
|
||||
//
|
||||
// Do, however, block auto-selection of any other suggestions if we would
|
||||
// have auto-selected the 'keep'; even if it is comparatively unlikely /
|
||||
// low-frequency, we 'keep' the current context.
|
||||
return;
|
||||
} else if(suggestionDistribution.length == 1) {
|
||||
return;
|
||||
|
|
@ -1370,12 +1365,6 @@ export function finalizeSuggestions(
|
|||
|
||||
const suggestions = deduplicatedSuggestionTuples.map((tuple) => {
|
||||
const prediction = tuple.components.prediction;
|
||||
|
||||
// Is sometimes not set during unit tests.
|
||||
if(prediction.transformId !== undefined) {
|
||||
prediction.transform.id = prediction.transformId;
|
||||
}
|
||||
|
||||
const probs = tuple.metadata.probabilities;
|
||||
|
||||
if(!verbose) {
|
||||
|
|
@ -1478,9 +1467,40 @@ export function toAnnotatedSuggestion(
|
|||
result.appendedTransform = suggestion.appendedTransform;
|
||||
}
|
||||
|
||||
if(suggestion.transformId !== undefined) {
|
||||
result.transformId = suggestion.transformId;
|
||||
if(suggestion.transform.id !== undefined) {
|
||||
result.transform.id = suggestion.transform.id;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* For applicable scenarios, this mutates the passed-in suggestion array by
|
||||
* prepending a predictive-text reversion that restores the context to a prior
|
||||
* state. Otherwise, it leaves the suggestion array unaltered.
|
||||
* @param suggestions
|
||||
* @param transitionToRevert
|
||||
* @returns
|
||||
*/
|
||||
export function prependReversion(suggestions: Suggestion[], transitionToRevert: TransitionReversionView) {
|
||||
if(transitionToRevert) {
|
||||
const reversion = transitionToRevert.reversion;
|
||||
if(reversion) {
|
||||
if(suggestions[0]?.tag == 'keep') {
|
||||
const appliedId = -reversion.id;
|
||||
const appliedSuggestion = transitionToRevert.final.suggestions.find((s) => s.id == appliedId);
|
||||
// If the selected suggestion was itself a `keep`, we don't need a
|
||||
// reversion. They'd do the same thing.
|
||||
if(appliedSuggestion.tag != 'keep') {
|
||||
const keep = suggestions.shift();
|
||||
suggestions.unshift(reversion);
|
||||
suggestions.unshift(keep);
|
||||
}
|
||||
} else {
|
||||
suggestions.unshift(reversion);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return suggestions;
|
||||
}
|
||||
|
|
@ -3,7 +3,7 @@ export * from './correction/context-state.js';
|
|||
export * from './correction/context-token.js';
|
||||
export * from './correction/context-tokenization.js';
|
||||
export { ContextTracker } from './correction/context-tracker.js';
|
||||
export { ContextTransition } from './correction/context-transition.js';
|
||||
export * from './correction/context-transition.js';
|
||||
export * from './correction/correction-searchable.js';
|
||||
export * from './correction/correction-result-mapping.js';
|
||||
export * from './correction/distance-modeler.js';
|
||||
|
|
|
|||
|
|
@ -266,14 +266,14 @@ export class PredictionContext extends EventEmitter<PredictionContextEventMap> {
|
|||
this._revertSuggestion = s as Reversion;
|
||||
}
|
||||
|
||||
if (this.langProcessor.mayAutoCorrect && s.autoAccept && !this.selected) {
|
||||
if (this.langProcessor.mayAutoCorrect && s.autoAccept && !this.selected && s.tag != 'keep') {
|
||||
this.selected = s;
|
||||
}
|
||||
}
|
||||
|
||||
// Verify that the transition IDs are still valid and remove special entries.
|
||||
this._currentSuggestions = suggestions.filter(s => {
|
||||
return this.langProcessor.hasState(Math.abs(s.transformId)) &&
|
||||
return this.langProcessor.hasState(Math.abs(s.transform.id)) &&
|
||||
s != this._keepSuggestion &&
|
||||
s != this._revertSuggestion
|
||||
});
|
||||
|
|
|
|||
|
|
@ -386,7 +386,7 @@ export class InputProcessor {
|
|||
// If so, since it behaves the same in either case, and it's a known word-breaking mark,
|
||||
// let's apply the selected suggestion automatically.
|
||||
if(postApplyTransform.insert == ruleTransform.insert && transformMatchesPattern(postApplyTransform, breakingMarks)) {
|
||||
const baseTransition = this.contextCache.get(predictionContext.selected.transformId);
|
||||
const baseTransition = this.contextCache.get(predictionContext.selected.transform.id);
|
||||
|
||||
// Somehow, the base state is out of context - abort!
|
||||
if(!baseTransition) {
|
||||
|
|
|
|||
|
|
@ -166,6 +166,12 @@ export class LanguageProcessor extends EventEmitter<LanguageProcessorEventMap> {
|
|||
return !!this.recentTranscriptions.get(transitionId);
|
||||
}
|
||||
|
||||
/**
|
||||
* When called, this requests the worker to generate predictions for the transcribed
|
||||
* context transition.
|
||||
* @param transcription
|
||||
* @param layerId
|
||||
*/
|
||||
public predict(transcription: Transcription, layerId: string): Promise<Suggestion[]> {
|
||||
if(!this.isActive) {
|
||||
return null;
|
||||
|
|
@ -218,13 +224,13 @@ export class LanguageProcessor extends EventEmitter<LanguageProcessorEventMap> {
|
|||
// Find the state of the context at the time the suggestion was generated.
|
||||
// This may refer to the context before an input keystroke or before application
|
||||
// of a predictive suggestion.
|
||||
const original = this.getPredictionState(suggestion.transformId);
|
||||
const original = this.getPredictionState(suggestion.transform.id);
|
||||
if(!original) {
|
||||
console.warn("Could not apply the Suggestion!");
|
||||
return null;
|
||||
}
|
||||
|
||||
this.recentTranscriptions.rewindTo(suggestion.transformId);
|
||||
this.recentTranscriptions.rewindTo(suggestion.transform.id);
|
||||
|
||||
// Apply the Suggestion!
|
||||
|
||||
|
|
@ -306,10 +312,7 @@ export class LanguageProcessor extends EventEmitter<LanguageProcessorEventMap> {
|
|||
// Find the state of the context at the time the suggestion was generated.
|
||||
// This may refer to the context before an input keystroke or before application
|
||||
// of a predictive suggestion.
|
||||
//
|
||||
// Reversions use the additive inverse of the id token of the Transcription being
|
||||
// reverted to.
|
||||
const reversionId = appendedOnly ? reversion.appendedTransform.id : -reversion.transformId;
|
||||
const reversionId = appendedOnly ? reversion.appendedTransform.id : reversion.transform.id;
|
||||
const original = this.getPredictionState(reversionId);
|
||||
if(!original) {
|
||||
console.warn("Could not apply the Suggestion!");
|
||||
|
|
@ -373,6 +376,9 @@ export class LanguageProcessor extends EventEmitter<LanguageProcessorEventMap> {
|
|||
this.lmEngine.resetContext(context, transcription.token);
|
||||
}
|
||||
|
||||
// The "may correct" setting is enforced here, in the main Web engine - not
|
||||
// in the worker. As the desired setting may vary language-to-language,
|
||||
// we'd rather do this than reconfigure the worker constantly.
|
||||
let alternates = transcription.alternates;
|
||||
if(!this.mayCorrect || !alternates || alternates.length == 0) {
|
||||
alternates = [{
|
||||
|
|
@ -386,6 +392,12 @@ export class LanguageProcessor extends EventEmitter<LanguageProcessorEventMap> {
|
|||
|
||||
return promise.then((suggestions: Suggestion[]) => {
|
||||
if(promise == this.currentPromise) {
|
||||
if(!this.mayAutoCorrect) {
|
||||
// We disable the auto-accept flag here, rather than in the worker -
|
||||
// that way, we don't need to reconfigure the worker when settings
|
||||
// change.
|
||||
suggestions.forEach((s) => delete s.autoAccept);
|
||||
}
|
||||
const result = new ReadySuggestions(suggestions, transform.id);
|
||||
this.emit("suggestionsready", result);
|
||||
this.currentPromise = null;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { Transcription } from "keyman/engine/keyboard";
|
||||
import { RewindableCache } from "keyman/common/web-utils";
|
||||
|
||||
const TRANSCRIPTION_BUFFER_SIZE = 10;
|
||||
const TRANSCRIPTION_BUFFER_SIZE = 20;
|
||||
|
||||
export class TranscriptionCache extends RewindableCache<Transcription> {
|
||||
constructor() {
|
||||
|
|
|
|||
|
|
@ -283,9 +283,8 @@ describe('InputProcessor', function() {
|
|||
[],
|
||||
[
|
||||
{
|
||||
transform: { insert: 'testing', deleteLeft: 4 },
|
||||
transform: { insert: 'testing', deleteLeft: 4, id: 0 },
|
||||
appendedTransform: { insert: ' ', deleteLeft: 0 },
|
||||
transformId: 0, // will be overwritten by the DummyModel to match the transition ID.
|
||||
id: 1,
|
||||
displayAs: 'testing'
|
||||
}
|
||||
|
|
|
|||
|
|
@ -191,7 +191,6 @@ describe('Common utility functions', function() {
|
|||
deleteLeft: 0,
|
||||
id: 0
|
||||
},
|
||||
transformId: 0,
|
||||
displayAs: 'hello'
|
||||
};
|
||||
|
||||
|
|
@ -205,7 +204,6 @@ describe('Common utility functions', function() {
|
|||
deleteLeft: 0,
|
||||
id: 0
|
||||
},
|
||||
transformId: 0,
|
||||
displayAs: 'hello',
|
||||
p: 0
|
||||
};
|
||||
|
|
@ -220,7 +218,6 @@ describe('Common utility functions', function() {
|
|||
deleteLeft: 0,
|
||||
id: 0
|
||||
},
|
||||
transformId: 0,
|
||||
displayAs: 'hello',
|
||||
p: 0.5
|
||||
};
|
||||
|
|
@ -228,14 +225,13 @@ describe('Common utility functions', function() {
|
|||
assert.deepEqual(models.transformToSuggestion(suggestion.transform, 0.5), suggestion);
|
||||
});
|
||||
|
||||
it('properly handles the transformId', function() {
|
||||
it('properly handles the transition ID', function() {
|
||||
let suggestion = {
|
||||
transform: {
|
||||
insert: 'hello',
|
||||
deleteLeft: 0,
|
||||
id: 3
|
||||
},
|
||||
transformId: 3, // Ensures there isn't a separate ID seed in use.
|
||||
displayAs: 'hello'
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -28,7 +28,8 @@ import {
|
|||
models,
|
||||
SearchQuotientSpur,
|
||||
traceInsertEdits,
|
||||
LegacyQuotientSpur
|
||||
LegacyQuotientSpur,
|
||||
TransitionEdge
|
||||
} from '@keymanapp/lm-worker/test-index';
|
||||
|
||||
import Transform = LexicalModelTypes.Transform;
|
||||
|
|
@ -45,8 +46,8 @@ function toToken(text: string) {
|
|||
}
|
||||
|
||||
let TOKEN_TRANSFORM_SEED = 0;
|
||||
function toTransformToken(text: string, transformId?: number) {
|
||||
let idSeed = transformId === undefined ? TOKEN_TRANSFORM_SEED++ : transformId;
|
||||
function toTransitionToken(text: string, transitionId?: number) {
|
||||
let idSeed = transitionId === undefined ? TOKEN_TRANSFORM_SEED++ : transitionId;
|
||||
let isWhitespace = text == ' ';
|
||||
let token = ContextToken.fromRawText(plainModel, '');
|
||||
const textAsTransform = { insert: text, deleteLeft: 0, id: idSeed };
|
||||
|
|
@ -105,7 +106,7 @@ describe('ContextTokenization', function() {
|
|||
|
||||
it("constructs from a token array + alignment data", () => {
|
||||
const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
|
||||
const tokens = rawTextTokens.map((text => toTransformToken(text)));
|
||||
const tokens = rawTextTokens.map((text => toTransitionToken(text)));
|
||||
|
||||
let tokenization = new ContextTokenization(tokens);
|
||||
|
||||
|
|
@ -117,8 +118,28 @@ describe('ContextTokenization', function() {
|
|||
|
||||
it('clones', () => {
|
||||
const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
|
||||
const tokens = rawTextTokens.map((text => toTransformToken(text)));
|
||||
let baseTokenization = new ContextTokenization(tokens);
|
||||
const tokens = rawTextTokens.map((text => toTransitionToken(text)));
|
||||
const emptyTransform = { insert: '', deleteLeft: 0, deleteRight: 0 };
|
||||
|
||||
// We _could_ flesh this out a bit more... but it's not really needed for this test.
|
||||
const edgeWindow = buildEdgeWindow(tokens, emptyTransform, false, testEdgeWindowSpec);
|
||||
let transitionEdits: TransitionEdge = {
|
||||
alignment: {
|
||||
merges: [],
|
||||
splits: [],
|
||||
unmappedEdits: [],
|
||||
edgeWindow: {...edgeWindow, retokenization: rawTextTokens.slice(edgeWindow.sliceIndex)},
|
||||
removedTokenCount: 0
|
||||
},
|
||||
inputs: [{sample: (() => {
|
||||
const map = new Map<number, Transform>();
|
||||
map.set(0, emptyTransform);
|
||||
return map;
|
||||
})(), p: 1}],
|
||||
inputSubsetId: generateSubsetId()
|
||||
};
|
||||
|
||||
let baseTokenization = new ContextTokenization(tokens, transitionEdits);
|
||||
let cloned = new ContextTokenization(baseTokenization);
|
||||
|
||||
assert.sameOrderedMembers(
|
||||
|
|
@ -353,6 +374,43 @@ describe('ContextTokenization', function() {
|
|||
);
|
||||
});
|
||||
|
||||
it('handles simple case - deletion of final context content via backspace', () => {
|
||||
const baseTokens = ['a'];
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t)));
|
||||
|
||||
const targetTokens = [''].map((t) => ({text: t, isWhitespace: t == ' '}));
|
||||
const inputTransform = { insert: '', deleteLeft: 1, deleteRight: 0, id: 42 };
|
||||
const inputTransformMap: Map<number, Transform> = new Map();
|
||||
inputTransformMap.set(0, { insert: '', deleteLeft: 1, id: 42 });
|
||||
|
||||
const edgeWindow = buildEdgeWindow(baseTokenization.tokens, inputTransform, false, testEdgeWindowSpec);
|
||||
const tokenization = baseTokenization.evaluateTransition({
|
||||
alignment: {
|
||||
merges: [],
|
||||
splits: [],
|
||||
unmappedEdits: [],
|
||||
edgeWindow: {
|
||||
...edgeWindow,
|
||||
// The range within the window constructed by the prior call for its parameterization.
|
||||
// Any adjustments on the boundary token itself are included here.
|
||||
retokenization: [...targetTokens.slice(edgeWindow.sliceIndex).map(t => t.text)]
|
||||
},
|
||||
removedTokenCount: 1
|
||||
},
|
||||
inputs: [{ sample: inputTransformMap, p: 1 }],
|
||||
inputSubsetId: generateSubsetId()
|
||||
},
|
||||
inputTransform.id,
|
||||
1
|
||||
);
|
||||
|
||||
assert.isOk(tokenization);
|
||||
assert.equal(tokenization.tokens.length, targetTokens.length);
|
||||
assert.deepEqual(tokenization.tokens.map((t) => ({text: t.exampleInput, isWhitespace: t.isWhitespace})),
|
||||
targetTokens
|
||||
);
|
||||
});
|
||||
|
||||
it('handles simple case - new character added to last token', () => {
|
||||
const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'da'];
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t)));
|
||||
|
|
@ -779,6 +837,198 @@ describe('ContextTokenization', function() {
|
|||
assert.equal(preTail.exampleInput, '\'');
|
||||
assert.equal(tail.exampleInput, '.');
|
||||
});
|
||||
|
||||
describe('properly handles previously-applied transition IDs', () => {
|
||||
it('does not preserve applied transition IDs on edited tokens', () => {
|
||||
const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day', ' ', 'can'];
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t)));
|
||||
|
||||
const REVERTABLE_TRANSITION_ID = 31415;
|
||||
baseTokenization.tail.appliedTransitionId = REVERTABLE_TRANSITION_ID;
|
||||
const NEW_TRANSITION_ID = REVERTABLE_TRANSITION_ID + 1;
|
||||
|
||||
const dist = [
|
||||
{
|
||||
sample: { insert: 't', deleteLeft: 0, deleteRight: 0, id: NEW_TRANSITION_ID },
|
||||
p: 1
|
||||
}
|
||||
];
|
||||
|
||||
const resultTokenization = baseTokenization.evaluateTransition(
|
||||
{
|
||||
alignment: {
|
||||
merges: [],
|
||||
splits: [],
|
||||
unmappedEdits: [],
|
||||
edgeWindow: {
|
||||
...buildEdgeWindow(baseTokenization.tokens, dist[0].sample, false),
|
||||
retokenization: baseTokenization.tokens.map((t) => t.exampleInput),
|
||||
retokenizationText: baseTokenization.tokens.map((t) => t.exampleInput).reduce((accum, curr) => accum + curr, '')
|
||||
},
|
||||
removedTokenCount: 0
|
||||
},
|
||||
inputs: (() => {
|
||||
const map: Map<number, Transform> = new Map();
|
||||
map.set(0, dist[0].sample);
|
||||
|
||||
return [
|
||||
{sample: map, p: 1}
|
||||
];
|
||||
})(),
|
||||
inputSubsetId: 0
|
||||
},
|
||||
NEW_TRANSITION_ID,
|
||||
1
|
||||
)
|
||||
|
||||
resultTokenization.tokens.forEach((t) => {
|
||||
assert.isUndefined(t.appliedTransitionId);
|
||||
});
|
||||
});
|
||||
|
||||
it('preserves applied transition IDs on applicable tokens', () => {
|
||||
const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day', ' ', 'can'];
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t)));
|
||||
|
||||
const REVERTABLE_TRANSITION_ID = 31415;
|
||||
baseTokenization.tail.appliedTransitionId = REVERTABLE_TRANSITION_ID;
|
||||
const NEW_TRANSITION_ID = REVERTABLE_TRANSITION_ID + 1;
|
||||
|
||||
|
||||
const dist = [
|
||||
{
|
||||
sample: { insert: ' ', deleteLeft: 0, deleteRight: 0, id: NEW_TRANSITION_ID },
|
||||
p: 1
|
||||
}
|
||||
];
|
||||
|
||||
const resultTokenization = baseTokenization.evaluateTransition(
|
||||
{
|
||||
alignment: {
|
||||
merges: [],
|
||||
splits: [],
|
||||
unmappedEdits: [],
|
||||
edgeWindow: {
|
||||
...buildEdgeWindow(baseTokenization.tokens, dist[0].sample, false),
|
||||
retokenization: baseTokenization.tokens.map((t) => t.exampleInput),
|
||||
retokenizationText: baseTokenization.tokens.map((t) => t.exampleInput).reduce((accum, curr) => accum + curr, '')
|
||||
},
|
||||
removedTokenCount: 0
|
||||
},
|
||||
inputs: (() => {
|
||||
const map: Map<number, Transform> = new Map();
|
||||
map.set(1, dist[0].sample);
|
||||
|
||||
return [
|
||||
{sample: map, p: 1}
|
||||
];
|
||||
})(),
|
||||
inputSubsetId: 0
|
||||
},
|
||||
NEW_TRANSITION_ID,
|
||||
1
|
||||
)
|
||||
|
||||
const resultTokenLength = resultTokenization.tokens.length;
|
||||
|
||||
resultTokenization.tokens.forEach((t, i) => {
|
||||
// The space will add TWO tokens.
|
||||
if(i == resultTokenLength - 3) {
|
||||
assert.equal(t.appliedTransitionId, REVERTABLE_TRANSITION_ID);
|
||||
} else {
|
||||
assert.isUndefined(t.appliedTransitionId);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Performs the two above in sequence in a manner that could cause cross-effects
|
||||
// if implemented incorrectly.
|
||||
it('does not conflate effects between different tokenization transitions', () => {
|
||||
const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day', ' ', 'can'];
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t)));
|
||||
|
||||
const REVERTABLE_TRANSITION_ID = 31415;
|
||||
baseTokenization.tail.appliedTransitionId = REVERTABLE_TRANSITION_ID;
|
||||
const NEW_TRANSITION_ID = REVERTABLE_TRANSITION_ID + 1;
|
||||
|
||||
const dist = [
|
||||
{
|
||||
sample: { insert: ' ', deleteLeft: 0, deleteRight: 0, id: NEW_TRANSITION_ID },
|
||||
p: .8
|
||||
}, {
|
||||
sample: { insert: ' ', deleteLeft: 0, deleteRight: 0, id: NEW_TRANSITION_ID },
|
||||
p: .2
|
||||
}
|
||||
];
|
||||
|
||||
const baseTransitionEdge: TransitionEdge = {
|
||||
alignment: {
|
||||
merges: [],
|
||||
splits: [],
|
||||
unmappedEdits: [],
|
||||
edgeWindow: {
|
||||
...buildEdgeWindow(baseTokenization.tokens, dist[0].sample, false),
|
||||
retokenization: baseTokenization.tokens.map((t) => t.exampleInput),
|
||||
retokenizationText: baseTokenization.tokens.map((t) => t.exampleInput).reduce((accum, curr) => accum + curr, '')
|
||||
},
|
||||
removedTokenCount: 0
|
||||
},
|
||||
inputs: (() => {
|
||||
const map: Map<number, Transform> = new Map();
|
||||
map.set(1, dist[0].sample);
|
||||
|
||||
return [
|
||||
{sample: map, p: dist[0].p}
|
||||
];
|
||||
})(),
|
||||
inputSubsetId: 0
|
||||
};
|
||||
|
||||
// We don't care about the results here. What we care about is that
|
||||
// this call doesn't remove the appliedTransitionId from the source
|
||||
// token, preventing it from being marked on later tokenization
|
||||
// transitions.
|
||||
baseTokenization.evaluateTransition(
|
||||
{
|
||||
alignment: {
|
||||
...baseTransitionEdge.alignment,
|
||||
edgeWindow: {
|
||||
...baseTransitionEdge.alignment.edgeWindow,
|
||||
...buildEdgeWindow(baseTokenization.tokens, dist[1].sample, false)
|
||||
}
|
||||
},
|
||||
inputs: (() => {
|
||||
const map: Map<number, Transform> = new Map();
|
||||
map.set(0, dist[1].sample);
|
||||
|
||||
return [
|
||||
{sample: map, p: dist[1].p}
|
||||
];
|
||||
})(),
|
||||
inputSubsetId: 0
|
||||
},
|
||||
NEW_TRANSITION_ID,
|
||||
dist[1].p
|
||||
)
|
||||
|
||||
const resultTokenization = baseTokenization.evaluateTransition(
|
||||
baseTransitionEdge,
|
||||
NEW_TRANSITION_ID,
|
||||
dist[0].p
|
||||
);
|
||||
|
||||
const resultTokenLength = resultTokenization.tokens.length;
|
||||
|
||||
resultTokenization.tokens.forEach((t, i) => {
|
||||
// The space will add TWO tokens.
|
||||
if(i == resultTokenLength - 3) {
|
||||
assert.equal(t.appliedTransitionId, REVERTABLE_TRANSITION_ID);
|
||||
} else {
|
||||
assert.isUndefined(t.appliedTransitionId);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildEdgeWindow', () => {
|
||||
|
|
@ -786,7 +1036,7 @@ describe('ContextTokenization', function() {
|
|||
it('handles empty contexts', () => {
|
||||
const baseTokens = [''];
|
||||
const idSeed = TOKEN_TRANSFORM_SEED;
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t)));
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t)));
|
||||
|
||||
const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 0 }, true, testEdgeWindowSpec);
|
||||
assert.deepEqual(results, {
|
||||
|
|
@ -806,7 +1056,7 @@ describe('ContextTokenization', function() {
|
|||
it('handles empty contexts and invalid Transforms', () => {
|
||||
const baseTokens = [''];
|
||||
const idSeed = TOKEN_TRANSFORM_SEED;
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t)));
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t)));
|
||||
|
||||
const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 2 }, true, testEdgeWindowSpec);
|
||||
assert.deepEqual(results, {
|
||||
|
|
@ -826,7 +1076,7 @@ describe('ContextTokenization', function() {
|
|||
it('builds edge windows for the start of context with no edits', () => {
|
||||
const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
|
||||
const idSeed = TOKEN_TRANSFORM_SEED;
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t)));
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t)));
|
||||
|
||||
const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 0 }, true, testEdgeWindowSpec);
|
||||
assert.deepEqual(results, {
|
||||
|
|
@ -866,7 +1116,7 @@ describe('ContextTokenization', function() {
|
|||
it('builds edge windows for the start of context with deletion edits (1)', () => {
|
||||
const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
|
||||
const idSeed = TOKEN_TRANSFORM_SEED;
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t)));
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t)));
|
||||
|
||||
const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 2 }, true, testEdgeWindowSpec);
|
||||
assert.deepEqual(results, {
|
||||
|
|
@ -906,7 +1156,7 @@ describe('ContextTokenization', function() {
|
|||
it('builds edge windows for the start of context with deletion edits (2)', () => {
|
||||
const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
|
||||
const idSeed = TOKEN_TRANSFORM_SEED;
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t)));
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t)));
|
||||
|
||||
const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 4 }, true, testEdgeWindowSpec);
|
||||
assert.deepEqual(results, {
|
||||
|
|
@ -926,7 +1176,7 @@ describe('ContextTokenization', function() {
|
|||
it('builds edge windows for the end of context with no edits', () => {
|
||||
const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
|
||||
const idSeed = TOKEN_TRANSFORM_SEED;
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t)));
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t)));
|
||||
baseTokenization.tail.isPartial = true;
|
||||
|
||||
const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 0 }, false, testEdgeWindowSpec);
|
||||
|
|
@ -947,7 +1197,7 @@ describe('ContextTokenization', function() {
|
|||
it('builds edge windows for the end of context with no edits, trailing whitespace', () => {
|
||||
const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day', ' ', ''];
|
||||
const idSeed = TOKEN_TRANSFORM_SEED;
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t)));
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t)));
|
||||
|
||||
const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 0 }, false, testEdgeWindowSpec);
|
||||
assert.deepEqual(results, {
|
||||
|
|
@ -1734,7 +1984,7 @@ describe('ContextTokenization', function() {
|
|||
it('returns the standard edge window for empty transform inputs', () => {
|
||||
const baseTokens = ['quick', ' ', 'brown', ' ', 'fox'];
|
||||
const idSeed = TOKEN_TRANSFORM_SEED;
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t)));
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t)));
|
||||
|
||||
const editTransform = {
|
||||
insert: '',
|
||||
|
|
@ -1768,7 +2018,7 @@ describe('ContextTokenization', function() {
|
|||
it('returns the standard edge window for empty transforms with context-final whitespace', () => {
|
||||
const baseTokens = ['quick', ' ', 'brown', ' ', 'fox', ' '];
|
||||
const idSeed = TOKEN_TRANSFORM_SEED;
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t)));
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t)));
|
||||
|
||||
const editTransform = {
|
||||
insert: '',
|
||||
|
|
@ -1803,7 +2053,7 @@ describe('ContextTokenization', function() {
|
|||
it('returns the standard edge window for pure transform w insert inputs', () => {
|
||||
const baseTokens = ['quick', ' ', 'brown', ' ', 'fox'];
|
||||
const idSeed = TOKEN_TRANSFORM_SEED;
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t)));
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t)));
|
||||
|
||||
const editTransform = {
|
||||
insert: ' jumped',
|
||||
|
|
@ -1837,7 +2087,7 @@ describe('ContextTokenization', function() {
|
|||
it('returns the proper edge window for transforms w deleteLeft inputs (1)', () => {
|
||||
const baseTokens = ['quick', ' ', 'brown', ' ', 'fox'];
|
||||
const idSeed = TOKEN_TRANSFORM_SEED;
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t)));
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t)));
|
||||
|
||||
const editTransform = {
|
||||
insert: 'rog',
|
||||
|
|
@ -1872,7 +2122,7 @@ describe('ContextTokenization', function() {
|
|||
it('returns the proper edge window for transforms w deleteLeft inputs (2)', () => {
|
||||
const baseTokens = ['quick', ' ', 'brown', ' ', 'fox'];
|
||||
const idSeed = TOKEN_TRANSFORM_SEED;
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t)));
|
||||
const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransitionToken(t)));
|
||||
|
||||
const editTransform = {
|
||||
insert: 'fox and brown fox', // => quick fox and brown fox
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ describe('ContextTracker', function() {
|
|||
deleteLeft: 0,
|
||||
id: 15
|
||||
},
|
||||
transformId: 2,
|
||||
transitionId: 2,
|
||||
id: 1,
|
||||
displayAs: 'world'
|
||||
};
|
||||
|
|
|
|||
|
|
@ -118,9 +118,8 @@ describe('ContextTransition', () => {
|
|||
appendedTransform: {
|
||||
insert: ' ',
|
||||
deleteLeft: 0,
|
||||
id: 2
|
||||
id: 3
|
||||
},
|
||||
transformId: 2,
|
||||
id: 10,
|
||||
displayAs: 'world'
|
||||
}, {
|
||||
|
|
@ -132,9 +131,8 @@ describe('ContextTransition', () => {
|
|||
appendedTransform: {
|
||||
insert: ' ',
|
||||
deleteLeft: 0,
|
||||
id: 2
|
||||
id: 3
|
||||
},
|
||||
transformId: 2,
|
||||
id: 11,
|
||||
displayAs: 'won'
|
||||
}];
|
||||
|
|
@ -155,7 +153,7 @@ describe('ContextTransition', () => {
|
|||
// 3 long, only last token was edited.
|
||||
appliedTransition.base.final.displayTokenization.tokens.forEach((token, index) => {
|
||||
if(index >= 2) {
|
||||
assert.equal(token.appliedTransitionId, suggestions[0].transformId);
|
||||
assert.equal(token.appliedTransitionId, suggestions[0].transform.id);
|
||||
} else {
|
||||
assert.isUndefined(token.appliedTransitionId);
|
||||
}
|
||||
|
|
@ -163,7 +161,7 @@ describe('ContextTransition', () => {
|
|||
|
||||
appliedTransition.appended.final.displayTokenization.tokens.forEach((token, index) => {
|
||||
if(index >= 2) {
|
||||
assert.equal(token.appliedTransitionId, suggestions[0].transformId);
|
||||
assert.equal(token.appliedTransitionId, suggestions[0].transform.id);
|
||||
} else {
|
||||
assert.isUndefined(token.appliedTransitionId);
|
||||
}
|
||||
|
|
@ -202,9 +200,8 @@ describe('ContextTransition', () => {
|
|||
appendedTransform: {
|
||||
insert: ' ',
|
||||
deleteLeft: 0,
|
||||
id: 2
|
||||
id: 3
|
||||
},
|
||||
transformId: 2,
|
||||
id: 10,
|
||||
displayAs: 'the'
|
||||
}, {
|
||||
|
|
@ -216,9 +213,8 @@ describe('ContextTransition', () => {
|
|||
appendedTransform: {
|
||||
insert: ' ',
|
||||
deleteLeft: 0,
|
||||
id: 2
|
||||
id: 3
|
||||
},
|
||||
transformId: 2,
|
||||
id: 11,
|
||||
displayAs: 'and'
|
||||
}];
|
||||
|
|
@ -239,7 +235,7 @@ describe('ContextTransition', () => {
|
|||
// 3 long, only last token was edited.
|
||||
appliedTransition.base.final.displayTokenization.tokens.forEach((token, index) => {
|
||||
if(index >= 4) {
|
||||
assert.equal(token.appliedTransitionId, suggestions[0].transformId);
|
||||
assert.equal(token.appliedTransitionId, suggestions[0].transform.id);
|
||||
} else {
|
||||
assert.isUndefined(token.appliedTransitionId);
|
||||
}
|
||||
|
|
@ -247,7 +243,7 @@ describe('ContextTransition', () => {
|
|||
|
||||
appliedTransition.appended.final.displayTokenization.tokens.forEach((token, index) => {
|
||||
if(index >= 4) {
|
||||
assert.equal(token.appliedTransitionId, suggestions[0].transformId);
|
||||
assert.equal(token.appliedTransitionId, suggestions[0].transform.id);
|
||||
} else {
|
||||
assert.isUndefined(token.appliedTransitionId);
|
||||
}
|
||||
|
|
@ -287,9 +283,8 @@ describe('ContextTransition', () => {
|
|||
appendedTransform: {
|
||||
insert: ' ',
|
||||
deleteLeft: 0,
|
||||
id: 2
|
||||
id: 3
|
||||
},
|
||||
transformId: 2,
|
||||
id: 10,
|
||||
displayAs: 'world'
|
||||
}, {
|
||||
|
|
@ -301,9 +296,8 @@ describe('ContextTransition', () => {
|
|||
appendedTransform: {
|
||||
insert: ' ',
|
||||
deleteLeft: 0,
|
||||
id: 2
|
||||
id: 3
|
||||
},
|
||||
transformId: 2,
|
||||
id: 11,
|
||||
displayAs: 'won'
|
||||
}];
|
||||
|
|
@ -337,9 +331,8 @@ describe('ContextTransition', () => {
|
|||
appendedTransform: {
|
||||
insert: ' ',
|
||||
deleteLeft: 0,
|
||||
id: 2
|
||||
id: 3
|
||||
},
|
||||
transformId: 2,
|
||||
id: 10,
|
||||
displayAs: 'world'
|
||||
}, {
|
||||
|
|
@ -351,9 +344,8 @@ describe('ContextTransition', () => {
|
|||
appendedTransform: {
|
||||
insert: ' ',
|
||||
deleteLeft: 0,
|
||||
id: 2
|
||||
id: 3
|
||||
},
|
||||
transformId: 2,
|
||||
id: 11,
|
||||
displayAs: 'won'
|
||||
}];
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import {
|
|||
ContextToken,
|
||||
ContextTokenization,
|
||||
LegacyQuotientSpur,
|
||||
legacySubsetKeyer,
|
||||
models,
|
||||
precomputationSubsetKeyer,
|
||||
TokenizationTransitionEdits,
|
||||
|
|
@ -31,7 +32,7 @@ import Distribution = LexicalModelTypes.Distribution;
|
|||
import Transform = LexicalModelTypes.Transform;
|
||||
import TrieModel = models.TrieModel;
|
||||
|
||||
var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'),
|
||||
const plainModel = new TrieModel(jsonFixture('models/tries/english-1000'),
|
||||
{wordBreaker: defaultBreaker});
|
||||
|
||||
function toToken(text: string) {
|
||||
|
|
@ -41,6 +42,54 @@ function toToken(text: string) {
|
|||
return token;
|
||||
}
|
||||
|
||||
describe('legacySubsetKeyer', () => {
|
||||
it('does not map backspace inputs to same result as standard key inputs', () => {
|
||||
const appleToken = ContextToken.fromRawText(plainModel, 'apple', true);
|
||||
|
||||
const bksp = { insert: '', deleteLeft: 1, deleteRight: 0, id: 3 };
|
||||
const bkspKey = legacySubsetKeyer({
|
||||
alignment: {
|
||||
merges: [],
|
||||
splits: [],
|
||||
unmappedEdits: [],
|
||||
edgeWindow: {
|
||||
...buildEdgeWindow([appleToken], bksp, false),
|
||||
retokenization: ['appl'],
|
||||
retokenizationText: 'appl'
|
||||
},
|
||||
removedTokenCount: 0
|
||||
},
|
||||
tokenizedTransform: (() => {
|
||||
const map: Map<number, Transform> = new Map();
|
||||
map.set(0, bksp);
|
||||
return map;
|
||||
})()
|
||||
});
|
||||
|
||||
const input = { insert: 's', deleteLeft: 0, deleteRight: 0, id: bksp.id };
|
||||
const inputKey = legacySubsetKeyer({
|
||||
alignment: {
|
||||
merges: [],
|
||||
splits: [],
|
||||
unmappedEdits: [],
|
||||
edgeWindow: {
|
||||
...buildEdgeWindow([appleToken], input, false),
|
||||
retokenization: ['apples'],
|
||||
retokenizationText: 'apples'
|
||||
},
|
||||
removedTokenCount: 0
|
||||
},
|
||||
tokenizedTransform: (() => {
|
||||
const map: Map<number, Transform> = new Map();
|
||||
map.set(0, input);
|
||||
return map;
|
||||
})()
|
||||
});
|
||||
|
||||
assert.notEqual(bkspKey, inputKey);
|
||||
});
|
||||
});
|
||||
|
||||
describe('precomputationSubsetKeyer', function() {
|
||||
it("safely generates keys for empty transition + empty contexts", () => {
|
||||
const rawTextTokens = [''];
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ describe('predictionAutoSelect', () => {
|
|||
assert.sameDeepOrderedMembers(predictions, originalPredictions);
|
||||
});
|
||||
|
||||
it(`selects solitary 'keep' suggestion that does match the model`, () => {
|
||||
it(`selects nothing if solitary 'keep' suggestion does match the model`, () => {
|
||||
const predictions: CompositedIntermediatePrediction[] = [
|
||||
{
|
||||
components: {
|
||||
|
|
@ -47,7 +47,7 @@ describe('predictionAutoSelect', () => {
|
|||
assert.sameDeepOrderedMembers(predictions, originalPredictions);
|
||||
|
||||
const autoselected = predictions.find((entry) => entry.components.prediction.autoAccept);
|
||||
assert.isOk(autoselected);
|
||||
assert.isNotOk(autoselected);
|
||||
});
|
||||
|
||||
it(`does not select suggestions if the root correction has no letters`, () => {
|
||||
|
|
@ -139,7 +139,7 @@ describe('predictionAutoSelect', () => {
|
|||
assert.isNotOk(autoselected);
|
||||
});
|
||||
|
||||
it(`selects 'keep' suggestion that does match the model over any alternatives`, () => {
|
||||
it(`selects nothing for 'keep' suggestion that does match the model even with alternatives`, () => {
|
||||
const keepSuggestion: CompositedIntermediatePrediction = {
|
||||
components: {
|
||||
prediction: {
|
||||
|
|
@ -234,7 +234,7 @@ describe('predictionAutoSelect', () => {
|
|||
assert.sameDeepMembers(predictions, originalPredictions);
|
||||
|
||||
const autoselected = predictions.find((entry) => entry.components.prediction.autoAccept);
|
||||
assert.equal(autoselected, keepSuggestion);
|
||||
assert.isNotOk(autoselected);
|
||||
});
|
||||
|
||||
it(`selects solitary non-'keep' suggestion when 'keep' does not match model`, () => {
|
||||
|
|
|
|||
|
|
@ -168,7 +168,6 @@ describe('matchBaseContextState', () => {
|
|||
const suggestion: Suggestion= {
|
||||
transform: { insert: 'dramatically', deleteLeft: 0, id: 3 },
|
||||
displayAs: 'dramatically',
|
||||
transformId: 3,
|
||||
id: 5
|
||||
}
|
||||
|
||||
|
|
@ -234,7 +233,6 @@ describe('matchBaseContextState', () => {
|
|||
const suggestion: Suggestion= {
|
||||
transform: { insert: '', deleteLeft: 'dramatically'.length, id: 3 },
|
||||
displayAs: '""',
|
||||
transformId: 3,
|
||||
id: 5
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -110,7 +110,6 @@ describe('createDefaultKeep', () => {
|
|||
deleteLeft: 4,
|
||||
id: transformId
|
||||
},
|
||||
transformId,
|
||||
displayAs: '<appl>',
|
||||
matchesModel: false,
|
||||
tag: 'keep'
|
||||
|
|
|
|||
|
|
@ -135,7 +135,6 @@ describe('determineContextTransition', () => {
|
|||
insert: ' ',
|
||||
deleteLeft: 0
|
||||
},
|
||||
transformId: 0,
|
||||
displayAs: 'testing'
|
||||
};
|
||||
baseTransition.final.suggestions = [pred_testing];
|
||||
|
|
@ -149,7 +148,7 @@ describe('determineContextTransition', () => {
|
|||
}
|
||||
};
|
||||
|
||||
compositor.acceptSuggestion(applied_testing, baseContext, { insert: '', deleteLeft: 0 });
|
||||
compositor.acceptSuggestion(applied_testing, baseContext, { insert: '', deleteLeft: 0, id: applied_testing.transform.id });
|
||||
const acceptingTransition = tracker.latest;
|
||||
|
||||
const inputDistribution: Distribution<Transform> = [{sample: applied_testing.appendedTransform, p: 1}];
|
||||
|
|
@ -190,7 +189,7 @@ describe('determineContextTransition', () => {
|
|||
transform: {
|
||||
insert: 'testing',
|
||||
deleteLeft: 4,
|
||||
id: 1
|
||||
id: 0
|
||||
},
|
||||
appendedTransform: {
|
||||
insert: ' ',
|
||||
|
|
@ -198,12 +197,11 @@ describe('determineContextTransition', () => {
|
|||
id: 2
|
||||
},
|
||||
id: 4,
|
||||
transformId: 0,
|
||||
displayAs: 'testing'
|
||||
};
|
||||
baseTransition.final.suggestions = [pred_testing];
|
||||
|
||||
compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0 });
|
||||
compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0, id: pred_testing.transform.id });
|
||||
|
||||
const inputDistribution: Distribution<Transform> = [{sample: { insert: 'a', deleteLeft: 0, id: 5 }, p: 1}];
|
||||
|
||||
|
|
@ -224,8 +222,8 @@ describe('determineContextTransition', () => {
|
|||
assert.notEqual(extendingTransition, baseTransition);
|
||||
|
||||
// These values support delayed reversions.
|
||||
assert.equal(extendingTransition.final.displayTokenization.tokens[6].appliedTransitionId, pred_testing.transformId);
|
||||
assert.equal(extendingTransition.final.displayTokenization.tokens[7].appliedTransitionId, pred_testing.transformId);
|
||||
assert.equal(extendingTransition.final.displayTokenization.tokens[6].appliedTransitionId, pred_testing.transform.id);
|
||||
assert.equal(extendingTransition.final.displayTokenization.tokens[7].appliedTransitionId, pred_testing.transform.id);
|
||||
|
||||
// We start a new token here, rather than continue (and/or replace) an old one;
|
||||
// this shouldn't be set here yet.
|
||||
|
|
@ -251,19 +249,18 @@ describe('determineContextTransition', () => {
|
|||
transform: {
|
||||
insert: 'testing',
|
||||
deleteLeft: 4,
|
||||
id: 1
|
||||
id: 0
|
||||
},
|
||||
appendedTransform: {
|
||||
insert: ' ',
|
||||
deleteLeft: 0,
|
||||
id: 2
|
||||
},
|
||||
transformId: 0,
|
||||
displayAs: 'testing'
|
||||
};
|
||||
baseTransition.final.suggestions = [pred_testing];
|
||||
|
||||
compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0 });
|
||||
compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0, id: pred_testing.transform.id });
|
||||
|
||||
const inputDistribution: Distribution<Transform> = [{sample: { insert: 'a', deleteLeft: 0, id: 5 }, p: 1}];
|
||||
|
||||
|
|
@ -291,7 +288,7 @@ describe('determineContextTransition', () => {
|
|||
[{sample: { insert: '', deleteLeft: 1 }, p: 1}]
|
||||
);
|
||||
|
||||
assert.equal(extensionDeletingTransition.revertableTransitionId, pred_testing.transformId);
|
||||
assert.equal(extensionDeletingTransition.revertableTransitionId, pred_testing.transform.id);
|
||||
} finally {
|
||||
warningEmitterSpy.restore();
|
||||
}
|
||||
|
|
@ -313,19 +310,18 @@ describe('determineContextTransition', () => {
|
|||
transform: {
|
||||
insert: 'testing',
|
||||
deleteLeft: 4,
|
||||
id: 1
|
||||
id: 0
|
||||
},
|
||||
appendedTransform: {
|
||||
insert: ' ',
|
||||
deleteLeft: 0,
|
||||
id: 2
|
||||
},
|
||||
transformId: 0,
|
||||
displayAs: 'testing'
|
||||
};
|
||||
baseTransition.final.suggestions = [pred_testing];
|
||||
|
||||
compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0 });
|
||||
compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0, id: pred_testing.transform.id });
|
||||
|
||||
const inputDistribution: Distribution<Transform> = [{sample: { insert: 'a', deleteLeft: 0, id: 5 }, p: 1}];
|
||||
|
||||
|
|
@ -365,7 +361,7 @@ describe('determineContextTransition', () => {
|
|||
[{sample: { insert: '', deleteLeft: 1 }, p: 1}]
|
||||
);
|
||||
|
||||
assert.equal(appendDeletingTransition.revertableTransitionId, pred_testing.transformId);
|
||||
assert.equal(appendDeletingTransition.revertableTransitionId, pred_testing.transform.id);
|
||||
} finally {
|
||||
warningEmitterSpy.restore();
|
||||
}
|
||||
|
|
@ -387,19 +383,18 @@ describe('determineContextTransition', () => {
|
|||
transform: {
|
||||
insert: 'testing',
|
||||
deleteLeft: 4,
|
||||
id: 1
|
||||
id: 0
|
||||
},
|
||||
appendedTransform: {
|
||||
insert: ' ',
|
||||
deleteLeft: 0,
|
||||
id: 2
|
||||
},
|
||||
transformId: 0,
|
||||
displayAs: 'testing'
|
||||
};
|
||||
baseTransition.final.suggestions = [pred_testing];
|
||||
|
||||
compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0 });
|
||||
compositor.acceptSuggestion(pred_testing, baseContext, { insert: '', deleteLeft: 0, id: pred_testing.transform.id });
|
||||
|
||||
const inputDistribution: Distribution<Transform> = [{sample: { insert: 'a', deleteLeft: 0, id: 5 }, p: 1}];
|
||||
|
||||
|
|
|
|||
|
|
@ -137,7 +137,6 @@ describe('predictFromCorrectionSequence', () => {
|
|||
|
||||
assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components[0].prediction), dummied_suggestions.map((s) => {
|
||||
delete s.p;
|
||||
s.transformId = transitionID;
|
||||
s.transform.id = transitionID;
|
||||
return s;
|
||||
}));
|
||||
|
|
@ -205,14 +204,13 @@ describe('predictFromCorrectionSequence', () => {
|
|||
assert.sameOrderedMembers(predictions.map((entry) => entry.components[0].prediction.displayAs), ["it's", "its"]);
|
||||
assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components[0].prediction), dummied_suggestions.map((entry) => {
|
||||
entry = deepCopy(entry);
|
||||
entry.transformId = transitionID;
|
||||
entry.transform.id = transitionID;
|
||||
return entry;
|
||||
}));
|
||||
|
||||
assert.approximately(predictions[0].metadata.probabilities.total, 0.18 * 0.6, 0.00001);
|
||||
assert.approximately(predictions[1].metadata.probabilities.total, 0.02 * 0.6, 0.00001);
|
||||
predictions.forEach((prediction) => assert.equal(prediction.components[0].prediction.transformId, transitionID));
|
||||
predictions.forEach((prediction) => assert.equal(prediction.components[0].prediction.transform.id, transitionID));
|
||||
});
|
||||
|
||||
it('constructs suggestions without input (as if after a context reset)', () => {
|
||||
|
|
@ -266,7 +264,6 @@ describe('predictFromCorrectionSequence', () => {
|
|||
|
||||
assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components.map((c) => c.prediction)), [dummied_suggestions.map((s) => {
|
||||
delete s.p;
|
||||
s.transformId = transitionID;
|
||||
s.transform.id = transitionID;
|
||||
return s;
|
||||
})]);
|
||||
|
|
@ -355,24 +352,21 @@ describe('predictFromCorrectionSequence', () => {
|
|||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
displayAs: 'g',
|
||||
transformId: transitionID,
|
||||
displayAs: 'g'
|
||||
}, {
|
||||
transform: {
|
||||
insert: ' ',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
displayAs: ' ',
|
||||
transformId: transitionID
|
||||
displayAs: ' '
|
||||
}, {
|
||||
transform: {
|
||||
insert: 'apple',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
displayAs: 'apple',
|
||||
transformId: transitionID
|
||||
displayAs: 'apple'
|
||||
}
|
||||
];
|
||||
|
||||
|
|
@ -565,24 +559,21 @@ describe('predictFromCorrectionSequence', () => {
|
|||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
displayAs: 'g',
|
||||
transformId: transitionID
|
||||
displayAs: 'g'
|
||||
}, {
|
||||
transform: {
|
||||
insert: ' ',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
displayAs: ' ',
|
||||
transformId: transitionID
|
||||
displayAs: ' '
|
||||
}, {
|
||||
transform: {
|
||||
insert: 'apple',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
displayAs: 'apple',
|
||||
transformId: transitionID
|
||||
displayAs: 'apple'
|
||||
}
|
||||
];
|
||||
|
||||
|
|
@ -710,16 +701,14 @@ describe('predictFromCorrectionSequence', () => {
|
|||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
displayAs: 'golden',
|
||||
transformId: transitionID
|
||||
displayAs: 'golden'
|
||||
}, {
|
||||
transform: {
|
||||
insert: ' ',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
displayAs: ' ',
|
||||
transformId: transitionID
|
||||
displayAs: ' '
|
||||
}
|
||||
];
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,101 @@
|
|||
|
||||
/*
|
||||
* Keyman is copyright (C) SIL Global. MIT License.
|
||||
*
|
||||
* Created by jahorton on 2026-07-27
|
||||
*
|
||||
* This file contains tests designed to ensure predictive text does not
|
||||
* provide matching 'keep' and 'revert' suggestions in any context.
|
||||
*/
|
||||
|
||||
|
||||
import { assert } from 'chai';
|
||||
|
||||
import { LexicalModelTypes } from '@keymanapp/common-types';
|
||||
import { prependReversion, type TransitionReversionView } from "@keymanapp/lm-worker/test-index";
|
||||
|
||||
import Suggestion = LexicalModelTypes.Suggestion;
|
||||
|
||||
describe('prependReversion', () => {
|
||||
it(`prepends reversions when reverting a non-'keep' suggestion`, () => {
|
||||
// context: Original was appl+u, corrected to apply. Reached via bksp.
|
||||
const suggestions: Suggestion[] = [{
|
||||
tag: 'keep',
|
||||
transform: { insert: 'apply', deleteLeft: 6, id: 3 },
|
||||
displayAs: '"apply"',
|
||||
id: 5,
|
||||
matchesModel: false
|
||||
} as Suggestion];
|
||||
|
||||
const revertable: TransitionReversionView = {
|
||||
reversion: {
|
||||
tag: 'revert',
|
||||
transform: { insert: 'u', deleteLeft: 0, id: 3 },
|
||||
id: -3,
|
||||
displayAs: '"applu"'
|
||||
},
|
||||
final: {
|
||||
suggestions: [{
|
||||
tag: 'keep',
|
||||
transform: { insert: 'applu', deleteLeft: 4, id: 3 },
|
||||
displayAs: '"applu"',
|
||||
id: 2,
|
||||
matchesModel: false
|
||||
} as Suggestion, {
|
||||
transform: { insert: 'apply', deleteLeft: 4, id: 3 },
|
||||
displayAs: 'apply',
|
||||
id: 3
|
||||
}, {
|
||||
transform: { insert: 'applied', deleteLeft: 4, id: 3 },
|
||||
displayAs: 'applied',
|
||||
id: 4
|
||||
}]
|
||||
}
|
||||
};
|
||||
|
||||
prependReversion(suggestions, revertable);
|
||||
|
||||
assert.includeMembers(suggestions, [revertable.reversion]);
|
||||
});
|
||||
|
||||
it(`does not prepend reversions when reverting a 'keep' suggestion`, () => {
|
||||
// context: Original was appl+u, corrected to apply
|
||||
const suggestions: Suggestion[] = [{
|
||||
tag: 'keep',
|
||||
transform: { insert: 'applu', deleteLeft: 5, id: 3 },
|
||||
displayAs: '"applu"',
|
||||
id: 5,
|
||||
matchesModel: false
|
||||
} as Suggestion];
|
||||
|
||||
const revertable: TransitionReversionView = {
|
||||
reversion: {
|
||||
tag: 'revert',
|
||||
transform: { insert: 'u', deleteLeft: 0, id: 3 },
|
||||
id: -2,
|
||||
displayAs: '"applu"'
|
||||
},
|
||||
final: {
|
||||
suggestions: [{
|
||||
tag: 'keep',
|
||||
transform: { insert: 'applu', deleteLeft: 4, id: 3 },
|
||||
displayAs: '"applu"',
|
||||
id: 2,
|
||||
matchesModel: false
|
||||
} as Suggestion, {
|
||||
transform: { insert: 'apply', deleteLeft: 4, id: 3 },
|
||||
displayAs: 'apply',
|
||||
id: 3
|
||||
}, {
|
||||
transform: { insert: 'applied', deleteLeft: 4, id: 3 },
|
||||
displayAs: 'applied',
|
||||
id: 4
|
||||
}]
|
||||
}
|
||||
};
|
||||
|
||||
prependReversion(suggestions, revertable);
|
||||
|
||||
assert.notIncludeMembers(suggestions, [revertable.reversion]);
|
||||
});
|
||||
});
|
||||
|
|
@ -296,7 +296,6 @@ describe('processSimilarity', () => {
|
|||
deleteLeft: 4,
|
||||
id: transformId
|
||||
},
|
||||
transformId,
|
||||
displayAs: 'apple'
|
||||
},
|
||||
correction: 'appl'
|
||||
|
|
|
|||
|
|
@ -588,7 +588,6 @@ describe('ModelCompositor', function() {
|
|||
deleteLeft: 0,
|
||||
id: 0
|
||||
},
|
||||
transformId: 0,
|
||||
displayAs: 'hello'
|
||||
};
|
||||
|
||||
|
|
@ -668,13 +667,12 @@ describe('ModelCompositor', function() {
|
|||
}
|
||||
|
||||
it('first word of context, postTransform provided, .deleteLeft = 0', function() {
|
||||
let baseSuggestion = {
|
||||
let baseSuggestion: Suggestion = {
|
||||
transform: {
|
||||
insert: 'hello ',
|
||||
deleteLeft: 2,
|
||||
id: 0
|
||||
},
|
||||
transformId: 0,
|
||||
id: 1,
|
||||
displayAs: 'hello'
|
||||
};
|
||||
|
|
@ -687,11 +685,12 @@ describe('ModelCompositor', function() {
|
|||
// of the Context when the suggestion is built.
|
||||
let postTransform = {
|
||||
insert: 'l',
|
||||
deleteLeft: 0
|
||||
deleteLeft: 0,
|
||||
id: baseSuggestion.transform.id
|
||||
}
|
||||
|
||||
let reversion = acceptanceTest(englishPunctuation, baseSuggestion, baseContext, postTransform);
|
||||
assert.equal(reversion.transformId, baseSuggestion.transformId);
|
||||
assert.equal(reversion.transform.id, baseSuggestion.transform.id);
|
||||
assert.equal(reversion.id, -baseSuggestion.id);
|
||||
|
||||
// Check #1: Does the returned reversion properly revert the context to its pre-application state?
|
||||
|
|
@ -711,13 +710,12 @@ describe('ModelCompositor', function() {
|
|||
});
|
||||
|
||||
it('second word of context, postTransform provided, .deleteLeft = 0', function() {
|
||||
let baseSuggestion = {
|
||||
let baseSuggestion: Suggestion = {
|
||||
transform: {
|
||||
insert: 'world ',
|
||||
deleteLeft: 3,
|
||||
id: 0
|
||||
},
|
||||
transformId: 0,
|
||||
id: 0,
|
||||
displayAs: 'world'
|
||||
};
|
||||
|
|
@ -730,11 +728,12 @@ describe('ModelCompositor', function() {
|
|||
// of the Context when the suggestion is built.
|
||||
let postTransform = {
|
||||
insert: 'l',
|
||||
deleteLeft: 0
|
||||
deleteLeft: 0,
|
||||
id: baseSuggestion.id
|
||||
}
|
||||
|
||||
let reversion = acceptanceTest(englishPunctuation, baseSuggestion, baseContext, postTransform);
|
||||
assert.equal(reversion.transformId, baseSuggestion.transformId);
|
||||
assert.equal(reversion.transform.id, baseSuggestion.transform.id);
|
||||
assert.equal(reversion.id, -baseSuggestion.id);
|
||||
|
||||
// Check #1: Does the returned reversion properly revert the context to its pre-application state?
|
||||
|
|
@ -754,13 +753,12 @@ describe('ModelCompositor', function() {
|
|||
});
|
||||
|
||||
it('second word of context, postTransform undefined', function() {
|
||||
let baseSuggestion = {
|
||||
let baseSuggestion: Suggestion = {
|
||||
transform: {
|
||||
insert: 'world ',
|
||||
deleteLeft: 3,
|
||||
id: 0
|
||||
},
|
||||
transformId: 0,
|
||||
id: 0,
|
||||
displayAs: 'world'
|
||||
};
|
||||
|
|
@ -770,7 +768,7 @@ describe('ModelCompositor', function() {
|
|||
}
|
||||
|
||||
let reversion = acceptanceTest(englishPunctuation, baseSuggestion, baseContext);
|
||||
assert.equal(reversion.transformId, baseSuggestion.transformId);
|
||||
assert.equal(reversion.transform.id, baseSuggestion.transform.id);
|
||||
assert.equal(reversion.id, -baseSuggestion.id);
|
||||
|
||||
// Check #1: Does the returned reversion properly revert the context to its pre-application state?
|
||||
|
|
@ -790,13 +788,12 @@ describe('ModelCompositor', function() {
|
|||
});
|
||||
|
||||
it('first word of context + postTransform provided, .deleteLeft > 0', function() {
|
||||
let baseSuggestion = {
|
||||
let baseSuggestion: Suggestion = {
|
||||
transform: {
|
||||
insert: 'hello ',
|
||||
deleteLeft: 2,
|
||||
id: 0
|
||||
},
|
||||
transformId: 0,
|
||||
id: 0,
|
||||
displayAs: 'hello'
|
||||
};
|
||||
|
|
@ -809,11 +806,12 @@ describe('ModelCompositor', function() {
|
|||
// of the Context when the suggestion is built.
|
||||
let postTransform = {
|
||||
insert: 'i',
|
||||
deleteLeft: 1
|
||||
deleteLeft: 1,
|
||||
id: baseSuggestion.transform.id
|
||||
}
|
||||
|
||||
let reversion = acceptanceTest(englishPunctuation, baseSuggestion, baseContext, postTransform);
|
||||
assert.equal(reversion.transformId, baseSuggestion.transformId);
|
||||
assert.equal(reversion.transform.id, baseSuggestion.transform.id);
|
||||
assert.equal(reversion.id, -baseSuggestion.id);
|
||||
|
||||
// Check #1: Does the returned reversion properly revert the context to its pre-application state?
|
||||
|
|
@ -847,13 +845,12 @@ describe('ModelCompositor', function() {
|
|||
// it('first word of context + postTransform provided, .deleteLeft > 0')
|
||||
// seen earlier in the file.
|
||||
|
||||
let baseSuggestion = {
|
||||
let baseSuggestion: Suggestion = {
|
||||
transform: {
|
||||
insert: 'hello ',
|
||||
deleteLeft: 2,
|
||||
id: 0
|
||||
},
|
||||
transformId: 0,
|
||||
id: 0,
|
||||
displayAs: 'hello'
|
||||
};
|
||||
|
|
@ -866,7 +863,8 @@ describe('ModelCompositor', function() {
|
|||
// of the Context when the suggestion is built.
|
||||
let postTransform = {
|
||||
insert: 'i',
|
||||
deleteLeft: 1
|
||||
deleteLeft: 1,
|
||||
id: baseSuggestion.transform.id
|
||||
}
|
||||
|
||||
// Future adjustment: add the 'baseSuggestion' to DummyModel so that it actually
|
||||
|
|
@ -876,7 +874,7 @@ describe('ModelCompositor', function() {
|
|||
let compositor = new ModelCompositor(model, true);
|
||||
|
||||
let reversion = compositor.acceptSuggestion(baseSuggestion, baseContext, postTransform);
|
||||
assert.equal(reversion.transformId, baseSuggestion.transformId);
|
||||
assert.equal(reversion.transform.id, baseSuggestion.transform.id);
|
||||
assert.equal(reversion.id, -baseSuggestion.id);
|
||||
|
||||
let appliedContext = models.applyTransform(baseSuggestion.transform, baseContext);
|
||||
|
|
@ -891,7 +889,8 @@ describe('ModelCompositor', function() {
|
|||
|
||||
let expectedTransform = {
|
||||
insert: 'hi', // Keeps current context the same, though it adds a wordbreak.
|
||||
deleteLeft: 2
|
||||
deleteLeft: 2,
|
||||
id: 0
|
||||
}
|
||||
assert.deepEqual(suggestions[0].transform, expectedTransform);
|
||||
assert.deepEqual(suggestions[0].appendedTransform, {
|
||||
|
|
@ -927,7 +926,7 @@ describe('ModelCompositor', function() {
|
|||
let baseSuggestion = initialSuggestions[1];
|
||||
baseSuggestion.appendedTransform.id = 15; // set an id for the applied transform.
|
||||
let reversion = compositor.acceptSuggestion(baseSuggestion, baseContext, postTransform);
|
||||
assert.equal(reversion.transformId, -baseSuggestion.transformId);
|
||||
assert.equal(reversion.transform.id, baseSuggestion.transform.id);
|
||||
assert.equal(reversion.id, -baseSuggestion.id);
|
||||
|
||||
let appliedContext = models.applyTransform(baseSuggestion.transform, baseContext);
|
||||
|
|
@ -974,7 +973,7 @@ describe('ModelCompositor', function() {
|
|||
assert.equal(compositor.contextTracker.unitTestEndPoints.cache().size, 2);
|
||||
let contextIds = compositor.contextTracker.unitTestEndPoints.cache().keys();
|
||||
|
||||
assert.equal(reversion.transformId, -baseSuggestion.transformId);
|
||||
assert.equal(reversion.transform.id, baseSuggestion.transform.id);
|
||||
assert.equal(reversion.id, -baseSuggestion.id);
|
||||
|
||||
const appliedContextState = compositor.contextTracker.unitTestEndPoints.cache().get(15);
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue