erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-08-07 01:15:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

Merge branch 'epic/autocorrect' into chore/merge-master-into-autocorrect

Browse source
This commit is contained in:
Keyman Server 2025-08-28 01:33:18 -07:00 • committed by GitHub
commit fc4e3d007c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
33 changed files with 2396 additions and 892 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/LanguageSettingsActivity.java
View file

@ -110,11 +110,11 @@ public final class LanguageSettingsActivity extends BaseActivity {
RadioGroup radioGroup = (RadioGroup) findViewById(R.id.suggestion_radio_group);
radioGroup.clearCheck();
// Auto-correct disabled for Keyman 18.0 #12767
int[] RadioButtonArray = {
R.id.suggestion_radio_0,
R.id.suggestion_radio_1,
R.id.suggestion_radio_2};
R.id.suggestion_radio_2,
R.id.suggestion_radio_3};
RadioButton radioButton = (RadioButton)radioGroup.findViewById(RadioButtonArray[maySuggest]);
radioButton.setChecked(true);

5
android/KMEA/app/src/main/res/layout/language_settings_list_layout.xml
View file

@ -83,13 +83,12 @@
android:layout_gravity="center_vertical"
android:text="@string/suggestions_radio_2" />
<!-- Auto-correct disabled for Keyman 18.0 #12767 -->
<!--com.google.android.material.radiobutton.MaterialRadioButton
<com.google.android.material.radiobutton.MaterialRadioButton
android:id="@+id/suggestion_radio_3"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_gravity="center_vertical"
android:text="@string/suggestions_radio_3" /-->
android:text="@string/suggestions_radio_3" />
</RadioGroup>

72
common/test/resources/json/models/future_suggestions/i_got_distracted_by_hazel.json
View file

@ -2,21 +2,33 @@
[
{
"transform": {
"insert": "I ",
"insert": "I",
"deleteLeft": 0
},
"appendedTransform": {
"insert": " ",
"deleteLeft": 0
},
"displayAs": "I"
},
{
"transform": {
"insert": "I'm ",
"insert": "I'm",
"deleteLeft": 0
},
"appendedTransform": {
"insert": " ",
"deleteLeft": 0
},
"displayAs": "I'm"
},
{
"transform": {
"insert": "Oh ",
"insert": "Oh",
"deleteLeft": 0
},
"appendedTransform": {
"insert": " ",
"deleteLeft": 0
},
"displayAs": "Oh"
@ -25,21 +37,33 @@
[
{
"transform": {
"insert": "love ",
"insert": "love",
"deleteLeft": 0
},
"appendedTransform": {
"insert": " ",
"deleteLeft": 0
},
"displayAs": "love"
},
{
"transform": {
"insert": "am ",
"insert": "am",
"deleteLeft": 0
},
"appendedTransform": {
"insert": " ",
"deleteLeft": 0
},
"displayAs": "am"
},
{
"transform": {
"insert": "got ",
"insert": "got",
"deleteLeft": 0
},
"appendedTransform": {
"insert": " ",
"deleteLeft": 0
},
"displayAs": "got"
@ -48,21 +72,33 @@
[
{
"transform": {
"insert": "distracted by ",
"insert": "distracted by",
"deleteLeft": 0
},
"appendedTransform": {
"insert": " ",
"deleteLeft": 0
},
"displayAs": "distracted by"
},
{
"transform": {
"insert": "distracted ",
"insert": "distracted",
"deleteLeft": 0
},
"appendedTransform": {
"insert": " ",
"deleteLeft": 0
},
"displayAs": "distracted"
},
{
"transform": {
"insert": "a ",
"insert": "a",
"deleteLeft": 0
},
"appendedTransform": {
"insert": " ",
"deleteLeft": 0
},
"displayAs": "a"
@ -71,21 +107,33 @@
[
{
"transform": {
"insert": "Hazel ",
"insert": "Hazel",
"deleteLeft": 0
},
"appendedTransform": {
"insert": " ",
"deleteLeft": 0
},
"displayAs": "Hazel"
},
{
"transform": {
"insert": "the ",
"insert": "the",
"deleteLeft": 0
},
"appendedTransform": {
"insert": " ",
"deleteLeft": 0
},
"displayAs": "the"
},
{
"transform": {
"insert": "a ",
"insert": "a",
"deleteLeft": 0
},
"appendedTransform": {
"insert": " ",
"deleteLeft": 0
},
"displayAs": "a"

11
common/web/types/src/lexical-model-types.ts
View file

@ -303,11 +303,18 @@ export interface Suggestion {
id?: number;
/**
* The suggested update to the buffer. Note that this transform should
* be applied AFTER the instigating transform, if any.
* Specifies the edits needed to correct and extend the currently-edited word
* (within the text buffer) to match the suggested word from the lexicon.
* Note that this transform should be applied BEFORE the instigating transform, if any.
*/
readonly transform: Transform;
/**
* Applies extra language-appropriate whitespace and/or punctuation after the main
* Suggestion body as specified by the source LexicalModel.
*/
appendedTransform?: Transform;
/**
* A string to display the suggestion to the typist.
* This should aid the typist understand what the transform

2
web/README.md
View file

@ -1,5 +1,5 @@
# Keyman Engine for Web
The Original Code is (C) SIL International
The Original Code is (C) SIL Global
## Prerequisites
See [build configuration](../docs/build/index.md) for details on how to

8
web/src/engine/main/src/headless/languageProcessor.ts
View file

@ -21,7 +21,7 @@ export class LanguageProcessor extends EventEmitter<LanguageProcessorEventMap> {
private _mayPredict: boolean = true;
private _mayCorrect: boolean = true;
private _mayAutoCorrect: boolean = false; // initialized to false - #12767
private _mayAutoCorrect: boolean = true;
private _state: StateChangeEnum = 'inactive';
@ -216,6 +216,9 @@ export class LanguageProcessor extends EventEmitter<LanguageProcessorEventMap> {
// Step 1: determine the final output text
const final = Mock.from(original.preInput, false);
final.apply(suggestion.transform);
if(suggestion.appendedTransform) {
final.apply(suggestion.appendedTransform);
}
// Step 2: build a final, master Transform that will produce the desired results from the CURRENT state.
// In embedded mode, both Android and iOS are best served by calculating this transform and applying its
@ -289,6 +292,9 @@ export class LanguageProcessor extends EventEmitter<LanguageProcessorEventMap> {
// Step 1: determine the final output text
const final = Mock.from(original.preInput, false);
final.apply(reversion.transform); // Should match original.transform, actually. (See applySuggestion)
if(reversion.appendedTransform) {
final.apply(reversion.appendedTransform);
}
// Step 2: build a final, master Transform that will produce the desired results from the CURRENT state.
// In embedded mode, both Android and iOS are best served by calculating this transform and applying its

5
web/src/engine/osk/src/banner/suggestionBanner.ts
View file

@ -773,11 +773,10 @@ export class SuggestionBanner extends Banner {
if(suggestions.length > i) {
const suggestion = suggestions[i];
d.update(suggestion, optionFormat);
if(this.predictionContext.selected == suggestion) {
d.highlight(true);
}
d.highlight(this.predictionContext.selected == suggestion)
} else {
d.update(null, optionFormat);
d.highlight(false);
}
}

11
web/src/engine/predictive-text/templates/src/common.ts
View file

@ -55,13 +55,18 @@ export function buildMergedTransform(first: Transform, second: Transform): Trans
}
}
return {
const returnedObj: Transform = {
insert: mergedFirstInsert + second.insert,
deleteLeft: first.deleteLeft + mergedSecondDelete,
deleteLeft: first.deleteLeft + mergedSecondDelete
}
if(first.deleteRight != undefined || second.deleteRight != undefined) {
// As `first` would affect the context before `second` could take effect,
// this is the correct way to merge `deleteRight`.
deleteRight: (first.deleteRight || 0) + (second.deleteRight || 0)
returnedObj.deleteRight = (first.deleteRight || 0) + (second.deleteRight || 0)
}
return returnedObj;
}
/**

12
web/src/engine/predictive-text/wordbreakers/src/main/default/index.ts
View file

@ -38,7 +38,7 @@ export interface DefaultWordBreakerOptions {
* @see http://unicode.org/reports/tr29/#Word_Boundaries
* @see https://github.com/eddieantonio/unicode-default-word-boundary/tree/v12.0.0
*/
export default function default_(text: string, options?: DefaultWordBreakerOptions): LexicalModelTypes.Span[] {
function default_(text: string, options?: DefaultWordBreakerOptions): LexicalModelTypes.Span[] {
let boundaries = findBoundaries(text, options);
if (boundaries.length == 0) {
return [];
@ -64,6 +64,16 @@ export default function default_(text: string, options?: DefaultWordBreakerOptio
return spans;
}
// Exposes `searchForProperty` for external use while associating it with this wordbreaker.
const def = Object.assign(default_, {
/**
* This method returns enum values corresponding to the character type as perceived by the wordbreaking algorithm.
*/
searchForProperty: searchForProperty
});
export default def;
/**
* A span that does not cut out the substring until it absolutely has to!
*/

5
web/src/engine/predictive-text/wordbreakers/src/main/index.ts
View file

@ -1,6 +1,7 @@
import placeholder from "./placeholder.js";
import ascii from "./ascii.js";
import default_ from "./default/index.js";
import { WordBreakProperty } from "./default/data.inc.js";
export { placeholder, ascii, default_ as default, default_ as defaultWordbreaker };
export { type BreakerContext } from "./default/index.js";
export { placeholder, ascii, default_ as default, default_ as defaultWordbreaker, WordBreakProperty };
export { type BreakerContext } from "./default/index.js";

124
web/src/engine/predictive-text/worker-thread/src/main/correction/alignment-helpers.ts Normal file
View file

@ -0,0 +1,124 @@
/*
* Keyman is copyright (C) SIL Global. MIT License.
*
* Created by jahorton on 2025-07-30
*
* This file defines methods used as helpers when aligning cached context state
* information with incoming contexts and when validating partial substitution
* edits for aligned context tokens.
*/
import { ClassicalDistanceCalculation, EditOperation } from "./classical-calculation.js";
/**
* Determines the proper 'last match' index for a tokenized sequence based on its edit path.
*
* In particular, this method is designed to handle the following case:
* ['to', 'apple', ' ', ''] => ['to', 'apply', ' ', 'n']
*
* Edit path for this example case:
* ['match', 'substitute', 'match', 'substitute']
*
* In cases such as these, the whitespace match should be considered 'edited'. While the ' '
* is unedited, it follows the edited 'apple' => 'apply', so it must have been deleted and
* then re-inserted. As a result, 'to' is the true "last matched" token.
* @param editPath
* @returns
*/
export function getEditPathLastMatch(editPath: EditOperation[]) {
const editLength = editPath.length;
// Special handling: appending whitespace to whitespace with the default wordbreaker.
// The default wordbreaker currently adds an empty token after whitespace; this would
// show up with 'substitute', 'match' at the end of the edit path. (This should remain.)
if(editLength >= 2 && editPath[editLength - 2] == 'substitute' && editPath[editLength - 1] == 'match') {
return editPath.lastIndexOf('match', editLength - 2);
} else {
return editPath.lastIndexOf('match');
}
}
/**
* Aligns two tokens on a character-by-character basis as needed for higher, token-level alignment
* operations.
* @param incomingToken The incoming token value
* @param matchingToken The pre-existing token value to use for comparison and alignment
* @param forNearCaret If `false`, disallows any substitutions and activates a leading-edge alignment
* validation mode.
* @returns
*/
export function isSubstitutionAlignable(
incomingToken: string,
matchingToken: string,
forNearCaret?: boolean
): boolean {
// 1 - Determine the edit path for the word.
let subEditPath = ClassicalDistanceCalculation.computeDistance(
[...matchingToken].map(value => ({key: value})),
[...incomingToken].map(value => ({key: value})),
// Diagonal width to consider must be at least 2, as adding a single
// whitespace after a token tends to add two tokens: one for whitespace,
// one for the empty token to follow it.
3
).editPath();
const firstInsert = subEditPath.indexOf('insert');
const firstDelete = subEditPath.indexOf('delete');
// 2 - deletions and insertions should be mutually exclusive.
// A fixed, unedited word can't slide across both 'left' and 'right' boundaries at the same time.
if(firstInsert != -1 && firstDelete != -1) {
return false;
};
// 3 - checks exclusive to leading-edge conditions
if(!forNearCaret) {
const firstSubstitute = subEditPath.indexOf('substitute');
const firstMatch = subEditPath.indexOf('match');
if(firstSubstitute > -1) {
return false;
} else if(firstMatch > -1) {
// Should not have inserts on both sides of matched text!
if(firstInsert > -1 && firstInsert < firstMatch && subEditPath.lastIndexOf('insert') > firstMatch) {
return false;
} else if(firstDelete > -1 && firstDelete < firstMatch && subEditPath.lastIndexOf('delete') > firstMatch) {
return false;
}
}
// Further checks below are oriented for text/tokens at the caret.
return true;
}
// 4 - check the stats for total edits of each type and validate that edits don't overly exceed
// original characters.
const editCount = {
matchMove: 0,
rawEdit: 0
};
subEditPath.forEach((entry) => {
switch(entry) {
case 'transpose-end':
case 'transpose-start':
case 'match':
editCount.matchMove++;
break;
case 'insert':
case 'transpose-insert':
case 'delete':
case 'transpose-delete':
case 'substitute':
editCount.rawEdit++;
}
});
// We shouldn't have more raw substitutions, inserts, and deletes than matches + transposes,
// though allowing +1 as a fudge factor.
// The 'a' => 'à' pattern can be a reasonably common Keyman keyboard rule and
// is one substitution, zero matches in NFC.
if(editCount.matchMove + 1 < editCount.rawEdit) {
return false;
}
return true;
}

146
web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts Normal file
View file

@ -0,0 +1,146 @@
/*
* Keyman is copyright (C) SIL Global. MIT License.
*
* Created by jahorton on 2025-07-30
*
* Represents cached data about the state of the sliding context window either
* before or after a context transition event and related functionality.
*/
import { LexicalModelTypes } from '@keymanapp/common-types';
import { ContextTokenization } from './context-tokenization.js';
import Context = LexicalModelTypes.Context;
import Distribution = LexicalModelTypes.Distribution;
import LexicalModel = LexicalModelTypes.LexicalModel;
import Suggestion = LexicalModelTypes.Suggestion;
import Transform = LexicalModelTypes.Transform;
import { ContextToken } from './context-token.js';
import { determineModelTokenizer } from '#./model-helpers.js';
/**
* Represents a state of the active context at some point in time along with the
* results of all related, reusable predictive-text operations.
*/
export class ContextState {
/**
* The context window in view for the represented Context state,
* as passed between the predictive-text worker and its host.
*/
readonly context: Context;
/**
* The active lexical model operating upon the Context.
*/
readonly model: LexicalModel;
/**
* Denotes the most likely tokenization for the represented Context.
*/
tokenization: ContextTokenization;
/**
* Denotes the keystroke-sourced Transform that was last applied to a
* prior ContextState.
*
* Note: if this specific ContextState resulted from applying a
* Suggestion, this may not match text seen in the current Context!
*/
appliedInput?: Transform;
/**
* Denotes all keystroke data contributing to ContextTokens seen in
* .tokenization. For each contributing context transition, its ID
* may be used to retrieve the original fat-finger distribution for
* potential keystroke effects.
*/
inputTransforms: Map<number, Distribution<Transform>>;
/**
* The full set of Suggestions produced for the transition to this context state.
*/
suggestions: Suggestion[];
/**
* If set, denotes the suggestion ID for the suggestion (from .suggestions) that
* was applied for the final transition to this context state.
*/
appliedSuggestionId?: number;
/**
* Indicates whether or not the applied suggestion (if it exists) was applied
* directly by the user.
*
* - `true` if directly applied via banner interaction or other explicitly-intended
* behaviors
* - `false` if indirectly applied (say, by triggering whitespace/punctuation input)
* - `undefined` if no suggestion has been applied.
*/
isManuallyApplied?: boolean;
/**
* Deep-copies a prior instance.
* @param stateToClone
*/
constructor(stateToClone: ContextState);
/**
* Initializes a new ContextState instance based on the active model and context.
*
* If a precomputed tokenization of the context (with prior correction-search
* calculation data) is not available, it will be spun up from scratch.
*
* @param context The context available within the current sliding context-window
* @param model The active lexical model.
* @param tokenization Precomputed tokenization for the context, leveraging previous
* correction-search progress and results
*/
constructor(context: Context, model: LexicalModel, tokenization?: ContextTokenization);
constructor(param1: Context | ContextState, model?: LexicalModel, tokenization?: ContextTokenization) {
if(!(param1 instanceof ContextState)) {
this.context = param1;
this.model = model;
if(tokenization) {
this.tokenization = tokenization;
} else {
this.initFromReset();
}
} else {
const stateToClone = param1;
Object.assign(this, stateToClone);
this.inputTransforms = new Map(stateToClone.inputTransforms);
this.tokenization = new ContextTokenization(stateToClone.tokenization);
// A shallow copy of the array is fine, but we'd be best off
// not aliasing the array itself.
if(stateToClone.suggestions?.length ?? 0 > 0) {
this.suggestions = [].concat(stateToClone.suggestions);
}
}
}
/**
* Initializes the ContextState instance for use when no valid prior
* information is available - typically, immediately after engine
* initialization or a context reset.
*/
private initFromReset() {
const tokenizedContext = determineModelTokenizer(this.model)(this.context).left;
const baseTokens = tokenizedContext.map((entry) => {
const token = new ContextToken(this.model, entry.text);
if(entry.isWhitespace) {
token.isWhitespace = true;
}
return token;
});
// And now build the final context state object, which includes whitespace 'tokens'.);
if(baseTokens.length == 0) {
baseTokens.push(new ContextToken(this.model));
}
this.tokenization = new ContextTokenization(baseTokens);
}
}

129
web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts Normal file
View file

@ -0,0 +1,129 @@
/*
* Keyman is copyright (C) SIL Global. MIT License.
*
* Created by jahorton on 2025-07-30
*
* Represents cached data about one token (either a word or a unit of whitespace)
* in the context and associated correction-search progress and results.
*/
import { buildMergedTransform } from "@keymanapp/models-templates";
import { LexicalModelTypes } from '@keymanapp/common-types';
import { SearchSpace } from "./distance-modeler.js";
import Distribution = LexicalModelTypes.Distribution;
import LexicalModel = LexicalModelTypes.LexicalModel;
import Suggestion = LexicalModelTypes.Suggestion;
import Transform = LexicalModelTypes.Transform;
/**
* Breaks apart a raw text string into individual, single-codepoint
* transforms, all set with the specified transform ID.
*
* This is designed for use when initializing a new ContextToken without
* any prior cached data or for rewriting its probabilities after
* receiving backspace input.
* @param text
* @param transformId
* @returns
*/
function textToCharTransforms(text: string, transformId?: number): Transform[] {
return transformId ?
[...text].map(insert => ({insert, deleteLeft: 0, id: transformId})) :
[...text].map(insert => ({insert, deleteLeft: 0}));
}
/**
* Represents cached data about one token (either a word or a unit of whitespace)
* in the context and associated correction-search progress and results.
*/
export class ContextToken {
/**
* Indicates whether or not the token is considered whitespace.
*/
isWhitespace: boolean;
/**
* Contains all relevant correction-search data for use in generating
* corrections for this ContextToken instance.
*/
readonly searchSpace: SearchSpace;
/* The next two fields will **not land here** in the final version for
epic/autocorrect / 19.0-beta! That said, their future location has
not yet been reworked, so we'll keep them here for now. */
/**
* The set of suggestions generated for the current token
*/
suggestions: Suggestion[];
/**
* The ID of the suggestion applied to the current token, if any.
*
* Should be set to undefined when no such suggestion exists.
*/
appliedSuggestionId?: number;
/**
* Constructs a new, empty instance for use with the specified LexicalModel.
* @param model
*/
constructor(model: LexicalModel);
/**
* Constructs a new instance with pre-existing text for use with the specified LexicalModel.
* @param model
* @param rawText
*/
constructor(model: LexicalModel, rawText: string);
/**
* This constructor deep-copies the specified instance.
* @param baseToken
*/
constructor(baseToken: ContextToken);
constructor(param: ContextToken | LexicalModel, rawText?: string) {
if(param instanceof ContextToken) {
const priorToken = param;
this.isWhitespace = priorToken.isWhitespace;
// We need to construct a separate search space from other token copies.
//
// In case we are unable to perfectly track context (say, due to multitaps)
// we need to ensure that only fully-utilized keystrokes are considered.
this.searchSpace = new SearchSpace(priorToken.searchSpace);
this.suggestions = priorToken.suggestions.slice();
// because of unit tests.
if(priorToken.appliedSuggestionId !== undefined) {
this.appliedSuggestionId = priorToken.appliedSuggestionId;
}
} else {
const model = param;
// May be altered outside of the constructor.
this.isWhitespace = false;
this.searchSpace = new SearchSpace(model);
rawText ||= '';
// Supports the old pathway for: updateWithBackspace(tokenText: string, transformId: number)
const rawTransformDistributions: Distribution<Transform>[] = textToCharTransforms(rawText).map(function(transform) {
return [{sample: transform, p: 1.0}];
});
rawTransformDistributions.forEach((entry) => this.searchSpace.addInput(entry));
this.suggestions = [];
}
}
/**
* Displays text corresponding to the net effects of the most likely inputs received
* that can correspond to the current instance.
*/
get exampleInput(): string {
const transforms = this.searchSpace.inputSequence.map((dist) => dist[0].sample)
const composite = transforms.reduce((accum, current) => buildMergedTransform(accum, current), { insert: '', deleteLeft: 0});
return composite.insert;
}
}

302
web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts Normal file
View file

@ -0,0 +1,302 @@
/*
* Keyman is copyright (C) SIL Global. MIT License.
*
* Created by jahorton on 2025-07-30
*
* Represents cached data about one potential tokenization of contents of
* the sliding context window for one specific instance of context state.
*/
import { ContextToken } from './context-token.js';
import { ClassicalDistanceCalculation } from './classical-calculation.js';
import { getEditPathLastMatch, isSubstitutionAlignable } from './alignment-helpers.js';
/**
* Represents token-count values resulting from an alignment attempt between two
* different modeled context states.
*/
export type ContextStateAlignment = {
/**
* Denotes whether or not alignment is possible between two contexts.
*/
canAlign: false
} | {
/**
* Denotes whether or not alignment is possible between two contexts.
*/
canAlign: true,
/**
* Notes the number of tokens added to the head of the 'incoming'/'new' context
* of the contexts being aligned. If negative, the incoming context deleted
* a token found in the 'original' / base context.
*
* For the alignment, [base context index] + leadTokenShift = [incoming context index].
*/
leadTokenShift: number,
/**
* The count of tokens perfectly aligned, with no need for edits, for two successfully-
* alignable contexts.
*/
matchLength: number,
/**
* The count of tokens at the tail perfectly aligned (existing in both contexts) but
* edited for two successfully-alignable contexts. These tokens directly follow those
* that need no edits.
*/
tailEditLength: number,
/**
* The count of new tokens added at the end of the incoming context for two aligned contexts.
* If negative, the incoming context deleted a previously-existing token from the original.
*/
tailTokenShift: number
};
/**
* This class represents the sequence of tokens (words and whitespace blocks)
* held within the active sliding context-window at a single point in time.
*/
export class ContextTokenization {
readonly tokens: ContextToken[];
readonly alignment?: ContextStateAlignment;
constructor(priorToClone: ContextTokenization);
constructor(tokens: ContextToken[], alignment?: ContextStateAlignment);
constructor(param1: ContextToken[] | ContextTokenization, alignment?: ContextStateAlignment) {
if(!(param1 instanceof ContextTokenization)) {
const tokens = param1;
this.tokens = [].concat(tokens);
this.alignment = alignment;
} else {
const priorToClone = param1;
this.tokens = priorToClone.tokens.map((entry) => new ContextToken(entry));
this.alignment = {...priorToClone.alignment};
}
}
/**
* Returns the token adjacent to the text insertion point.
*/
get tail(): ContextToken {
return this.tokens[this.tokens.length - 1];
}
/**
* Returns a plain-text string representing the most probable representation for all
* tokens represented by this tokenization instance.
*/
get exampleInput(): string[] {
return this.tokens
// Hide any tokens representing invisible wordbreaks. (Thinking ahead to phrase-level possibilities)
.filter(token => token.exampleInput !== null)
.map(token => token.exampleInput);
}
/**
* Determines the alignment between a new, incoming tokenization source and the
* tokenization modeled by the current instance.
* @param incomingTokenization Raw strings corresponding to the tokenization of the incoming context
* @returns Alignment data that details if and how the incoming tokenization aligns with
* the tokenization modeled by this instance.
*/
computeAlignment(incomingTokenization: string[]): ContextStateAlignment {
// Map the tokenized state to an edit-distance friendly version.
const tokenizationToMatch = this.exampleInput;
// Inverted order, since 'match' existed before our new context.
let mapping = ClassicalDistanceCalculation.computeDistance(
tokenizationToMatch.map(value => ({key: value})),
incomingTokenization.map(value => ({key: value})),
// Diagonal width to consider must be at least 2, as adding a single
// whitespace after a token tends to add two tokens: one for whitespace,
// one for the empty token to follow it.
3
);
let editPath = mapping.editPath();
// Special case: new context bootstrapping - first token often substitutes.
// The text length is small enough that no words should be able to rotate out the start of the context.
// Special handling needed in case of no 'match'; the rest of the method assumes at least one 'match'.
if(editPath.length <= 3 && (editPath[0] == 'substitute' || editPath[0] == 'match')) {
let matchCount = 0;
let subCount = 0;
for(let i = 0; i < editPath.length; i++) {
if(editPath[i] == 'substitute') {
subCount++;
if(!isSubstitutionAlignable(incomingTokenization[i], tokenizationToMatch[i], true)) {
return {
canAlign: false
};
}
} else if(editPath[i] == 'match') {
// If a substitution is already recorded, treat the 'match' as a substitution.
if(subCount > 0) {
subCount++;
} else {
matchCount++;
}
}
}
const insertCount = editPath.filter((entry) => entry == 'insert').length;
const deleteCount = editPath.filter((entry) => entry == 'delete').length;
return {
canAlign: true,
matchLength: matchCount,
leadTokenShift: 0,
tailEditLength: subCount,
tailTokenShift: insertCount - deleteCount
}
}
// From here on assumes that at least one 'match' exists on the path.
// It all works great... once the context is long enough for at least one stable token.
const firstMatch = editPath.indexOf('match');
const lastMatch = getEditPathLastMatch(editPath);
if(firstMatch == -1) {
// If there are no matches, there's no alignment.
return {
canAlign: false
};
}
// Transpositions are not allowed at the token level during context alignment.
if(editPath.find((entry) => entry.indexOf('transpose') > -1)) {
return {
canAlign: false
};
}
let matchLength = lastMatch - firstMatch + 1;
let tailInsertLength = 0;
let tailDeleteLength = 0;
for(let i = lastMatch; i < editPath.length; i++) {
if(editPath[i] == 'insert') {
tailInsertLength++;
} else if(editPath[i] == 'delete') {
tailDeleteLength++;
}
}
if(tailInsertLength > 0 && tailDeleteLength > 0) {
// Something's gone weird if this happens; that should appear as a substitution instead.
// Otherwise, we have a VERY niche edit scenario.
return {
canAlign: false
};
}
const tailSubstituteLength = (editPath.length - 1 - lastMatch) - tailInsertLength - tailDeleteLength;
// Assertion: for a long context, the bulk of the edit path should be a
// continuous block of 'match' entries. If there's anything else in
// the middle, we have a context mismatch.
if(firstMatch > -1) {
for(let i = firstMatch+1; i < lastMatch; i++) {
if(editPath[i] != 'match') {
return {
canAlign: false
};
}
}
}
// If we have a perfect match with a pre-existing context, no mutations have
// happened; we have a 100% perfect match.
if(firstMatch == 0 && lastMatch == editPath.length - 1) {
return {
canAlign: true,
leadTokenShift: 0,
matchLength,
tailEditLength: tailSubstituteLength,
tailTokenShift: tailInsertLength - tailDeleteLength
};
}
// The edit path calc tries to put substitutes first, before inserts.
// We don't want that on the leading edge.
const lastEarlyInsert = editPath.lastIndexOf('insert', firstMatch);
const firstSubstitute = editPath.indexOf('substitute');
if(firstSubstitute > -1 && firstSubstitute < firstMatch && firstSubstitute < lastEarlyInsert) {
editPath[firstSubstitute] = 'insert';
editPath[lastEarlyInsert] = 'substitute';
}
// If mutations HAVE happened, we need to double-check the context-state alignment.
let priorEdit: typeof editPath[0];
let leadTokensRemoved = 0;
let leadSubstitutions = 0;
// The `i` index below aligns based upon the index within the `tokenizationToMatch` sequence
// and how it would have to be edited to align to the `incomingTokenization` sequence.
for(let i = 0; i < firstMatch; i++) {
switch(editPath[i]) {
case 'delete':
// All deletions should appear at the sliding window edge; if a deletion appears
// after the edge, but before the first match, something's wrong.
if(priorEdit && priorEdit != 'delete') {
return {
canAlign: false
};
}
leadTokensRemoved++;
break;
case 'substitute':
// We only allow for one leading token to be substituted.
//
// Any extras in the front would be pure inserts, not substitutions, due to
// the sliding context window and its implications.
if(leadSubstitutions++ > 0) {
return {
canAlign: false
};
}
// Find the word before and after substitution.
const incomingSub = incomingTokenization[i - (leadTokensRemoved > 0 ? leadTokensRemoved : 0)];
const matchingSub = tokenizationToMatch[i + (leadTokensRemoved < 0 ? leadTokensRemoved : 0)];
// Double-check the word - does the 'substituted' word itself align?
if(!isSubstitutionAlignable(incomingSub, matchingSub)) {
return {
canAlign: false
};
}
// There's no major need to drop parts of a token being 'slid' out of the context window.
// We'll leave it intact and treat it as a 'match'
matchLength++;
break;
case 'insert':
// Only allow an insert at the leading edge, as with 'delete's.
if(priorEdit && priorEdit != 'insert') {
return {
canAlign: false
};
}
// In case of backspaces, it's also possible to 'insert' a 'new'
// token - an old one that's slid back into view.
leadTokensRemoved--;
break;
default:
// No 'match' can exist before the first found index for a 'match'.
// No 'transpose-' edits should exist within this section, either.
return {
canAlign: false
};
}
priorEdit = editPath[i];
}
// If we need some form of tail-token substitution verification, add that here.
return {
canAlign: true,
// leadTokensRemoved represents the number of tokens that must be removed from the base context
// when aligning the contexts. Externally, it's more helpful to think in terms of the count added
// to the incoming context.
leadTokenShift: -leadTokensRemoved + 0, // add 0 in case of a 'negative zero', which affects unit tests.
matchLength,
tailEditLength: tailSubstituteLength,
tailTokenShift: tailInsertLength - tailDeleteLength
};
}
}

799
web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
View file

@ -1,227 +1,17 @@
import { applyTransform, buildMergedTransform, Token } from '@keymanapp/models-templates';
import { KMWString } from '@keymanapp/web-utils';
import { applyTransform, buildMergedTransform } from '@keymanapp/models-templates';
import { ClassicalDistanceCalculation, EditOperation } from './classical-calculation.js';
import { SearchSpace } from './distance-modeler.js';
import TransformUtils from '../transformUtils.js';
import { determineModelTokenizer } from '../model-helpers.js';
import { tokenizeTransform, tokenizeTransformDistribution } from './transform-tokenization.js';
import { tokenizeAndFilterDistribution } from './transform-tokenization.js';
import { LexicalModelTypes } from '@keymanapp/common-types';
import Context = LexicalModelTypes.Context;
import Distribution = LexicalModelTypes.Distribution;
import LexicalModel = LexicalModelTypes.LexicalModel;
import Suggestion = LexicalModelTypes.Suggestion;
import Transform = LexicalModelTypes.Transform;
function textToCharTransforms(text: string, transformId?: number) {
let perCharTransforms: Transform[] = [];
for(let i=0; i < KMWString.length(text); i++) {
let char = KMWString.charAt(text, i); // is SMP-aware
let transform: Transform = {
insert: char,
deleteLeft: 0,
id: transformId
};
perCharTransforms.push(transform);
}
return perCharTransforms;
}
export function getEditPathLastMatch(editPath: EditOperation[]) {
const editLength = editPath.length;
// Special handling: appending whitespace to whitespace with the default wordbreaker.
// The default wordbreaker currently adds an empty token after whitespace; this would
// show up with 'substitute', 'match' at the end of the edit path. (This should remain.)
if(editLength >= 2 && editPath[editLength - 2] == 'substitute' && editPath[editLength - 1] == 'match') {
return editPath.lastIndexOf('match', editLength - 2);
} else {
return editPath.lastIndexOf('match');
}
}
export class TrackedContextSuggestion {
suggestion: Suggestion;
tokenWidth: number;
}
export class TrackedContextToken {
raw: string;
replacementText: string;
isWhitespace?: boolean;
transformDistributions: Distribution<Transform>[] = [];
replacements: TrackedContextSuggestion[] = [];
activeReplacementId: number = -1;
constructor();
constructor(instance: TrackedContextToken);
constructor(instance?: TrackedContextToken) {
if(instance) {
Object.assign(this, instance);
// We don't alter the values in replacements, but we do wish to prevent aliasing
// of the array containing them.
this.replacements = instance.replacements.slice();
}
}
get currentText(): string {
if(this.replacementText === undefined || this.replacementText === null) {
return this.raw;
} else {
return this.replacementText;
}
}
get replacement(): TrackedContextSuggestion {
let replacementId = this.activeReplacementId;
return this.replacements.find(function(replacement) {
return replacement.suggestion.id == replacementId;
});
}
clearReplacements() {
this.activeReplacementId = -1;
this.replacements = []
}
/**
* Used for 14.0's backspace workaround, which flattens all previous Distribution<Transform>
* entries because of limitations with direct use of backspace transforms.
* @param tokenText
* @param transformId
*/
updateWithBackspace(tokenText: string, transformId: number) {
// It's a backspace transform; time for special handling!
//
// For now, with 14.0, we simply compress all remaining Transforms for the token into
// multiple single-char transforms. Probabalistically modeling BKSP is quite complex,
// so we simplify by assuming everything remaining after a BKSP is 'true' and 'intended' text.
//
// Note that we cannot just use a single, monolithic transform at this point b/c
// of our current edit-distance optimization strategy; diagonalization is currently...
// not very compatible with that.
let backspacedTokenContext: Distribution<Transform>[] = textToCharTransforms(tokenText, transformId).map(function(transform) {
return [{sample: transform, p: 1.0}];
});
this.raw = tokenText;
this.transformDistributions = backspacedTokenContext;
this.clearReplacements();
}
update(transformDistribution: Distribution<Transform>, tokenText?: string) {
// Preserve existing text if new text isn't specified.
tokenText = tokenText || (tokenText === '' ? '' : this.raw);
if(transformDistribution?.length > 0) {
this.transformDistributions.push(transformDistribution);
}
// Replace old token's raw-text with new token's raw-text.
this.raw = tokenText;
this.clearReplacements();
}
}
export class TrackedContextState {
// Stores the post-transform Context. Useful as a debugging reference, but also used to
// pre-validate context state matches in case of discarded changes from multitaps.
taggedContext: Context;
model: LexicalModel;
tokens: TrackedContextToken[];
/**
* How many tokens were removed from the start of the best-matching ancestor.
* Useful for restoring older states, e.g., when the user moves the caret backwards, we can recover the context at that position.
*/
indexOffset: number;
// Tracks all search spaces starting at the current token.
// In the lm-layer's current form, this should only ever have one entry.
// Leaves 'design space' for if/when we add support for phrase-level corrections/predictions.
searchSpace: SearchSpace[] = [];
constructor(source: TrackedContextState);
constructor(model: LexicalModel);
constructor(obj: TrackedContextState | LexicalModel) {
if(obj instanceof TrackedContextState) {
let source = obj;
// Be sure to deep-copy the tokens! Pointer-aliasing is bad here.
this.tokens = source.tokens.map(function(token) {
let copy = new TrackedContextToken();
Object.assign(copy, token);
copy.replacements = copy.replacements.slice();
copy.transformDistributions = copy.transformDistributions.slice();
return copy;
});
this.indexOffset = 0;
const lexicalModel = this.model = obj.model;
this.taggedContext = obj.taggedContext;
if(lexicalModel?.traverseFromRoot) {
// We need to construct a separate search space from other ContextStates.
//
// In case we are unable to perfectly track context (say, due to multitaps)
// we need to ensure that only fully-utilized keystrokes are considered.
this.searchSpace = obj.searchSpace.map((space) => new SearchSpace(space));
}
} else {
let lexicalModel = obj;
this.tokens = [];
this.indexOffset = Number.MIN_SAFE_INTEGER;
this.model = lexicalModel;
if(lexicalModel && lexicalModel.traverseFromRoot) {
this.searchSpace = [new SearchSpace(lexicalModel)];
}
}
}
get head(): TrackedContextToken {
return this.tokens[0];
}
get tail(): TrackedContextToken {
return this.tokens[this.tokens.length - 1];
}
popHead() {
this.tokens.splice(0, 1);
this.indexOffset -= 1;
}
pushTail(token: TrackedContextToken) {
if(this.model && this.model.traverseFromRoot) {
this.searchSpace = [new SearchSpace(this.model)]; // yeah, need to update SearchSpace for compatibility
} else {
this.searchSpace = [];
}
this.tokens.push(token);
let state = this;
if(state.searchSpace.length > 0) {
token.transformDistributions.forEach(distrib => state.searchSpace[0].addInput(distrib));
}
}
toRawTokenization() {
let sequence: string[] = [];
for(let token of this.tokens) {
// Hide any tokens representing wordbreaks. (Thinking ahead to phrase-level possibilities)
if(token.currentText !== null) {
sequence.push(token.currentText);
}
}
return sequence;
}
}
import { ContextToken } from './context-token.js';
import { ContextTokenization } from './context-tokenization.js';
import { ContextState } from './context-state.js';
import { ContextTransition } from './context-transition.js';
class CircularArray<Item> {
static readonly DEFAULT_ARRAY_SIZE = 5;
@ -314,365 +104,227 @@ class CircularArray<Item> {
}
}
interface ContextMatchResult {
/**
* Represents the current state of the context after applying incoming keystroke data.
*/
state: TrackedContextState;
/**
* Represents the previously-cached context state that best matches `state` if available.
* May be `null` if no such state could be found within the context-state cache.
*/
baseState: TrackedContextState;
/**
* Indicates the portion of the incoming keystroke data, if any, that applies to
* tokens before the last pre-caret token and thus should not be replaced by predictions
* based upon `state`. If the provided context state + the incoming transform do not
* adequately match the current context, the match attempt will fail with a `null` result.
*
* Should generally be non-null if the token before the caret did not previously exist.
*
* The result may be null if it does not match the prior context state or if bookkeeping
* based upon it is problematic - say, if wordbreaking effects shift due to new input,
* causing a mismatch with the prior state's tokenization.
* (Refer to #12494 for an example case.)
*/
preservationTransform?: Transform;
headTokensRemoved: number;
tailTokensAdded: number;
}
export class ContextTracker extends CircularArray<TrackedContextState> {
export class ContextTracker extends CircularArray<ContextState> {
// Aim: relocate to ContextTransition in some form?
// Or can we split it up in some manner across the different types?
static attemptMatchContext(
tokenizedContext: Token[],
matchState: TrackedContextState,
transformSequenceDistribution?: Distribution<Transform[]>
): ContextMatchResult {
// Map the previous tokenized state to an edit-distance friendly version.
let matchContext: string[] = matchState.toRawTokenization();
context: Context,
lexicalModel: LexicalModel,
matchState: ContextState,
// the distribution should be tokenized already.
transformDistribution?: Distribution<Transform> // transform distribution is needed here.
): ContextTransition {
const baseTransition = new ContextTransition(matchState, matchState.appliedInput?.id);
const transformSequenceDistribution = tokenizeAndFilterDistribution(context, lexicalModel, transformDistribution);
// Inverted order, since 'match' existed before our new context.
let mapping = ClassicalDistanceCalculation.computeDistance(
matchContext.map(value => ({key: value})),
tokenizedContext.map(value => ({key: value.text})),
// Must be at least 2, as adding a single whitespace after a token tends
// to add two tokens: one for whitespace, one for the empty token to
// follow it.
3
);
if(transformDistribution?.[0]) {
context = applyTransform(transformDistribution[0].sample, context);
}
const tokenizedContext = determineModelTokenizer(lexicalModel)(context).left;
const alignmentResults = matchState.tokenization.computeAlignment(tokenizedContext.map((token) => token.text));
let editPath = mapping.editPath();
const firstMatch = editPath.indexOf('match');
const lastMatch = getEditPathLastMatch(editPath);
// Assertion: for a long context, the bulk of the edit path should be a
// continuous block of 'match' entries. If there's anything else in
// the middle, we have a context mismatch.
if(firstMatch) {
for(let i = firstMatch+1; i < lastMatch; i++) {
if(editPath[i] != 'match') {
return null;
}
}
if(!alignmentResults.canAlign) {
return null;
}
// If we have a perfect match with a pre-existing context, no mutations have
// happened; just re-use the old context state.
if(firstMatch == 0 && lastMatch == editPath.length - 1) {
return { state: matchState, baseState: matchState, headTokensRemoved: 0, tailTokensAdded: 0 };
}
// If mutations HAVE happened, we have work to do.
let state = matchState;
let priorEdit: typeof editPath[0];
let poppedTokenCount = 0;
for(let i = 0; i < firstMatch; i++) {
switch(editPath[i]) {
case 'delete':
if(priorEdit && priorEdit != 'delete') {
return null;
}
if(state == matchState) {
state = new TrackedContextState(state);
}
state.popHead();
poppedTokenCount++;
break;
case 'substitute':
// There's no major need to drop parts of a token being 'slid' out of the context window.
// We'll leave it intact.
break;
default:
// No 'insert' should exist on the leading edge of context when the
// context window slides.
//
// No 'transform' edits should exist within this section, either.
return null;
}
}
const {
leadTokenShift,
matchLength,
tailEditLength,
tailTokenShift
} = alignmentResults;
const hasDistribution = transformSequenceDistribution && Array.isArray(transformSequenceDistribution);
// Reset priorEdit for the end-of-context updating loop.
priorEdit = undefined;
// Used to construct and represent the part of the incoming transform that
// does not land as part of the final token in the resulting context. This
// component should be preserved by any suggestions that get applied.
let preservationTransform: Transform;
let pushedTokenCount = 0;
// Now to update the end of the context window.
for(let i = lastMatch+1; i < editPath.length; i++) {
const isLastToken = i == editPath.length - 1;
// If we have a perfect match with a pre-existing context, no mutations have
// happened; just re-use the old context state.
if(tailEditLength == 0 && leadTokenShift == 0 && tailTokenShift == 0) {
baseTransition.finalize(matchState, transformDistribution);
return baseTransition;
} else {
// If we didn't get any input, we really should perfectly match
// a previous context state. If such a state is out of our cache,
// it should simply be rebuilt.
if(!hasDistribution) {
return null;
}
const transformDistIndex = i - (lastMatch + 1);
const tokenDistribution = transformSequenceDistribution.map((entry) => {
const sample = entry.sample[transformDistIndex];
if(!sample) {
return null;
}
return {
sample,
p: entry.p
};
});
const incomingToken = tokenizedContext[i - poppedTokenCount];
// If the tokenized part of the input is a completely empty transform,
// replace it with null. This can happen with our default wordbreaker
// immediately after a whitespace. We don't want to include this
// transform as part of the input when doing correction-search.
let primaryInput = hasDistribution ? tokenDistribution[0]?.sample : null;
// If the incoming token has text but we have no transform (or 'insert') to match
// it with, abort the matching attempt. We can't match this case well yet.
if(editPath[i] != 'delete') {
if(!incomingToken) {
return null;
} else if(!(primaryInput || editPath[i] == 'insert' ) && incomingToken?.text != '') {
return null;
}
}
if(primaryInput && primaryInput.insert == "" && primaryInput.deleteLeft == 0 && !primaryInput.deleteRight) {
primaryInput = null;
}
// If this token's transform component is not part of the final token,
// it's something we'll want to preserve even when applying suggestions
// for the final token.
//
// Note: will need a either a different approach or more specialized
// handling if/when supporting phrase-level (multi-token) suggestions.
if(!isLastToken) {
preservationTransform = preservationTransform && primaryInput ? buildMergedTransform(preservationTransform, primaryInput) : (primaryInput ?? preservationTransform);
}
const isBackspace = primaryInput && TransformUtils.isBackspace(primaryInput);
switch(editPath[i]) {
case 'substitute':
if(isLastToken) {
state = new TrackedContextState(state);
}
const sourceToken = matchState.tokens[i];
state.tokens[i - poppedTokenCount] = sourceToken;
const token = state.tokens[i - poppedTokenCount];
// TODO: I'm beginning to believe that searchSpace should (eventually) be tracked
// on the tokens, rather than on the overall 'state'.
// - Reason: phrase-level corrections / predictions would likely need a search-state
// across per potentially-affected token.
// - Shifting the paradigm should be a separate work unit than the
// context-tracker rework currently being done, though.
if(isBackspace) {
token.updateWithBackspace(incomingToken.text, primaryInput.id);
if(isLastToken) {
state.tokens.pop(); // pops `token`
// puts it back in, rebuilding a fresh search-space that uses the rebuilt
// keystroke distribution from updateWithBackspace.
state.pushTail(token);
}
} else {
token.update(
tokenDistribution,
incomingToken.text
);
if(isLastToken) {
// Search spaces may not exist during some unit tests; the state
// may not have an associated model during some.
state.searchSpace[0]?.addInput(tokenDistribution);
}
}
// For this case, we were _likely_ called by
// ModelCompositor.acceptSuggestion(), which would have marked the
// accepted suggestion.
//
// Upon inspection, this doesn't seem entirely ideal. It works for
// the common case, but not for specially crafted keystroke
// transforms. That said, it's also very low impact. Best as I can
// see, this is only really used for debugging info?
if(state != matchState && !isLastToken) {
token.replacementText = incomingToken.text;
}
break;
case 'insert':
if(priorEdit && priorEdit != 'substitute' && priorEdit != 'match' && priorEdit != 'insert') {
return null;
}
if(!preservationTransform) {
// Allows for consistent handling of "insert" cases; even if there's no edit
// from a prior token, having a defined transform here indicates that
// a new token has been produced. This serves as a useful conditional flag
// for prediction logic.
preservationTransform = { insert: '', deleteLeft: 0 };
}
if(state == matchState) {
state = new TrackedContextState(state);
}
let pushedToken = new TrackedContextToken();
pushedToken.raw = incomingToken.text;
// TODO: assumes that there was no shift in wordbreaking from the
// prior context to the current one. This may actually be a major
// issue for dictionary-based wordbreaking!
//
// If there was such a shift, then we may have extra transforms
// originally on a 'previous' token that got moved into this one!
//
// Suppose we're using a dictionary-based wordbreaker and have
// `butterfl` for our context, which could become butterfly. If the
// next keystroke results in `butterfli`, this would likely be
// tokenized `butter` `fli`. (e.g: `fli` leads to `flight`.) How do
// we know to properly relocate the `f` and `l` transforms?
if(primaryInput) {
pushedToken.transformDistributions = tokenDistribution ? [tokenDistribution] : [];
} else if(incomingToken.text) {
// We have no transform data to match against an inserted token with text; abort!
// Refer to #12494 for an example case; we currently can't map previously-committed
// input transforms to a newly split-off token.
return null;
}
pushedToken.isWhitespace = incomingToken.isWhitespace;
// Auto-replaces the search space to correspond with the new token.
state.pushTail(pushedToken);
pushedTokenCount++;
break;
case 'match':
// The default (Unicode) wordbreaker returns an empty token after whitespace blocks.
// Adding new whitespace extends the whitespace block but preserves the empty token
// following it.
if(priorEdit == 'substitute' && tokenizedContext[tokenizedContext.length-1].text == '') {
// Keep the blank token as-is; no edit needed!
continue;
}
// else 'fallthrough' / return null
case 'delete':
// While we do keep a cache of recent contexts, logic constraints for handling
// multitaps makes it tricky to reliably use in all situations.
// It's best to handle `delete` cases directly for this reason.
for(let j = i + 1; j < editPath.length; j++) {
// If something _other_ than delete follows a 'delete' on the edit path,
// we probably have a context mismatch.
//
// It's possible to construct cases where this isn't true, but it's likely not
// worth trying to handle such rare cases.
if(editPath[j] != 'delete') {
return null;
}
}
// If ALL that remains are deletes, we're good to go.
//
// This may not be the token at the index, but since all that remains are deletes,
// we'll have deleted the correct total number from the end once all iterations
// are done.
if(state == matchState) {
state = new TrackedContextState(state);
}
state.tokens.pop();
break;
default:
// No 'transform' edits should exist within this section.
return null;
}
priorEdit = editPath[i];
}
return {
state,
baseState: matchState,
preservationTransform,
headTokensRemoved: poppedTokenCount,
tailTokensAdded: pushedTokenCount
};
}
// If mutations HAVE happened, we have work to do.
const tokenization = matchState.tokenization.tokens.map((token) => new ContextToken(token));
static modelContextState(
tokenizedContext: Token[],
lexicalModel: LexicalModel
): TrackedContextState {
let baseTokens = tokenizedContext.map(function(entry) {
let token = new TrackedContextToken();
token.raw = entry.text;
if(entry.isWhitespace) {
token.isWhitespace = true;
}
if(leadTokenShift < 0) {
tokenization.splice(0, -leadTokenShift);
} else if(leadTokenShift > 0) {
// TODO: insert token(s) at the start to match the text that's back within the
// sliding context window.
//
// (was not part of original `attemptContextMatch`)
return null;
}
if(token.raw) {
token.transformDistributions = textToCharTransforms(token.raw).map(function(transform) {
return [{sample: transform, p: 1.0}];
});
} else {
// Helps model context-final wordbreaks.
token.transformDistributions = [];
}
return token;
// If no TAIL mutations have happened, we're safe to return now.
if(tailEditLength == 0 && tailTokenShift == 0) {
const state = new ContextState(context, lexicalModel);
state.tokenization = new ContextTokenization(tokenization, alignmentResults);
baseTransition.finalize(state, transformDistribution);
return baseTransition;
}
// ***
// first non-matched tail index within the incoming context
const incomingTailUpdateIndex = matchLength + (leadTokenShift > 0 ? leadTokenShift : 0);
// first non-matched tail index in `matchState`, the base context state.
const matchingTailUpdateIndex = matchLength - (leadTokenShift < 0 ? leadTokenShift : 0);
// The assumed input from the input distribution is always at index 0.
const tokenizedPrimaryInput = hasDistribution ? transformSequenceDistribution[0].sample : null;
// first index: original sample's tokenization
// second index: token index within original sample
const tokenDistribution = transformSequenceDistribution.map((entry) => {
return entry.sample.map((sample) => {
return {
sample: sample,
p: entry.p
}
});
});
// And now build the final context state object, which includes whitespace 'tokens'.
let state = new TrackedContextState(lexicalModel);
// // Gets distribution of token index 1s as excerpted from the sequences' distribution.
// let a = tokenDistribution.map((sequence) => sequence[1]);
while(baseTokens.length > 0) {
// We don't have a pre-existing distribution for this token, so we'll build one as
// if we'd just produced the token from a backspace.
if(baseTokens.length == 1) {
baseTokens[0].updateWithBackspace(baseTokens[0].raw, null);
// Using these as base indices...
let tailIndex = 0;
// let lastTailIndex = tailEditLength + (tailTokenShift > 0 ? tailTokenShift : 0);
// Used to construct and represent the part of the incoming transform that
// does not land as part of the final token in the resulting context. This
// component should be preserved by any suggestions that get applied.
let preservationTransform: Transform;
for(let i = 0; i < tailEditLength; i++) {
// do tail edits
const incomingIndex = i + incomingTailUpdateIndex;
const matchingIndex = i + matchingTailUpdateIndex;
const incomingToken = tokenizedContext[incomingIndex];
const matchedToken = matchState.tokenization.tokens[matchingIndex];
let primaryInput = hasDistribution ? tokenizedPrimaryInput[i] : null;
const isBackspace = primaryInput && TransformUtils.isBackspace(primaryInput);
const isLastToken = incomingIndex == tokenizedContext.length - 1;
if(isLastToken) {
// If this token's transform component is not part of the final token,
// it's something we'll want to preserve even when applying suggestions
// for the final token.
//
// Note: will need a either a different approach or more specialized
// handling if/when supporting phrase-level (multi-token) suggestions.
} else {
preservationTransform = preservationTransform && primaryInput ? buildMergedTransform(preservationTransform, primaryInput) : (primaryInput ?? preservationTransform);
}
state.pushTail(baseTokens.splice(0, 1)[0]);
let token: ContextToken;
if(isBackspace) {
token = new ContextToken(lexicalModel, incomingToken.text);
token.searchSpace.inputSequence.forEach((entry) => entry[0].sample.id = primaryInput.id);
} else {
// Assumption: there have been no intervening keystrokes since the last well-aligned context.
// (May not be valid with epic/dict-breaker or with complex, word-boundary crossing transforms)
token = new ContextToken(matchedToken);
token.searchSpace.addInput(tokenDistribution.map((seq) => seq[tailIndex]));
}
tokenization[incomingIndex] = token;
tailIndex++;
}
if(state.tokens.length == 0) {
let token = new TrackedContextToken();
token.raw = '';
if(tailTokenShift < 0) {
// delete tail tokens
for(let i = 0; i > tailTokenShift; i--) {
// If ALL that remains are deletes, we're good to go.
//
// This may not be the token at the index, but since all that remains are deletes,
// we'll have deleted the correct total number from the end once all iterations
// are done.
tokenization.pop();
}
} else {
for(let i = tailEditLength; i < tailEditLength + tailTokenShift; i++) {
// create tail tokens
const incomingIndex = i + incomingTailUpdateIndex;
const incomingToken = tokenizedContext[incomingIndex];
// // Assertion: there should be no matching token; this should be a newly-appended token.
// const matchingIndex = i + tailEditLength + matchingTailUpdateIndex;
state.pushTail(token);
const primaryInput = hasDistribution ? tokenizedPrimaryInput[i] : null;
if(!preservationTransform) {
// Allows for consistent handling of "insert" cases; even if there's no edit
// from a prior token, having a defined transform here indicates that
// a new token has been produced. This serves as a useful conditional flag
// for prediction logic.
preservationTransform = { insert: '', deleteLeft: 0 };
}
const isLastToken = incomingIndex == tokenizedContext.length - 1;
if(!isLastToken) {
preservationTransform = preservationTransform && primaryInput ? buildMergedTransform(preservationTransform, primaryInput) : (primaryInput ?? preservationTransform);
}
let pushedToken = new ContextToken(lexicalModel);
// TODO: assumes that there was no shift in wordbreaking from the
// prior context to the current one. This may actually be a major
// issue for dictionary-based wordbreaking!
//
// If there was such a shift, then we may have extra transforms
// originally on a 'previous' token that got moved into this one!
//
// Suppose we're using a dictionary-based wordbreaker and have
// `butterfl` for our context, which could become butterfly. If the
// next keystroke results in `butterfli`, this would likely be
// tokenized `butter` `fli`. (e.g: `fli` leads to `flight`.) How do
// we know to properly relocate the `f` and `l` transforms?
let tokenDistribComponent = tokenDistribution.map((seq) => {
const entry = seq[tailIndex];
if(!entry || TransformUtils.isEmpty(entry.sample)) {
return null;
} else {
return entry;
}
}).filter((entry) => !!entry);
if(primaryInput) {
let transformDistribution = tokenDistribComponent.length > 0 ? tokenDistribComponent : null;
if(transformDistribution) {
pushedToken.searchSpace.addInput(transformDistribution);
}
} else if(incomingToken.text) {
// We have no transform data to match against an inserted token with text; abort!
// Refer to #12494 for an example case; we currently can't map previously-committed
// input transforms to a newly split-off token.
return null;
}
pushedToken.isWhitespace = incomingToken.isWhitespace;
// Auto-replaces the search space to correspond with the new token.
tokenization.push(pushedToken);
tailIndex++;
}
}
return state;
const state = new ContextState(context, lexicalModel);
state.tokenization = new ContextTokenization(tokenization, alignmentResults);
baseTransition.finalize(state, transformDistribution, preservationTransform);
return baseTransition;
}
// Aim: relocate to ContextState in some form... or ContextTransition?
/**
* Compares the current, post-input context against the most recently-seen contexts from previous prediction calls, returning
* the most information-rich `TrackedContextState` possible. If a match is found, the state will be annotated with the
@ -688,42 +340,31 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
context: Context,
transformDistribution?: Distribution<Transform>,
preserveMatchState?: boolean
): ContextMatchResult {
): ContextTransition {
if(!model.traverseFromRoot) {
// Assumption: LexicalModel provides a valid traverseFromRoot function. (Is technically optional)
// Without it, no 'corrections' may be made; the model can only be used to predict, not correct.
throw "This lexical model does not provide adequate data for correction algorithms and context reuse";
}
let tokenize = determineModelTokenizer(model);
const inputTransform = transformDistribution?.[0];
let transformTokenLength = 0;
let tokenizedDistribution: Distribution<Transform[]> = null;
if(inputTransform) {
// These two methods apply transforms internally; do not mutate context here.
// This particularly matters for the 'distribution' variant.
transformTokenLength = tokenizeTransform(tokenize, context, inputTransform.sample).length;
tokenizedDistribution = tokenizeTransformDistribution(tokenize, context, transformDistribution);
// Now we update the context used for context-state management based upon our input.
context = applyTransform(inputTransform.sample, context);
// While we lack phrase-based / phrase-oriented prediction support, we'll just extract the
// set that matches the token length that results from our input.
tokenizedDistribution = tokenizedDistribution.filter((entry) => entry.sample.length == transformTokenLength);
if(transformDistribution?.length == 0) {
transformDistribution = null;
}
const tokenizedContext = tokenize(context);
const inputTransform = transformDistribution?.[0];
const postContext = inputTransform ? applyTransform(inputTransform.sample, context) : context;
if(tokenizedContext.left.length > 0) {
const tokenize = determineModelTokenizer(model);
const tokenizedPostContext = tokenize(postContext)
if(tokenizedPostContext.left.length > 0) {
for(let i = this.count - 1; i >= 0; i--) {
const priorMatchState = this.item(i);
// Skip intermediate multitap-produced contexts.
// When multitapping, we skip all contexts from prior taps within the same interaction,
// but not any contexts from before the multitap started.
const priorTaggedContext = priorMatchState.taggedContext;
const priorTaggedContext = priorMatchState.context;
if(priorTaggedContext && transformDistribution && transformDistribution.length > 0) {
// Using the potential `matchState` + the incoming transform, do the results line up for
// our observed context? If not, skip it.
@ -733,27 +374,26 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
//
// `priorTaggedContext` must not be `null`!
const doublecheckContext = applyTransform(transformDistribution[0].sample, priorTaggedContext);
if(doublecheckContext.left != context.left) {
if(doublecheckContext.left != postContext.left) {
continue;
}
} else if(priorTaggedContext?.left != context.left) {
} else if(priorTaggedContext?.left != postContext.left) {
continue;
}
let result = ContextTracker.attemptMatchContext(tokenizedContext.left, this.item(i), tokenizedDistribution);
let result = ContextTracker.attemptMatchContext(context, model, this.item(i), transformDistribution);
if(result?.state) {
if(result?.final) {
// Keep it reasonably current! And it's probably fine to have it more than once
// in the history. However, if it's the most current already, there's no need
// to refresh it.
if(this.newest != result.state && this.newest != priorMatchState) {
if(this.newest != result.final && this.newest != priorMatchState) {
// Already has a taggedContext.
this.enqueue(priorMatchState);
}
result.state.taggedContext = context;
if(result.state != this.item(i)) {
this.enqueue(result.state);
if(result.final != this.item(i)) {
this.enqueue(result.final);
}
return result;
}
@ -765,10 +405,13 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
//
// Assumption: as a caret needs to move to context before any actual transform distributions occur,
// this state is only reached on caret moves; thus, transformDistribution is actually just a single null transform.
let state = ContextTracker.modelContextState(tokenizedContext.left, model);
state.taggedContext = context;
let state = new ContextState(context, model);
this.enqueue(state);
return { state, baseState: null, headTokensRemoved: 0, tailTokensAdded: 0 };
const transition = new ContextTransition(state, /* TODO: we need a clear value here in the future! */ null);
// Hacky, but holds the course for now. This should only really happen from context resets, which can
// then use a different path.
transition.finalize(state, []);
return transition;
}
clearCache() {

115
web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts Normal file
View file

@ -0,0 +1,115 @@
/*
* Keyman is copyright (C) SIL Global. MIT License.
*
* Created by jahorton on 2025-07-30
*
* Represents cached data about a single context transition event, as well
* as the state of the context both before and after the transition.
*/
import { LexicalModelTypes } from '@keymanapp/common-types';
import { ContextState } from './context-state.js';
import Distribution = LexicalModelTypes.Distribution;
import Transform = LexicalModelTypes.Transform;
/**
* Represents the transition between two context states as triggered
* by input keystrokes or applied suggestions.
*/
export class ContextTransition {
/**
* Represents the state of the context before the transition event occurred.
*/
readonly base: ContextState;
private _final: ContextState;
/**
* Indicates the fat-finger distribution for the incoming keystroke related to
* the context transition event.
*/
inputDistribution?: Distribution<Transform>;
// The transform ID in play.
private _transitionId?: number;
/**
* Indicates the portion of the incoming keystroke data, if any, that applies to
* tokens before the last pre-caret token and thus should not be replaced by predictions
* based upon `state`. If the provided context state + the incoming transform do not
* adequately match the current context, the match attempt will fail with a `null` result.
*
* Should generally be non-null if the token before the caret did not previously exist.
*
* The result may be null if it does not match the prior context state or if bookkeeping
* based upon it is problematic - say, if wordbreaking effects shift due to new input,
* causing a mismatch with the prior state's tokenization.
* (Refer to #12494 for an example case.)
*/
preservationTransform?: Transform;
/**
* Constructs a partial context transition object for use during the process
* of analyzing context transitions or for representing the base state of a
* reset context.
* @param context The base state for the represented context transition
* @param transitionId The unique ID corresponding to the transition event
* or context state.
*/
constructor(context: ContextState, transitionId: number);
/**
* Deep-copies a ContextTransition instance.
* @param baseTransition
*/
constructor(baseTransition: ContextTransition);
constructor(param: ContextState | ContextTransition, transitionId?: number) {
if(!(param instanceof ContextTransition)) {
const contextState = param;
// We're initializing a ContextTransition from a blank or reset context.
this.base = contextState;
this._final = null;
this._transitionId = transitionId;
} else {
const baseTransition = param;
Object.assign(this, baseTransition);
// These need to be deep-copied.
this.base = new ContextState(baseTransition.base);
this._final = new ContextState(baseTransition._final);
}
}
/**
* Gets the context state resulting from the context transition event,
* including any generated suggestions and data regarding potential
* application thereof.
*/
get final(): ContextState {
return this._final;
}
/**
* The unique ID corresponding to the transition event or context state.
*/
get transitionId(): number {
return this._transitionId;
}
/**
* Records the context state resulting from the context transition generated
* by a keystroke.
* @param state The context state to record as the result of the transition
* @param inputDistribution Fat-finger data corresponding to the triggering keystroke
* @param preservationTransform Portions of the most likely input that do not contribute to the final token
* in the final context's tokenization.
*/
finalize(state: ContextState, inputDistribution: Distribution<Transform>, preservationTransform?: Transform) {
this._final = state;
this.inputDistribution = inputDistribution;
// Long-term, this should never be null... but we need to allow it at this point
// in the refactoring process.
this._transitionId = inputDistribution?.find((entry) => entry.sample.id !== undefined)?.sample.id;
this.preservationTransform = preservationTransform;
}
}

23
web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts
View file

@ -411,7 +411,7 @@ export class SearchSpace {
private tierOrdering: SearchSpaceTier[] = [];
private selectionQueue: PriorityQueue<SearchSpaceTier>;
private inputSequence: Distribution<Transform>[] = [];
private _inputSequence: Distribution<Transform>[] = [];
private minInputCost: number[] = [];
private rootNode: SearchNode;
@ -455,7 +455,7 @@ export class SearchSpace {
this.buildQueueSpaceComparator();
if(arg1 instanceof SearchSpace) {
this.inputSequence = [].concat(arg1.inputSequence);
this._inputSequence = [].concat(arg1._inputSequence);
this.minInputCost = [].concat(arg1.minInputCost);
this.rootNode = arg1.rootNode;
// Re-use already-checked Nodes.
@ -470,9 +470,9 @@ export class SearchSpace {
const model = arg1;
if(!model) {
throw "The LexicalModel parameter must not be null / undefined.";
throw new Error("The LexicalModel parameter must not be null / undefined.");
} else if(!model.traverseFromRoot) {
throw "The provided model does not implement the `traverseFromRoot` function, which is needed to support robust correction searching.";
throw new Error("The provided model does not implement the `traverseFromRoot` function, which is needed to support robust correction searching.");
}
this.selectionQueue = new PriorityQueue<SearchSpaceTier>(this.QUEUE_SPACE_COMPARATOR);
@ -534,6 +534,13 @@ export class SearchSpace {
}
}
/**
* Retrieves the sequence of inputs
*/
public get inputSequence() {
return [...this._inputSequence];
}
increaseMaxEditDistance() {
this.tierOrdering.forEach(function(tier) { tier.increaseMaxEditDistance() });
}
@ -541,7 +548,7 @@ export class SearchSpace {
get correctionsEnabled() {
// When corrections are disabled, the Web engine will only provide individual Transforms
// for an input, not a distribution. No distributions means we shouldn't do corrections.
return !!this.inputSequence.find((distribution) => distribution.length > 1);
return !!this._inputSequence.find((distribution) => distribution.length > 1);
}
/**
@ -551,7 +558,7 @@ export class SearchSpace {
* just the raw keystroke if corrections are disabled)
*/
addInput(inputDistribution: Distribution<Transform>) {
this.inputSequence.push(inputDistribution);
this._inputSequence.push(inputDistribution);
// Assumes that `inputDistribution` is already sorted.
this.minInputCost.push(-Math.log(inputDistribution[0].p));
@ -683,9 +690,9 @@ export class SearchSpace {
let deletionEdges: SearchNode[] = [];
if(!substitutionsOnly) {
deletionEdges = currentNode.buildDeletionEdges(this.inputSequence[inputIndex-1]);
deletionEdges = currentNode.buildDeletionEdges(this._inputSequence[inputIndex-1]);
}
let substitutionEdges = currentNode.buildSubstitutionEdges(this.inputSequence[inputIndex-1]);
let substitutionEdges = currentNode.buildSubstitutionEdges(this._inputSequence[inputIndex-1]);
// Note: we're live-modifying the tier's cost here! The priority queue loses its guarantees as a result.
nextTier.correctionQueue.enqueueAll(deletionEdges.concat(substitutionEdges));

52
web/src/engine/predictive-text/worker-thread/src/main/correction/transform-tokenization.ts
View file

@ -1,8 +1,10 @@
import { LexicalModelTypes } from '@keymanapp/common-types';
import Context = LexicalModelTypes.Context;
import Distribution = LexicalModelTypes.Distribution;
import LexicalModel = LexicalModelTypes.LexicalModel;
import Transform = LexicalModelTypes.Transform;
import { applyTransform, type Tokenization } from "@keymanapp/models-templates";
import { determineModelTokenizer } from '#./model-helpers.js';
/**
* Determines a tokenization-aware sequence of (`Transform`) edits, one per
@ -85,4 +87,54 @@ export function tokenizeTransformDistribution(
p: transform.p
};
});
}
/**
* Given an incoming distribution of Transforms, this method applies
* `tokenizeTransform` for each, mapping each transform to its tokenized form in
* the returned distribution.
*
* It then filters out all incoming Transforms that do not result in the same
* number of tokens as the "primary input" when applied, as the context-tracker
* and predictive-text engine cannot handle word-breaking divergence well at
* this time.
* @param context
* @param model
* @param transformDistribution
* @returns
*/
export function tokenizeAndFilterDistribution(
context: Context,
model: LexicalModel,
transformDistribution?: Distribution<Transform>
) {
let tokenize = determineModelTokenizer(model);
const inputTransform = transformDistribution?.[0];
let transformTokenLength = 0;
let tokenizedDistribution: Distribution<Transform[]> = null;
if(inputTransform) {
// These two methods apply transforms internally; do not mutate context here.
// This particularly matters for the 'distribution' variant.
// What if a pre-whitespace token has a final substitution as PART of an edit?
// Say, ['apple', ' ', ''] => ['apply', ' ', 'n']
// For now... we can't really handle that case well - modeling the 'e' => 'y' part.
// Will likely require improvements to tokenizeTransform(), which doesn't yet handle
// deleteLeft tokenization for transforms spanning tokens & whitespace.
//
// See: #14361.
// There's a good shot attemptTokenizedAlignment would be useful for it.
transformTokenLength = tokenizeTransform(tokenize, context, inputTransform.sample).length;
tokenizedDistribution = tokenizeTransformDistribution(tokenize, context, transformDistribution);
// Now we update the context used for context-state management based upon our input.
context = applyTransform(inputTransform.sample, context);
// While we lack phrase-based / phrase-oriented prediction support, we'll just extract the
// set that matches the token length that results from our input.
tokenizedDistribution = tokenizedDistribution.filter((entry) => entry.sample.length == transformTokenLength);
}
return tokenizedDistribution;
}

26
web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts
View file

@ -191,12 +191,7 @@ export class ModelCompositor {
// Store the suggestions on the final token of the current context state (if it exists).
// Or, once phrase-level suggestions are possible, on whichever token serves as each prediction's root.
if(postContextState) {
postContextState.tail.replacements = suggestions.map(function(suggestion) {
return {
suggestion: suggestion,
tokenWidth: 1
}
});
postContextState.tokenization.tail.suggestions = suggestions;
}
return suggestions;
@ -204,7 +199,7 @@ export class ModelCompositor {
acceptSuggestion(suggestion: Suggestion, context: Context, postTransform?: Transform): Reversion {
// Step 1: generate and save the reversion's Transform.
let sourceTransform = suggestion.transform;
let sourceTransform = models.buildMergedTransform(suggestion.transform, suggestion.appendedTransform ?? { insert: '', deleteLeft: 0});
let deletedLeftChars = KMWString.substr(context.left, -sourceTransform.deleteLeft, sourceTransform.deleteLeft);
let insertedLength = KMWString.length(sourceTransform.insert);
@ -260,11 +255,14 @@ export class ModelCompositor {
if(this.contextTracker) {
let contextState = this.contextTracker.newest;
if(!contextState) {
contextState = this.contextTracker.analyzeState(this.lexicalModel, context).state;
contextState = this.contextTracker.analyzeState(this.lexicalModel, context).final;
}
contextState.tail.activeReplacementId = suggestion.id;
contextState.tokenization.tail.appliedSuggestionId = suggestion.id;
let acceptedContext = models.applyTransform(suggestion.transform, context);
if(suggestion.appendedTransform) {
acceptedContext = models.applyTransform(suggestion.appendedTransform, acceptedContext);
}
this.contextTracker.analyzeState(this.lexicalModel, acceptedContext);
}
@ -299,7 +297,7 @@ export class ModelCompositor {
for(let c = this.contextTracker.count - 1; c >= 0; c--) {
let contextState = this.contextTracker.item(c);
if(contextState.tail.activeReplacementId == -reversion.id) {
if(contextState.tokenization.tail.appliedSuggestionId == -reversion.id) {
contextMatchFound = true;
break;
}
@ -310,18 +308,16 @@ export class ModelCompositor {
}
// Remove all contexts more recent than the one we're reverting to.
while(this.contextTracker.newest.tail.activeReplacementId != -reversion.id) {
while(this.contextTracker.newest.tokenization.tail.appliedSuggestionId != -reversion.id) {
this.contextTracker.popNewest();
}
this.contextTracker.newest.tail.activeReplacementId = -1;
this.contextTracker.newest.tokenization.tail.appliedSuggestionId = undefined;
// Will need to be modified a bit if/when phrase-level suggestions are implemented.
// Those will be tracked on the first token of the phrase, which won't be the tail
// if they cover multiple tokens.
let suggestions = this.contextTracker.newest.tail.replacements.map(function(trackedSuggestion) {
return trackedSuggestion.suggestion;
});
let suggestions = this.contextTracker.newest.tokenization.tail.suggestions;
suggestions.forEach(function(suggestion) {
// A reversion's transform ID is the additive inverse of its original suggestion;

123
web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts
View file

@ -1,12 +1,17 @@
import * as models from '@keymanapp/models-templates';
import { KMWString } from '@keymanapp/web-utils';
import { LexicalModelTypes } from '@keymanapp/common-types';
import { defaultWordbreaker, WordBreakProperty } from '@keymanapp/models-wordbreakers';
import TransformUtils from './transformUtils.js';
import { determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js';
import { ContextTracker, TrackedContextState } from './correction/context-tracker.js';
import { ContextTracker } from './correction/context-tracker.js';
import { ContextState } from './correction/context-state.js';
import { ExecutionTimer } from './correction/execution-timer.js';
import ModelCompositor from './model-compositor.js';
const searchForProperty = defaultWordbreaker.searchForProperty;
import CasingForm = LexicalModelTypes.CasingForm;
import Context = LexicalModelTypes.Context;
import Distribution = LexicalModelTypes.Distribution;
@ -165,7 +170,7 @@ export async function correctAndEnumerate(
*
* Otherwise, is `null`.
*/
postContextState?: TrackedContextState;
postContextState?: ContextState;
/**
* The suggestions generated based on the user's input state.
@ -236,11 +241,11 @@ export async function correctAndEnumerate(
// facilitates a more thorough correction-search pattern.
// Token replacement benefits greatly from knowledge of the prior context state.
let { state: contextState } = contextTracker.analyzeState(
let contextState = contextTracker.analyzeState(
lexicalModel,
context,
null
);
).final;
// Corrections and predictions are based upon the post-context state, though.
const contextChangeAnalysis = contextTracker.analyzeState(
@ -250,7 +255,7 @@ export async function correctAndEnumerate(
? transformDistribution
: null
);
const postContextState = contextChangeAnalysis.state;
const postContextState = contextChangeAnalysis.final;
// TODO: Should we filter backspaces & whitespaces out of the transform distribution?
// Ideally, the answer (in the future) will be no, but leaving it in right now may pose an issue.
@ -259,7 +264,7 @@ export async function correctAndEnumerate(
// let's just note that right now, there will only ever be one.
//
// The 'eventual' logic will be significantly more complex, though still manageable.
let searchSpace = postContextState.searchSpace[0];
const searchSpace = postContextState.tokenization.tail.searchSpace;
// No matter the prediction, once we know the root of the prediction, we'll always 'replace' the
// same amount of text. We can handle this before the big 'prediction root' loop.
@ -267,9 +272,9 @@ export async function correctAndEnumerate(
// The amount of text to 'replace' depends upon whatever sort of context change occurs
// from the received input.
const postContextTokens = postContextState.tokens;
const postContextTokens = postContextState.tokenization.tokens;
// Only use of `contextState`.
let contextLengthDelta = postContextTokens.length - contextState.tokens.length;
let contextLengthDelta = postContextTokens.length - contextState.tokenization.tokens.length;
// If the context now has more tokens, the token we'll be 'predicting' didn't originally exist.
if(contextChangeAnalysis.preservationTransform) {
// As the word/token being corrected/predicted didn't originally exist, there's no
@ -307,11 +312,11 @@ export async function correctAndEnumerate(
// Did the wordbreaker (or similar) append a blank token before the caret? If so,
// preserve that by preventing corrections from triggering left-deletion.
if(tailToken.raw == '') {
if(tailToken.exampleInput == '') {
deleteLeft = 0;
}
const isTokenStart = tailToken.transformDistributions.length <= 1;
const isTokenStart = tailToken.searchSpace.inputSequence.length <= 1;
// TODO: whitespace, backspace filtering. Do it here.
// Whitespace is probably fine, actually. Less sure about backspace.
@ -700,6 +705,35 @@ export function processSimilarity(
});
}
/**
* This function may be used to prevent auto-selection/auto-correct from applying in
* unexpected ways. For example, when typing numbers in English, we don't expect
* '5' to auto-correct to '5th' just because there are no pure-number entries in
* the lexicon rooted on '5'.
* @param correction
* @returns
*/
export function correctionValidForAutoSelect(correction: string) {
let chars = [...correction];
// If the _correction_ - the actual, existing text - does not include any letters,
// then predictions built upon it should not be considered valid for auto-correction.
for(let c of chars) {
// Found even one letter? We'll consider it valid.
switch(searchForProperty(c.codePointAt(0))) {
case WordBreakProperty.ALetter:
case WordBreakProperty.Hebrew_Letter:
case WordBreakProperty.Katakana:
return true;
default:
}
}
// Only reached when the correction has nothing that passes as a letter in-context.
// (MidLet and MidNumLet only count when there are adjacent letters.)
return false;
}
export function predictionAutoSelect(suggestionDistribution: CorrectionPredictionTuple[]) {
if(suggestionDistribution.length == 0) {
return;
@ -718,6 +752,11 @@ export function predictionAutoSelect(suggestionDistribution: CorrectionPredictio
suggestionDistribution = suggestionDistribution.slice(1);
if(suggestionDistribution.length == 1) {
// Prevent auto-acceptance when the root doesn't meet validation criteria.
if(!correctionValidForAutoSelect(suggestionDistribution[0].correction.sample)) {
return;
}
// Mark for auto-acceptance; there are no alternatives.
suggestionDistribution[0].prediction.sample.autoAccept = true;
return;
@ -761,6 +800,10 @@ export function predictionAutoSelect(suggestionDistribution: CorrectionPredictio
return;
}
if(!correctionValidForAutoSelect(bestSuggestion.correction.sample)) {
return;
}
// compare correction-cost aspects? We disable if the base correction is lower than best,
// but should we do other comparisons too?
@ -829,32 +872,42 @@ export function finalizeSuggestions(
}
});
// Apply 'after word' punctuation and other post-processing, setting suggestion IDs.
// We delay until now so that utility functions relying on the unmodified Transform may execute properly.
suggestions.forEach((suggestion) => {
// Valid 'keep' suggestions may have zero length; we still need to evaluate the following code
// for such cases.
if(punctuation.insertAfterWord !== "") {
// Apply 'after word' punctuation and other post-processing, setting suggestion IDs.
// We delay until now so that utility functions relying on the unmodified Transform may execute properly.
suggestions.forEach((suggestion) => {
// Valid 'keep' suggestions may have zero length; we still need to evaluate the following code
// for such cases.
// If we're mid-word, delete its original post-caret text.
const tokenization = tokenize(context);
if(tokenization && tokenization.caretSplitsToken) {
// While we wait on the ability to provide a more 'ideal' solution, let's at least
// go with a more stable, if slightly less ideal, solution for now.
//
// A predictive text default (on iOS, at least) - immediately wordbreak
// on suggestions accepted mid-word.
suggestion.transform.insert += punctuation.insertAfterWord;
// If we're mid-word, delete its original post-caret text.
const tokenization = tokenize(context);
if(tokenization && tokenization.caretSplitsToken) {
// While we wait on the ability to provide a more 'ideal' solution, let's at least
// go with a more stable, if slightly less ideal, solution for now.
//
// A predictive text default (on iOS, at least) - immediately wordbreak
// on suggestions accepted mid-word.
suggestion.appendedTransform = {
insert: punctuation.insertAfterWord,
deleteLeft: 0
};
// Do we need to manipulate the suggestion's transform based on the current state of the context?
} else if(!context.right) {
suggestion.transform.insert += punctuation.insertAfterWord;
} else if(punctuation.insertAfterWord != '') {
if(context.right.indexOf(punctuation.insertAfterWord) != 0) {
suggestion.transform.insert += punctuation.insertAfterWord;
// Do we need to manipulate the suggestion's transform based on the current state of the context?
} else if(!context.right) {
suggestion.appendedTransform = {
insert: punctuation.insertAfterWord,
deleteLeft: 0
};
} else if(punctuation.insertAfterWord != '') {
if(context.right.indexOf(punctuation.insertAfterWord) != 0) {
suggestion.appendedTransform = {
insert: punctuation.insertAfterWord,
deleteLeft: 0
};
}
}
}
});
});
};
return suggestions;
}
@ -898,6 +951,10 @@ export function toAnnotatedSuggestion(
tag: annotationType,
};
if(suggestion.appendedTransform) {
result.appendedTransform = suggestion.appendedTransform;
}
if(suggestion.transformId !== undefined) {
result.transformId = suggestion.transformId;
}

5
web/src/engine/predictive-text/worker-thread/src/main/test-index.ts
View file

@ -1,5 +1,10 @@
export { ClassicalDistanceCalculation } from './correction/classical-calculation.js';
export { ContextState } from './correction/context-state.js';
export { ContextToken } from './correction/context-token.js';
export { ContextTokenization } from './correction/context-tokenization.js';
export { ContextTracker } from './correction/context-tracker.js';
export { EditOperation } from './correction/classical-calculation.js';
export * from './correction/alignment-helpers.js';
export * as correction from './correction/index.js';
export * from './model-helpers.js';
export * as models from './models/index.js';

13
web/src/test/auto/headless/engine/interfaces/prediction/predictionContext.tests.js
View file

@ -159,21 +159,14 @@ describe("PredictionContext", () => {
assert.equal(updateFake.callCount, 3);
suggestions = updateFake.thirdCall.args[0];
// Note: this unit test was originally written with auto-correct on!
// #11941 was written 2024-07-25 (added unit test for auto-correction method)
// #12169 was written 2024-08-14, which is what added THIS unit test.
// This does re-use the apply-revert oriented mocking.
// Should skip the (second) "apple", "apply", "apps" round, as it became outdated
// by its following request before its response could be received.
assert.deepEqual(suggestions.map((obj) => obj.displayAs), ['applied']); // '“apple”' included with auto-correct enabled.
// Is not displayed; we only display it if auto-correct is on, as 'applied' would be automatic then.
assert.equal(predictiveContext.keepSuggestion.displayAs, '“apple”');
// assert.equal(suggestions.find((obj) => obj.tag == 'keep').displayAs, '“apple”'); // with auto-correct enabled.
assert.equal(suggestions.find((obj) => obj.transform.deleteLeft != 0).displayAs, 'applied');
assert.deepEqual(suggestions.map((obj) => obj.displayAs), ['“apple”', 'applied']);
assert.equal(suggestions.find((obj) => obj.tag == 'keep').displayAs, '“apple”');
// Our reused mocking doesn't directly provide the 'keep' suggestion; we
// need to remove it before testing for set equality.
assert.deepEqual(suggestions /*.splice(1)*/, expected);
assert.deepEqual(suggestions.splice(1), expected);
});
it('sendUpdateState retrieves the most recent suggestion set', async function() {

27
web/src/test/auto/headless/engine/main/headless/languageProcessor.tests.js
View file

@ -117,9 +117,11 @@ describe('LanguageProcessor', function() {
languageProcessor.predict(transcription).then(function(suggestions) {
assert.isOk(suggestions);
assert.equal(suggestions[0].displayAs, '«li»');
assert.equal(suggestions[0].transform.insert, 'li ');
assert.equal(suggestions[0].transform.insert, 'li');
assert.equal(suggestions[0].appendedTransform.insert, ' ');
assert.equal(suggestions[1].displayAs, 'like');
assert.equal(suggestions[1].transform.insert, 'like ');
assert.equal(suggestions[1].transform.insert, 'like');
assert.equal(suggestions[1].appendedTransform.insert, ' ');
done();
}).catch(done);
}).catch(function() {
@ -154,7 +156,8 @@ describe('LanguageProcessor', function() {
languageProcessor.predict(transcription).then(function(suggestions) {
assert.isOk(suggestions);
assert.equal(suggestions[1].displayAs, 'like');
assert.equal(suggestions[1].transform.insert, 'like ');
assert.equal(suggestions[1].transform.insert, 'like');
assert.equal(suggestions[1].appendedTransform.insert, ' ');
done();
}).catch(done);
}).catch(function() {
@ -172,7 +175,8 @@ describe('LanguageProcessor', function() {
// The source suggestion is simply 'like'.
assert.isOk(suggestions);
assert.equal(suggestions[1].displayAs, 'like');
assert.equal(suggestions[1].transform.insert, 'like ');
assert.equal(suggestions[1].transform.insert, 'like');
assert.equal(suggestions[1].appendedTransform.insert, ' ');
done();
}).catch(done);
}).catch(function() {
@ -189,7 +193,8 @@ describe('LanguageProcessor', function() {
languageProcessor.predict(transcription).then(function(suggestions) {
assert.isOk(suggestions);
assert.equal(suggestions[1].displayAs, 'I');
assert.equal(suggestions[1].transform.insert, 'I ');
assert.equal(suggestions[1].transform.insert, 'I');
assert.equal(suggestions[1].appendedTransform.insert, ' ');
done();
}).catch(done);
}).catch(function() {
@ -209,7 +214,8 @@ describe('LanguageProcessor', function() {
// The source suggestion is simply 'like'.
assert.isOk(suggestions);
assert.equal(suggestions[1].displayAs, 'LIKE');
assert.equal(suggestions[1].transform.insert, 'LIKE ');
assert.equal(suggestions[1].transform.insert, 'LIKE');
assert.equal(suggestions[1].appendedTransform.insert, ' ');
done();
}).catch(done);
}).catch(function() {
@ -226,7 +232,8 @@ describe('LanguageProcessor', function() {
languageProcessor.predict(transcription).then(function(suggestions) {
assert.isOk(suggestions);
assert.equal(suggestions[0].displayAs, 'I');
assert.equal(suggestions[0].transform.insert, 'I ');
assert.equal(suggestions[0].transform.insert, 'I');
assert.equal(suggestions[0].appendedTransform.insert, ' ');
done();
}).catch(done);
}).catch(function() {
@ -247,7 +254,8 @@ describe('LanguageProcessor', function() {
// The source suggestion is simply 'like'.
assert.isOk(suggestions);
assert.equal(suggestions[1].displayAs, 'Like');
assert.equal(suggestions[1].transform.insert, 'Like ');
assert.equal(suggestions[1].transform.insert, 'Like');
assert.equal(suggestions[1].appendedTransform.insert, ' ');
done();
}).catch(done);
}).catch(function() {
@ -267,7 +275,8 @@ describe('LanguageProcessor', function() {
// The source suggestion is simply 'like'.
assert.isOk(suggestions);
assert.equal(suggestions[1].displayAs, 'Like');
assert.equal(suggestions[1].transform.insert, 'Like ');
assert.equal(suggestions[1].transform.insert, 'Like');
assert.equal(suggestions[1].appendedTransform.insert, ' ');
done();
}).catch(done);
}).catch(function() {

12
web/src/test/auto/headless/engine/predictive-text/templates/common.tests.ts
View file

@ -22,8 +22,7 @@ describe('Common utility functions', function() {
let final = {
insert: 'applebanana',
deleteLeft: 0,
deleteRight: 0
deleteLeft: 0
};
let mergedTransform = models.buildMergedTransform(apple, banana);
@ -43,8 +42,7 @@ describe('Common utility functions', function() {
let final = {
insert: 'applebanana',
deleteLeft: 2,
deleteRight: 0
deleteLeft: 2
};
let mergedTransform = models.buildMergedTransform(apple, banana);
@ -64,8 +62,7 @@ describe('Common utility functions', function() {
let final = {
insert: 'bananapple', // the 'apple' transform removes the final 'a' from 'banana'.
deleteLeft: 0,
deleteRight: 0
deleteLeft: 0
};
let mergedTransform = models.buildMergedTransform(banana, apple);
@ -85,8 +82,7 @@ describe('Common utility functions', function() {
let final = {
insert: 'bananapple', // the 'apple' transform removes the final 'a' from 'banana'.
deleteLeft: 2,
deleteRight: 0
deleteLeft: 2
};
let mergedTransform = models.buildMergedTransform(banana, apple);

105
web/src/test/auto/headless/engine/predictive-text/worker-thread/context/alignment-helpers.tests.ts Normal file
View file

@ -0,0 +1,105 @@
/*
* Keyman is copyright (C) SIL Global. MIT License.
*
* Created by jahorton on 2025-07-30
*
* This file contains low-level tests designed to validate helper functions
* used when aligning cached context states to incoming contexts and when
* validating potential substitution edit operations.
*/
import { assert } from 'chai';
import { EditOperation, getEditPathLastMatch, isSubstitutionAlignable } from '@keymanapp/lm-worker/test-index';
describe('getEditPathLastMatch', () => {
it('returns the last match when no substitutions exist', () => {
const path: EditOperation[] = ['delete', 'delete', 'match', 'match', 'match', 'match', 'insert'];
assert.equal(getEditPathLastMatch(path), path.lastIndexOf('match'));
});
it('returns the last match when no substitutions exist left of a "match"', () => {
const path: EditOperation[] = ['delete', 'delete', 'match', 'match', 'match', 'match', 'substitute', 'insert'];
assert.equal(getEditPathLastMatch(path), path.lastIndexOf('match'));
});
// is intended to handle application of suggestions.
it('returns the second-to-last match when a substitution exists before final "match"', () => {
// limitation: if there is _anything_ after that last match, the first assertion will fail.
const path: EditOperation[] = ['delete', 'delete', 'match', 'match', 'match', 'substitute', 'match'];
assert.notEqual(getEditPathLastMatch(path), path.lastIndexOf('match'));
assert.equal(getEditPathLastMatch(path), path.lastIndexOf('match', path.lastIndexOf('match')-1));
});
});
describe('isSubstitutionAlignable', () => {
it(`returns true: 'ca' => 'can'`, () => {
assert.isTrue(isSubstitutionAlignable('can', 'ca'));
});
// Leading word in context window starts sliding out of said window.
it(`returns true: 'can' => 'an'`, () => {
assert.isTrue(isSubstitutionAlignable('an', 'can'));
});
// Same edits on both sides: not valid.
it(`returns false: 'apple' => 'grapples'`, () => {
assert.isFalse(isSubstitutionAlignable('grapples', 'apple'));
});
// Edits on one side: valid.
it(`returns true: 'apple' => 'grapple'`, () => {
assert.isTrue(isSubstitutionAlignable('grapple', 'apple'));
});
// Edits on one side: valid.
it(`returns true: 'apple' => 'grapple'`, () => {
assert.isTrue(isSubstitutionAlignable('apples', 'apple'));
});
// Same edits on both sides: not valid.
it(`returns false: 'grapples' => 'apple'`, () => {
assert.isFalse(isSubstitutionAlignable('apple', 'grapples'));
});
// Substitution: not valid when not permitted via parameter.
it(`returns false: 'apple' => 'banana'`, () => {
// edit path: 'insert' ('b' of banana), 'match' (on leading a), rest are 'substitute'.
assert.isFalse(isSubstitutionAlignable('banana', 'apple'));
});
// Substitution: not valid if too much is substituted, even if allowed via parameter.
it(`returns false: 'apple' => 'banana' (subs allowed)`, () => {
// edit path: 'insert' ('b' of banana), 'match' (on leading a), rest are 'substitute'.
// 1 match vs 4 substitute = no bueno. It'd require too niche of a keyboard rule.
assert.isFalse(isSubstitutionAlignable('banana', 'apple', true));
});
it(`returns true: 'a' => 'à' (subs allowed)`, () => {
assert.isTrue(isSubstitutionAlignable('à', 'a', true));
});
// Leading substitution: valid if enough of the remaining word matches.
// Could totally happen from a legit Keyman keyboard rule.
it(`returns true: 'can' => 'van' (subs allowed)`, () => {
assert.isTrue(isSubstitutionAlignable('van', 'can', true));
});
// Trailing substitution: invalid if not allowed.
it(`returns false: 'can' => 'cap' (subs not allowed)`, () => {
assert.isFalse(isSubstitutionAlignable('cap', 'can'));
});
// Trailing substitution: valid.
it(`returns false: 'can' => 'cap' (subs allowed)`, () => {
assert.isTrue(isSubstitutionAlignable('cap', 'can', true));
});
it(`returns true: 'clasts' => 'clasps' (subs allowed)`, () => {
assert.isTrue(isSubstitutionAlignable('clasps', 'clasts', true));
});
// random deletion at the start + later substitution = still permitted
it(`returns false: 'clasts' => 'lasps' (subs allowed)`, () => {
assert.isTrue(isSubstitutionAlignable('lasps', 'clasts', true));
});
});

60
web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts Normal file
View file

@ -0,0 +1,60 @@
/*
* Keyman is copyright (C) SIL Global. MIT License.
*
* Created by jahorton on 2025-08-01
*
* This file tests designed to validate the behavior of ContextState class and
* its integration with the lower-level classes that it utilizes.
*/
import { assert } from 'chai';
import { default as defaultBreaker } from '@keymanapp/models-wordbreakers';
import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs';
import { ContextState, models } from '@keymanapp/lm-worker/test-index';
import TrieModel = models.TrieModel;
var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'),
{wordBreaker: defaultBreaker});
describe('ContextState', () => {
it('<constructor>', () => {
let context = { left: '', right: '', startOfBuffer: true, endOfBuffer: true };
let state = new ContextState(context, plainModel);
assert.equal(state.context, context);
assert.equal(state.model, plainModel);
assert.isOk(state.tokenization);
assert.isUndefined(state.isManuallyApplied);
assert.isNotOk(state.suggestions);
assert.isNotOk(state.appliedSuggestionId);
});
describe('initializing without prior tokenization', () => {
it('creates one empty token for an empty context', () => {
let context = { left: '', right: '', startOfBuffer: true, endOfBuffer: true };
let state = new ContextState(context, plainModel);
assert.isOk(state.tokenization);
assert.equal(state.tokenization.tokens.length, 1);
assert.equal(state.tokenization.tail.exampleInput, '');
});
it('creates tokens for initial text (without ending whitespace)', () => {
let context = { left: 'the quick brown fox', right: '', startOfBuffer: true, endOfBuffer: true };
let state = new ContextState(context, plainModel);
assert.isOk(state.tokenization);
assert.equal(state.tokenization.tokens.length, 7);
assert.deepEqual(state.tokenization.exampleInput, ['the', ' ', 'quick', ' ', 'brown', ' ', 'fox']);
});
it('creates tokens for initial text (with extra empty token for ending whitespace)', () => {
let context = { left: 'the quick brown fox ', right: '', startOfBuffer: true, endOfBuffer: true };
let state = new ContextState(context, plainModel);
assert.isOk(state.tokenization);
assert.equal(state.tokenization.tokens.length, 9);
assert.deepEqual(state.tokenization.exampleInput, ['the', ' ', 'quick', ' ', 'brown', ' ', 'fox', ' ', '']);
});
});
});

109
web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts Normal file
View file

@ -0,0 +1,109 @@
/*
* Keyman is copyright (C) SIL Global. MIT License.
*
* Created by jahorton on 2025-07-30
*
* This file contains low-level unit tests designed to validate the behavior
* of the ContextToken class.
*/
import { assert } from 'chai';
// Aliased due to JS keyword.
import { default as defaultBreaker } from '@keymanapp/models-wordbreakers';
import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs';
import { ContextToken, correction, models } from '@keymanapp/lm-worker/test-index';
import ExecutionTimer = correction.ExecutionTimer;
import TrieModel = models.TrieModel;
var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'),
{wordBreaker: defaultBreaker});
describe('ContextToken', function() {
describe("<constructor>", () => {
it("(model: LexicalModel)", async () => {
let token = new ContextToken(plainModel);
assert.isEmpty(token.searchSpace.inputSequence);
assert.isEmpty(token.exampleInput);
assert.isFalse(token.isWhitespace);
assert.isEmpty(token.suggestions);
assert.isUndefined(token.appliedSuggestionId);
// While searchSpace has no inputs, it _can_ match lexicon entries (via insertions).
let searchIterator = token.searchSpace.getBestMatches(new ExecutionTimer(Number.POSITIVE_INFINITY, Number.POSITIVE_INFINITY));
let firstEntry = await searchIterator.next();
assert.isFalse(firstEntry.done);
});
it("(model: LexicalModel, text: string)", () => {
let token = new ContextToken(plainModel, "and");
assert.isNotEmpty(token.searchSpace.inputSequence);
assert.equal(token.searchSpace.inputSequence.map((entry) => entry[0].sample.insert).join(''), 'and');
token.searchSpace.inputSequence.forEach((entry) => assert.equal(entry[0].sample.deleteLeft, 0));
assert.deepEqual(token.searchSpace.inputSequence, [..."and"].map((char) => {
return [{
sample: {
insert: char,
deleteLeft: 0
},
p: 1.0
}];
}));
assert.equal(token.exampleInput, 'and');
assert.isFalse(token.isWhitespace);
// Is only set with a different value later, outside the constructor.
assert.isEmpty(token.suggestions);
assert.isUndefined(token.appliedSuggestionId);
});
it("(token: ContextToken", () => {
// Same as in a test above, since we verified that it works correctly.
let baseToken = new ContextToken(plainModel, "and");
baseToken.suggestions = [
{
transform: {
insert: 'd ',
deleteLeft: 0
},
id: 37,
transformId: 1,
displayAs: '"and"',
tag: 'keep',
autoAccept: true
},
{
transform: {
insert: 'Andes ',
deleteLeft: 2
},
id: 38,
transformId: 1,
displayAs: 'Andes',
}
]
baseToken.appliedSuggestionId = 37;
let clonedToken = new ContextToken(baseToken);
assert.notEqual(clonedToken.suggestions, baseToken.suggestions);
assert.deepEqual(clonedToken.suggestions, baseToken.suggestions);
assert.notEqual(clonedToken.searchSpace, baseToken.searchSpace);
// Deep equality on .searchSpace can't be directly checked due to the internal complexities involved.
// We CAN check for the most important members, though.
assert.notEqual(clonedToken.searchSpace.inputSequence, baseToken.searchSpace.inputSequence);
assert.deepEqual(clonedToken.searchSpace.inputSequence, baseToken.searchSpace.inputSequence);
assert.notEqual(clonedToken, baseToken);
// Perfectly deep-equal when we ignore .searchSpace.
assert.deepEqual({...clonedToken, searchSpace: null}, {...baseToken, searchSpace: null});
});
});
});

456
web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts Normal file
View file

@ -0,0 +1,456 @@
/*
* Keyman is copyright (C) SIL Global. MIT License.
*
* Created by jahorton on 2025-07-30
*
* This file contains low-level tests designed to validate the behavior of the
* of the ContextTokenization class and its integration with the lower-level
* classes that it utilizes.
*/
import { assert } from 'chai';
import { default as defaultBreaker } from '@keymanapp/models-wordbreakers';
import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs';
import { ContextToken, ContextTokenization, models } from '@keymanapp/lm-worker/test-index';
import TrieModel = models.TrieModel;
var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'),
{wordBreaker: defaultBreaker});
function buildBaseTokenization(textTokens: string[]) {
const tokens = textTokens.map((entry) => new ContextToken(plainModel, entry));
return new ContextTokenization(tokens);
}
function toToken(text: string) {
let isWhitespace = text == ' ';
let token = new ContextToken(plainModel, text);
token.isWhitespace = isWhitespace;
return token;
}
describe('ContextTokenization', function() {
describe("<constructor>", () => {
it("constructs from just a token array", () => {
const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
let tokenization = new ContextTokenization(rawTextTokens.map((text => toToken(text))));
assert.deepEqual(tokenization.tokens.map((entry) => entry.exampleInput), rawTextTokens);
assert.deepEqual(tokenization.tokens.map((entry) => entry.isWhitespace), rawTextTokens.map((entry) => entry == ' '));
assert.isNotOk(tokenization.alignment);
assert.equal(tokenization.tail.exampleInput, 'day');
assert.isFalse(tokenization.tail.isWhitespace);
assert.isUndefined(tokenization.tail.appliedSuggestionId);
});
it("constructs from a token array + alignment data", () => {
const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
let alignment = {
canAlign: true,
leadTokenShift: 0,
matchLength: 6,
tailEditLength: 1,
tailTokenShift: 0
};
let tokenization = new ContextTokenization(rawTextTokens.map((text => toToken(text))), alignment);
assert.deepEqual(tokenization.tokens.map((entry) => entry.exampleInput), rawTextTokens);
assert.deepEqual(tokenization.tokens.map((entry) => entry.isWhitespace), rawTextTokens.map((entry) => entry == ' '));
assert.isOk(tokenization.alignment);
assert.deepEqual(tokenization.alignment, alignment);
assert.equal(tokenization.tail.exampleInput, 'day');
assert.isFalse(tokenization.tail.isWhitespace);
assert.isUndefined(tokenization.tail.appliedSuggestionId);
});
it('clones', () => {
const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
let baseTokenization = new ContextTokenization(rawTextTokens.map((text => toToken(text))), {
canAlign: true,
leadTokenShift: 0,
matchLength: 6,
tailEditLength: 1,
tailTokenShift: 0
});
let cloned = new ContextTokenization(baseTokenization);
assert.notDeepEqual(cloned, baseTokenization);
assert.deepEqual(cloned.tokens.map((token) => token.searchSpace.inputSequence),
baseTokenization.tokens.map((token) => token.searchSpace.inputSequence));
// The `.searchSpace` instances will not be deep-equal; there are class properties
// that hold functions with closures, configured at runtime.
// @ts-ignore - TS2704 b/c deleting a readonly property.
baseTokenization.tokens.forEach((token) => delete token.searchSpace);
// @ts-ignore - TS2704 b/c deleting a readonly property.
cloned.tokens.forEach((token) => delete token.searchSpace);
assert.deepEqual(cloned, baseTokenization);
});
});
it('exampleInput', () => {
const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
let tokenization = new ContextTokenization(rawTextTokens.map((text => toToken(text))));
assert.deepEqual(tokenization.exampleInput, rawTextTokens);
});
describe('computeAlignment', () => {
it("properly matches and aligns when contexts match", () => {
const baseContext = [
'quick', 'brown', 'fox', 'jumped', 'over'
];
const newContext = [...baseContext];
const baseTokenization = buildBaseTokenization(baseContext);
const computedAlignment = baseTokenization.computeAlignment(newContext);
assert.deepEqual(computedAlignment, {
canAlign: true,
leadTokenShift: 0,
matchLength: 5,
tailEditLength: 0,
tailTokenShift: 0
});
});
it("detects unalignable contexts - no matching tokens", () => {
const baseContext = [
'swift', 'tan', 'wolf', 'leaped', 'across'
];
const newContext = [
'quick', 'brown', 'fox', 'jumped', 'over'
];
const baseTokenization = buildBaseTokenization(baseContext);
const computedAlignment = baseTokenization.computeAlignment(newContext);
assert.deepEqual(computedAlignment, {canAlign: false});
});
it("detects unalignable contexts - too many mismatching tokens", () => {
const baseContext = [
'swift', 'tan', 'fox', 'jumped', 'over'
];
const newContext = [
'quick', 'brown', 'fox', 'jumped', 'over'
];
const baseTokenization = buildBaseTokenization(baseContext);
const computedAlignment = baseTokenization.computeAlignment(newContext);
assert.deepEqual(computedAlignment, {canAlign: false});
});
it("fails alignment for leading-edge word substitutions", () => {
const baseContext = [
'swift', 'brown', 'fox', 'jumped', 'over'
];
const newContext = [
'quick', 'brown', 'fox', 'jumped', 'over'
];
const baseTokenization = buildBaseTokenization(baseContext);
const computedAlignment = baseTokenization.computeAlignment(newContext);
assert.deepEqual(computedAlignment, {canAlign: false});
});
it("fails alignment for small leading-edge word substitutions", () => {
const baseContext = [
'quick', 'brown', 'fox', 'jumped', 'over'
];
const newContext = [
'sick', 'brown', 'fox', 'jumped', 'over'
];
const baseTokenization = buildBaseTokenization(baseContext);
const computedAlignment = baseTokenization.computeAlignment(newContext);
assert.deepEqual(computedAlignment, {canAlign: false});
});
it("properly matches and aligns when lead token is modified", () => {
const baseContext = [
'quick', 'brown', 'fox', 'jumped', 'over'
];
const newContext = [
'uick', 'brown', 'fox', 'jumped', 'over'
];
const baseTokenization = buildBaseTokenization(baseContext);
const computedAlignment = baseTokenization.computeAlignment(newContext);
assert.deepEqual(computedAlignment, {
canAlign: true,
leadTokenShift: 0,
matchLength: 5,
tailEditLength: 0,
tailTokenShift: 0
});
});
it("properly matches and aligns when lead token is removed", () => {
const baseContext = [
'quick', 'brown', 'fox', 'jumped', 'over'
];
const newContext = [
'brown', 'fox', 'jumped', 'over'
];
const baseTokenization = buildBaseTokenization(baseContext);
const computedAlignment = baseTokenization.computeAlignment(newContext);
assert.deepEqual(computedAlignment, {
canAlign: true,
leadTokenShift: -1,
matchLength: 4,
tailEditLength: 0,
tailTokenShift: 0
});
});
it("properly matches and aligns when lead token is added", () => {
const baseContext = [
'brown', 'fox', 'jumped', 'over'
];
const newContext = [
'quick', 'brown', 'fox', 'jumped', 'over'
];
const baseTokenization = buildBaseTokenization(baseContext);
const computedAlignment = baseTokenization.computeAlignment(newContext);
assert.deepEqual(computedAlignment, {
canAlign: true,
leadTokenShift: 1,
matchLength: 4,
tailEditLength: 0,
tailTokenShift: 0
});
});
it("properly matches and aligns when lead tokens are removed and modified", () => {
const baseContext = [
'quick', 'brown', 'fox', 'jumped', 'over'
];
const newContext = [
'ox', 'jumped', 'over'
];
const baseTokenization = buildBaseTokenization(baseContext);
const computedAlignment = baseTokenization.computeAlignment(newContext);
assert.deepEqual(computedAlignment, {
canAlign: true,
leadTokenShift: -2,
matchLength: 3,
tailEditLength: 0,
tailTokenShift: 0
});
});
it("properly matches and aligns when lead tokens are added and modified", () => {
const baseContext = [
'rown', 'fox', 'jumped', 'over'
];
const newContext = [
'quick', 'brown', 'fox', 'jumped', 'over'
];
const baseTokenization = buildBaseTokenization(baseContext);
const computedAlignment = baseTokenization.computeAlignment(newContext);
assert.deepEqual(computedAlignment, {
canAlign: true,
leadTokenShift: 1,
matchLength: 4,
tailEditLength: 0,
tailTokenShift: 0
});
});
it("properly matches and aligns when lead token is removed and tail token is added", () => {
const baseContext = [
'quick', 'brown', 'fox', 'jumped', 'over'
];
const newContext = [
'brown', 'fox', 'jumped', 'over', 'the'
];
const baseTokenization = buildBaseTokenization(baseContext);
const computedAlignment = baseTokenization.computeAlignment(newContext);
assert.deepEqual(computedAlignment, {
canAlign: true,
leadTokenShift: -1,
matchLength: 4,
tailEditLength: 0,
tailTokenShift: 1
});
});
it("properly matches and aligns when lead token and tail token are modified", () => {
const baseContext = [
'quick', 'brown', 'fox', 'jumped', 'ove'
];
const newContext = [
'uick', 'brown', 'fox', 'jumped', 'over'
];
const baseTokenization = buildBaseTokenization(baseContext);
const computedAlignment = baseTokenization.computeAlignment(newContext);
assert.deepEqual(computedAlignment, {
canAlign: true,
leadTokenShift: 0,
matchLength: 4, // we treat 'quick' and 'uick' as the same
tailEditLength: 1,
tailTokenShift: 0
});
});
it("properly matches and aligns when lead token and tail token are modified + new token appended", () => {
const baseContext = [
'quick', 'brown', 'fox', 'jumped', 'ove'
];
const newContext = [
'uick', 'brown', 'fox', 'jumped', 'over', 't'
];
const baseTokenization = buildBaseTokenization(baseContext);
const computedAlignment = baseTokenization.computeAlignment(newContext);
assert.deepEqual(computedAlignment, {
canAlign: true,
leadTokenShift: 0,
matchLength: 4, // we treat 'quick' and 'uick' as the same
tailEditLength: 1,
tailTokenShift: 1
});
});
it("properly handles context window sliding backward", () => {
const baseContext = [
'quick', 'brown', 'fox', 'jumped', 'over'
];
const newContext = [
'e', 'quick', 'brown', 'fox', 'jumped', 'ove'
];
const baseTokenization = buildBaseTokenization(baseContext);
const computedAlignment = baseTokenization.computeAlignment(newContext);
assert.deepEqual(computedAlignment, {
canAlign: true,
leadTokenShift: 1,
matchLength: 4, // we treat 'quick' and 'uick' as the same
tailEditLength: 1,
tailTokenShift: 0
});
});
it("properly handles context window sliding far backward", () => {
const baseContext = [
'quick', 'brown', 'fox', 'jumped', 'over'
];
const newContext = [
'the', 'quick', 'brown', 'fox', 'jumped'
];
const baseTokenization = buildBaseTokenization(baseContext);
const computedAlignment = baseTokenization.computeAlignment(newContext);
assert.deepEqual(computedAlignment, {
canAlign: true,
leadTokenShift: 1,
matchLength: 4, // we treat 'quick' and 'uick' as the same
tailEditLength: 0,
tailTokenShift: -1
});
});
it("properly handles context window sliding farther backward", () => {
const baseContext = [
'quick', 'brown', 'fox', 'jumped', 'over'
];
const newContext = [
'the', 'quick', 'brown', 'fox', 'jumpe'
];
const baseTokenization = buildBaseTokenization(baseContext);
const computedAlignment = baseTokenization.computeAlignment(newContext);
assert.deepEqual(computedAlignment, {
canAlign: true,
leadTokenShift: 1,
matchLength: 3, // we treat 'quick' and 'uick' as the same
tailEditLength: 1,
tailTokenShift: -1
});
});
it("fails alignment for mid-head deletion", () => {
const baseContext = [
'quick', 'brown', 'fox', 'jumped', 'over'
];
const newContext = [
'quick', 'fox', 'jumped', 'over'
];
const baseTokenization = buildBaseTokenization(baseContext);
const computedAlignment = baseTokenization.computeAlignment(newContext);
assert.deepEqual(computedAlignment, {canAlign: false});
});
it("fails alignment for mid-head insertion", () => {
const baseContext = [
'quick', 'fox', 'jumped', 'over'
];
const newContext = [
'quick', 'brown', 'fox', 'jumped', 'over'
];
const baseTokenization = buildBaseTokenization(baseContext);
const computedAlignment = baseTokenization.computeAlignment(newContext);
assert.deepEqual(computedAlignment, {canAlign: false});
});
it("fails alignment for mid-tail deletion", () => {
const baseContext = [
'quick', 'brown', 'fox', 'jumped', 'over'
];
const newContext = [
'quick', 'brown', 'fox', 'over'
];
const baseTokenization = buildBaseTokenization(baseContext);
const computedAlignment = baseTokenization.computeAlignment(newContext);
assert.deepEqual(computedAlignment, {canAlign: false});
});
it("fails alignment for mid-tail insertion", () => {
const baseContext = [
'quick', 'brown', 'fox', 'jumped', 'over'
];
const newContext = [
'quick', 'brown', 'fox', 'jumped', 'far', 'over'
];
const baseTokenization = buildBaseTokenization(baseContext);
const computedAlignment = baseTokenization.computeAlignment(newContext);
assert.deepEqual(computedAlignment, {canAlign: false});
});
});
});

390
web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts
View file

@ -1,12 +1,21 @@
/*
* Keyman is copyright (C) SIL Global. MIT License.
*
* Created by jahorton on 2025-07-30
*
* This file contains tests designed to validate the context-caching
* and context-tracking components for the Keyman predictive-text worker.
*/
import { assert } from 'chai';
import { default as defaultBreaker } from '@keymanapp/models-wordbreakers';
import { deepCopy } from '@keymanapp/web-utils';
import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs';
import { LexicalModelTypes } from '@keymanapp/common-types';
import { ContextTracker, determineModelTokenizer, ModelCompositor, models, tokenizeTransformDistribution } from '@keymanapp/lm-worker/test-index';
import { ContextState, ContextTracker, ModelCompositor, models } from '@keymanapp/lm-worker/test-index';
import Suggestion = LexicalModelTypes.Suggestion;
import Transform = LexicalModelTypes.Transform;
import TrieModel = models.TrieModel;
@ -14,265 +23,316 @@ const plainModel = new TrieModel(jsonFixture('models/tries/english-1000'),
{wordBreaker: defaultBreaker}
);
const tokenizer = determineModelTokenizer(new models.DummyModel({wordbreaker: defaultBreaker}));
describe('ContextTracker', function() {
function toWrapperDistribution(transforms: Transform | Transform[]) {
transforms = Array.isArray(transforms) ? transforms : [transforms];
function toWrapperDistribution(transform: Transform) {
return [{
sample: transforms,
sample: transform,
p: 1.0
}];
}
describe('attemptMatchContext', function() {
it("properly matches and aligns when lead token is removed", function() {
let existingContext = models.tokenize(defaultBreaker, {
left: "an apple a day keeps the doctor"
});
let existingContext = {
left: "an apple a day keeps the doctor",
startOfBuffer: true,
endOfBuffer: true
};
let transform: Transform = {
insert: '',
deleteLeft: 0
}
let newContext = deepCopy(existingContext);
newContext.left.splice(0, 1);
};
let newContext = {
left: " apple a day keeps the doctor",
startOfBuffer: true,
endOfBuffer: true
};
let rawTokens = [" ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
assert.isNotNull(newContextMatch?.state);
assert.deepEqual(newContextMatch.state.tokens.map(token => token.raw), rawTokens);
assert.equal(newContextMatch.headTokensRemoved, 1);
assert.equal(newContextMatch.tailTokensAdded, 0);
let baseState = new ContextState(existingContext, plainModel);
let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
assert.isNotNull(newContextMatch?.final);
assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
if(!newContextMatch.final.tokenization.alignment.canAlign) {
// Done this way b/c TS can infer types correctly afterward.
assert.fail("context alignment failed");
}
assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, -1);
assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0);
});
it("properly matches and aligns when lead token + following whitespace are removed", function() {
let existingContext = models.tokenize(defaultBreaker, {
left: "an apple a day keeps the doctor"
});
let existingContext = {
left: "an apple a day keeps the doctor",
startOfBuffer: true,
endOfBuffer: true
};
let transform = {
insert: '',
deleteLeft: 0
}
let newContext = deepCopy(existingContext);
newContext.left.splice(0, 2);
};
let newContext = {
left: "apple a day keeps the doctor",
startOfBuffer: true,
endOfBuffer: true
};
let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
assert.isNotNull(newContextMatch?.state);
assert.deepEqual(newContextMatch.state.tokens.map(token => token.raw), rawTokens);
assert.equal(newContextMatch.headTokensRemoved, 2);
assert.equal(newContextMatch.tailTokensAdded, 0);
let baseState = new ContextState(existingContext, plainModel);
let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
assert.isNotNull(newContextMatch?.final);
assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
if(!newContextMatch.final.tokenization.alignment.canAlign) {
// Done this way b/c TS can infer types correctly afterward.
assert.fail("context alignment failed");
}
assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, -2);
assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0);
});
it("properly matches and aligns when final token is edited", function() {
let existingContext = models.tokenize(defaultBreaker, {
left: "an apple a day keeps the docto"
});
let existingContext = {
left: "an apple a day keeps the docto",
startOfBuffer: true,
endOfBuffer: true
};
let transform: Transform = {
insert: 'r',
deleteLeft: 0
}
let newContext = models.tokenize(defaultBreaker, {
left: "an apple a day keeps the doctor"
});
let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
assert.isNotNull(newContextMatch?.state);
assert.deepEqual(newContextMatch.state.tokens.map(token => token.raw), rawTokens);
assert.equal(newContextMatch.headTokensRemoved, 0);
assert.equal(newContextMatch.tailTokensAdded, 0);
let baseState = new ContextState(existingContext, plainModel);
let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform));
assert.isNotNull(newContextMatch?.final);
assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
if(!newContextMatch.final.tokenization.alignment.canAlign) {
// Done this way b/c TS can infer types correctly afterward.
assert.fail("context alignment failed");
}
assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0);
assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0);
});
// Needs improved context-state management (due to 2x tokens)
it("properly matches and aligns when a 'wordbreak' is added", function() {
let existingContext = models.tokenize(defaultBreaker, {
left: "an apple a day keeps the doctor"
});
let existingContext = {
left: "an apple a day keeps the doctor",
startOfBuffer: true,
endOfBuffer: true
};
let transform = {
insert: ' ',
deleteLeft: 0
}
let newContext = models.tokenize(defaultBreaker, {
left: "an apple a day keeps the doctor "
});
let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
assert.isNotNull(newContextMatch?.state);
assert.deepEqual(newContextMatch.state.tokens.map(token => token.raw), rawTokens);
let baseState = new ContextState(existingContext, plainModel);
let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform));
assert.isNotNull(newContextMatch?.final);
assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
// We want to preserve the added whitespace when predicting a token that follows after it.
assert.deepEqual(newContextMatch.preservationTransform, { insert: ' ', deleteLeft: 0 });
// The 'wordbreak' transform
let state = newContextMatch?.state;
assert.isNotEmpty(state.tokens[state.tokens.length - 2].transformDistributions);
assert.isEmpty(state.tokens[state.tokens.length - 1].transformDistributions);
assert.equal(newContextMatch.headTokensRemoved, 0);
assert.equal(newContextMatch.tailTokensAdded, 2);
let state = newContextMatch?.final;
assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence);
assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence);
if(!newContextMatch.final.tokenization.alignment.canAlign) {
// Done this way b/c TS can infer types correctly afterward.
assert.fail("context alignment failed");
}
assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0);
assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 2);
});
it("properly matches and aligns when a 'wordbreak' is removed via backspace", function() {
let existingContext = models.tokenize(defaultBreaker, {
left: "an apple a day keeps the doctor "
});
let existingContext = {
left: "an apple a day keeps the doctor ",
startOfBuffer: true,
endOfBuffer: true
};
let transform = {
insert: '',
deleteLeft: 1
}
let newContext = models.tokenize(defaultBreaker, {
left: "an apple a day keeps the doctor"
});
let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
assert.isOk(newContextMatch?.state);
assert.deepEqual(newContextMatch?.state.tokens.map(token => token.raw), rawTokens);
let baseState = new ContextState(existingContext, plainModel);
let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform));
assert.isOk(newContextMatch?.final);
assert.deepEqual(newContextMatch?.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
// The 'wordbreak' transform
assert.equal(newContextMatch.headTokensRemoved, 0);
assert.equal(newContextMatch.tailTokensAdded, 0);
if(!newContextMatch.final.tokenization.alignment.canAlign) {
// Done this way b/c TS can infer types correctly afterward.
assert.fail("context alignment failed");
}
assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0);
assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, -2);
});
it("properly matches and aligns when an implied 'wordbreak' occurs (as when following \"'\")", function() {
let existingContext = models.tokenize(defaultBreaker, {
left: "'"
});
let existingContext = {
left: "'",
startOfBuffer: true,
endOfBuffer: true
};
let transform = {
insert: 'a',
deleteLeft: 0
}
let newContext = models.tokenize(defaultBreaker, {
left: "'a"
});
let rawTokens = ["'", "a"];
let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
assert.isNotNull(newContextMatch?.state);
assert.deepEqual(newContextMatch.state.tokens.map(token => token.raw), rawTokens);
let baseState = new ContextState(existingContext, plainModel);
let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform));
assert.isNotNull(newContextMatch?.final);
assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
assert.deepEqual(newContextMatch.preservationTransform, { insert: '', deleteLeft: 0 });
// The 'wordbreak' transform
let state = newContextMatch.state;
assert.isNotEmpty(state.tokens[state.tokens.length - 2].transformDistributions);
assert.isNotEmpty(state.tokens[state.tokens.length - 1].transformDistributions);
let state = newContextMatch.final;
assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence);
assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence);
assert.equal(newContextMatch.headTokensRemoved, 0);
assert.equal(newContextMatch.tailTokensAdded, 1);
if(!newContextMatch.final.tokenization.alignment.canAlign) {
// Done this way b/c TS can infer types correctly afterward.
assert.fail("context alignment failed");
}
assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0);
assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 1);
})
// Needs improved context-state management (due to 2x tokens)
it("properly matches and aligns when lead token is removed AND a 'wordbreak' is added'", function() {
let existingContext = models.tokenize(defaultBreaker, {
left: "an apple a day keeps the doctor"
});
let existingContext = {
left: "an apple a day keeps the doctor",
startOfBuffer: true,
endOfBuffer: true
};
let transform = {
insert: ' ',
deleteLeft: 0
}
let newContext = models.tokenize(defaultBreaker, {
left: "apple a day keeps the doctor "
});
let newContext = {
left: "apple a day keeps the doctor ",
startOfBuffer: true,
endOfBuffer: true
};
let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
assert.isNotNull(newContextMatch?.state);
assert.deepEqual(newContextMatch.state.tokens.map(token => token.raw), rawTokens);
let baseState = new ContextState(existingContext, plainModel);
let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
assert.isNotNull(newContextMatch?.final);
assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
// We want to preserve the added whitespace when predicting a token that follows after it.
assert.deepEqual(newContextMatch.preservationTransform, { insert: ' ', deleteLeft: 0 });
// The 'wordbreak' transform
let state = newContextMatch.state;
assert.isNotEmpty(state.tokens[state.tokens.length - 2].transformDistributions);
assert.isEmpty(state.tokens[state.tokens.length - 1].transformDistributions);
let state = newContextMatch.final;
assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence);
assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence);
assert.equal(newContextMatch.headTokensRemoved, 2);
assert.equal(newContextMatch.tailTokensAdded, 2);
if(!newContextMatch.final.tokenization.alignment.canAlign) {
// Done this way b/c TS can infer types correctly afterward.
assert.fail("context alignment failed");
}
assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, -2);
assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 2);
});
it("properly matches and aligns when initial token is modified AND a 'wordbreak' is added'", function() {
let existingContext = models.tokenize(defaultBreaker, {
left: "an"
});
let existingContext = {
left: "an",
startOfBuffer: true,
endOfBuffer: true
};
let transform = {
insert: 'd ',
deleteLeft: 0
}
let newContext = models.tokenize(defaultBreaker, {
left: "and "
});
let rawTokens = ["and", " ", ""];
let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
let baseState = new ContextState(existingContext, plainModel);
let newContextMatch = ContextTracker.attemptMatchContext(
newContext.left,
baseContextMatch,
tokenizeTransformDistribution(tokenizer, {left: "an", startOfBuffer: true, endOfBuffer: true}, [{sample: transform, p: 1}])
existingContext,
plainModel,
baseState,
[{sample: transform, p: 1}]
);
assert.isNotNull(newContextMatch?.state);
assert.deepEqual(newContextMatch.state.tokens.map(token => token.raw), rawTokens);
assert.isNotNull(newContextMatch?.final);
assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
// We want to preserve all text preceding the new token when applying a suggestion.
assert.deepEqual(newContextMatch.preservationTransform, { insert: 'd ', deleteLeft: 0, deleteRight: 0});
assert.deepEqual(newContextMatch.preservationTransform, { insert: 'd ', deleteLeft: 0});
// The 'wordbreak' transform
let state = newContextMatch.state;
assert.isNotEmpty(state.tokens[state.tokens.length - 2].transformDistributions);
assert.isEmpty(state.tokens[state.tokens.length - 1].transformDistributions);
let state = newContextMatch.final;
assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence);
assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence);
assert.equal(newContextMatch.headTokensRemoved, 0);
assert.equal(newContextMatch.tailTokensAdded, 2);
if(!newContextMatch.final.tokenization.alignment.canAlign) {
// Done this way b/c TS can infer types correctly afterward.
assert.fail("context alignment failed");
}
assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0);
assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 2);
});
it("properly matches and aligns when tail token is modified AND a 'wordbreak' is added'", function() {
let existingContext = models.tokenize(defaultBreaker, {
left: "apple a day keeps the doc"
});
let existingContext = {
left: "apple a day keeps the doc",
startOfBuffer: true,
endOfBuffer: true
};
let transform = {
insert: 'tor ',
deleteLeft: 0
}
let newContext = models.tokenize(defaultBreaker, {
left: "apple a day keeps the doctor "
});
let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
let baseState = new ContextState(existingContext, plainModel);
let newContextMatch = ContextTracker.attemptMatchContext(
newContext.left,
baseContextMatch,
tokenizeTransformDistribution(tokenizer, {left: "apple a day keeps the doc", startOfBuffer: true, endOfBuffer: true}, [{sample: transform, p: 1}])
existingContext,
plainModel,
baseState,
[{sample: transform, p: 1}]
);
assert.isNotNull(newContextMatch?.state);
assert.deepEqual(newContextMatch.state.tokens.map(token => token.raw), rawTokens);
assert.isNotNull(newContextMatch?.final);
assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
// We want to preserve all text preceding the new token when applying a suggestion.
assert.deepEqual(newContextMatch.preservationTransform, { insert: 'tor ', deleteLeft: 0, deleteRight: 0 });
assert.deepEqual(newContextMatch.preservationTransform, { insert: 'tor ', deleteLeft: 0 });
// The 'wordbreak' transform
let state = newContextMatch.state;
assert.isNotEmpty(state.tokens[state.tokens.length - 2].transformDistributions);
assert.isEmpty(state.tokens[state.tokens.length - 1].transformDistributions);
let state = newContextMatch.final;
assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence);
assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence);
assert.equal(newContextMatch.headTokensRemoved, 0);
assert.equal(newContextMatch.tailTokensAdded, 2);
if(!newContextMatch.final.tokenization.alignment.canAlign) {
// Done this way b/c TS can infer types correctly afterward.
assert.fail("context alignment failed");
}
assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0);
assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 2);
});
it('rejects hard-to-handle case: tail token is split into three rather than two', function() {
let baseContext = models.tokenize(defaultBreaker, {
left: "text'"
left: "text'",
startOfBuffer: true,
endOfBuffer: true
});
assert.equal(baseContext.left.length, 1);
let baseContextMatch = ContextTracker.modelContextState(baseContext.left, plainModel);
let baseState = new ContextState({ left: "text'", startOfBuffer: true, endOfBuffer: true }, plainModel);
// Now the actual check.
let newContext = models.tokenize(defaultBreaker, {
left: "text'\""
left: "text'\"",
startOfBuffer: true,
endOfBuffer: true
});
// The reason it's a problem - current internal logic isn't prepared to shift
// from 1 to 3 tokens in a single step.
@ -283,9 +343,10 @@ describe('ContextTracker', function() {
deleteLeft: 0
}
let problemContextMatch = ContextTracker.attemptMatchContext(
newContext.left,
baseContextMatch,
tokenizeTransformDistribution(tokenizer, {left: "text'", startOfBuffer: true, endOfBuffer: true}, [{sample: transform, p: 1}])
{left: "text'", startOfBuffer: true, endOfBuffer: true},
plainModel,
baseState,
[{sample: transform, p: 1}]
);
assert.isNull(problemContextMatch);
});
@ -293,29 +354,27 @@ describe('ContextTracker', function() {
describe('modelContextState', function() {
it('models without final wordbreak', function() {
let tokenized = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"].map((entry) => {
return {
text: entry,
isWhitespace: entry == " "
};
});
let context = {
left: "an apple a day keeps the doctor",
startOfBuffer: true,
endOfBuffer: true
};
let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
let state = ContextTracker.modelContextState(tokenized, plainModel);
assert.deepEqual(state.tokens.map(token => token.raw), rawTokens);
let state = new ContextState(context, plainModel);
assert.deepEqual(state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
});
it('models with final wordbreak', function() {
let tokenized = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""].map((entry) => {
return {
text: entry,
isWhitespace: entry == " "
};
});
let context = {
left: "an apple a day keeps the doctor ",
startOfBuffer: true,
endOfBuffer: true
};
let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
let state = ContextTracker.modelContextState(tokenized, plainModel);
assert.deepEqual(state.tokens.map(token => token.raw), rawTokens);
let state = new ContextState(context, plainModel);
assert.deepEqual(state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
});
});
@ -327,7 +386,7 @@ describe('ContextTracker', function() {
// Needs improved context-state management (due to 2x tokens)
it('tracks an accepted suggestion', function() {
let baseSuggestion = {
let baseSuggestion: Suggestion = {
transform: {
insert: 'world ',
deleteLeft: 3,
@ -357,15 +416,12 @@ describe('ContextTracker', function() {
let compositor = new ModelCompositor(model);
let baseContextMatch = compositor.contextTracker.analyzeState(model, baseContext);
baseContextMatch.state.tail.replacements = [{
suggestion: baseSuggestion,
tokenWidth: 1
}];
baseContextMatch.final.tokenization.tail.suggestions = [ baseSuggestion ];
let reversion = compositor.acceptSuggestion(baseSuggestion, baseContext, postTransform);
// Actual test assertion - was the replacement tracked?
assert.equal(baseContextMatch.state.tail.activeReplacementId, baseSuggestion.id);
assert.equal(baseContextMatch.final.tokenization.tail.appliedSuggestionId, baseSuggestion.id);
assert.equal(reversion.id, -baseSuggestion.id);
// Next step - on the followup context, is the replacement still active?
@ -373,10 +429,10 @@ describe('ContextTracker', function() {
let postContextMatch = compositor.contextTracker.analyzeState(model, postContext);
// Penultimate token corresponds to whitespace, which does not have a 'raw' representation.
assert.equal(postContextMatch.state.tokens[postContextMatch.state.tokens.length - 2].raw, ' ');
assert.equal(postContextMatch.final.tokenization.tokens[postContextMatch.final.tokenization.tokens.length - 2].exampleInput, ' ');
// Final token is empty (follows a wordbreak)
assert.equal(postContextMatch.state.tail.raw, '');
assert.equal(postContextMatch.final.tokenization.tail.exampleInput, '');
});
});
});

93
web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts
View file

@ -20,7 +20,7 @@ describe('predictionAutoSelect', () => {
const predictions: CorrectionPredictionTuple[] = [
{
correction: {
sample: 'apple', // can be null / "mocked out"
sample: 'apple',
p: 1
},
prediction: {
@ -47,11 +47,60 @@ describe('predictionAutoSelect', () => {
assert.isOk(autoselected);
});
it(`does not select suggestions if the root correction has no letters`, () => {
const predictions: CorrectionPredictionTuple[] = [
{
correction: {
sample: '5',
p: 1
},
prediction: {
sample: {
tag: 'keep',
transform: {
insert: '5',
deleteLeft: 0
},
matchesModel: false,
displayAs: '5'
},
p: 0.01
},
totalProb: 0.01
},
{
correction: {
sample: '5',
p: 1
},
prediction: {
sample: {
transform: {
insert: '5th',
deleteLeft: 0
},
matchesModel: true,
displayAs: '5th'
},
p: 0.8
},
totalProb: 0.8
}
];
const originalPredictions = [...predictions];
assert.doesNotThrow(() => predictionAutoSelect(predictions));
assert.sameDeepOrderedMembers(predictions, originalPredictions);
const autoselected = predictions.find((entry) => entry.prediction.sample.autoAccept);
assert.isNotOk(autoselected);
});
it(`does not select solitary 'keep' suggestion that doesn't match the model`, () => {
const predictions: CorrectionPredictionTuple[] = [
{
correction: {
sample: 'appl', // can be null / "mocked out"
sample: 'appl',
p: 1
},
prediction: {
@ -81,7 +130,7 @@ describe('predictionAutoSelect', () => {
it(`selects 'keep' suggestion that does match the model over any alternatives`, () => {
const keepSuggestion: CorrectionPredictionTuple = {
correction: {
sample: 'thin', // can be null / "mocked out"
sample: 'thin',
p: .8
},
prediction: {
@ -101,7 +150,7 @@ describe('predictionAutoSelect', () => {
const highestNonKeepSuggestion: CorrectionPredictionTuple = {
correction: {
sample: 'thin', // can be null / "mocked out"
sample: 'thin',
p: .8
},
prediction: {
@ -122,7 +171,7 @@ describe('predictionAutoSelect', () => {
highestNonKeepSuggestion,
{
correction: {
sample: 'thin', // can be null / "mocked out"
sample: 'thin',
p: .8
},
prediction: {
@ -139,7 +188,7 @@ describe('predictionAutoSelect', () => {
},
{
correction: {
sample: 'thic', // can be null / "mocked out"
sample: 'thic',
p: .2
},
prediction: {
@ -167,7 +216,7 @@ describe('predictionAutoSelect', () => {
it(`selects solitary non-'keep' suggestion when 'keep' does not match model`, () => {
const keepSuggestion: CorrectionPredictionTuple = {
correction: {
sample: 'thin', // can be null / "mocked out"
sample: 'thin',
p: .8
},
prediction: {
@ -191,7 +240,7 @@ describe('predictionAutoSelect', () => {
// Refer to AUTOSELECT_PROPORTION_THRESHOLD in predict-helpers.ts.
const onlyNonKeepSuggestion: CorrectionPredictionTuple = {
correction: {
sample: 'thin', // can be null / "mocked out"
sample: 'thin',
p: .8
},
prediction: {
@ -228,7 +277,7 @@ describe('predictionAutoSelect', () => {
it(`does not select non-'keep' without sufficient winning probability`, () => {
const keepSuggestion: CorrectionPredictionTuple = {
correction: {
sample: 'thin', // can be null / "mocked out"
sample: 'thin',
p: .8
},
prediction: {
@ -252,7 +301,7 @@ describe('predictionAutoSelect', () => {
// Refer to AUTOSELECT_PROPORTION_THRESHOLD in predict-helpers.ts.
const highestNonKeepSuggestion: CorrectionPredictionTuple = {
correction: {
sample: 'thin', // can be null / "mocked out"
sample: 'thin',
p: .8
},
prediction: {
@ -273,7 +322,7 @@ describe('predictionAutoSelect', () => {
highestNonKeepSuggestion,
{
correction: {
sample: 'thin', // can be null / "mocked out"
sample: 'thin',
p: .8
},
prediction: {
@ -290,7 +339,7 @@ describe('predictionAutoSelect', () => {
},
{
correction: {
sample: 'thic', // can be null / "mocked out"
sample: 'thic',
p: .2
},
prediction: {
@ -323,7 +372,7 @@ describe('predictionAutoSelect', () => {
it(`does select non-'keep' with sufficient winning probability`, () => {
const keepSuggestion: CorrectionPredictionTuple = {
correction: {
sample: 'thin', // can be null / "mocked out"
sample: 'thin',
p: .8
},
prediction: {
@ -343,7 +392,7 @@ describe('predictionAutoSelect', () => {
const highestNonKeepSuggestion: CorrectionPredictionTuple = {
correction: {
sample: 'thin', // can be null / "mocked out"
sample: 'thin',
p: .9
},
prediction: {
@ -364,7 +413,7 @@ describe('predictionAutoSelect', () => {
highestNonKeepSuggestion,
{
correction: {
sample: 'thin', // can be null / "mocked out"
sample: 'thin',
p: .9
},
prediction: {
@ -381,7 +430,7 @@ describe('predictionAutoSelect', () => {
},
{
correction: {
sample: 'thic', // can be null / "mocked out"
sample: 'thic',
p: .1
},
prediction: {
@ -412,7 +461,7 @@ describe('predictionAutoSelect', () => {
it('ignores non key-matched suggestions when key-matched suggestions exist', () => {
const keepSuggestion: CorrectionPredictionTuple = {
correction: {
sample: 'cant', // can be null / "mocked out"
sample: 'cant',
p: 1
},
prediction: {
@ -433,7 +482,7 @@ describe('predictionAutoSelect', () => {
const expectedSuggestion: CorrectionPredictionTuple = {
correction: {
sample: 'cant', // can be null / "mocked out"
sample: 'cant',
p: 1
},
prediction: {
@ -455,7 +504,7 @@ describe('predictionAutoSelect', () => {
expectedSuggestion,
{
correction: {
sample: 'cant', // can be null / "mocked out"
sample: 'cant',
p: 1
},
prediction: {
@ -487,7 +536,7 @@ describe('predictionAutoSelect', () => {
it('does not auto-select suggestion if its root correction is not most likely', () => {
const keepSuggestion: CorrectionPredictionTuple = {
correction: {
sample: 'thi', // can be null / "mocked out"
sample: 'thi',
p: .7
},
prediction: {
@ -507,7 +556,7 @@ describe('predictionAutoSelect', () => {
const highestCorrectionSuggestion: CorrectionPredictionTuple = {
correction: {
sample: 'thi', // can be null / "mocked out"
sample: 'thi',
p: .7
},
prediction: {
@ -525,7 +574,7 @@ describe('predictionAutoSelect', () => {
const highestNonKeepSuggestion: CorrectionPredictionTuple = {
correction: {
sample: 'the', // can be null / "mocked out"
sample: 'the',
p: .3
},
prediction: {

25
web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-finalization.tests.ts
View file

@ -167,7 +167,10 @@ describe('finalizeSuggestions', () => {
const { unfinalized, expected } = build_its_is_set();
const finalized = finalizeSuggestions(testModelWithSpacing, unfinalized, context, transform, false);
expected.forEach((entry) => entry.transform.insert += testModelWithSpacing.punctuation.insertAfterWord);
expected.forEach((entry) => entry.appendedTransform = {
insert: testModelWithSpacing.punctuation.insertAfterWord,
deleteLeft: 0
});
assert.sameDeepOrderedMembers(finalized, expected);
});
@ -216,7 +219,10 @@ describe('finalizeSuggestions', () => {
// The character after the caret isn't the whitespace we'd usually insert,
// so we don't swallow it this time.
expected.forEach((entry) => entry.transform.insert += testModelWithSpacing.punctuation.insertAfterWord);
expected.forEach((entry) => entry.appendedTransform = {
insert: testModelWithSpacing.punctuation.insertAfterWord,
deleteLeft: 0
});
assert.sameDeepOrderedMembers(finalized, expected);
});
@ -241,7 +247,10 @@ describe('finalizeSuggestions', () => {
const { unfinalized, expected } = build_its_is_set();
const finalized = finalizeSuggestions(testModelWithSpacing, unfinalized, context, transform, false);
expected.forEach((entry) => entry.transform.insert += testModelWithSpacing.punctuation.insertAfterWord);
expected.forEach((entry) => entry.appendedTransform = {
insert: testModelWithSpacing.punctuation.insertAfterWord,
deleteLeft: 0
});
assert.sameDeepOrderedMembers(finalized, expected);
});
});
@ -340,7 +349,10 @@ describe('finalizeSuggestions', () => {
const { unfinalized, expected } = build_its_is_set('verbose');
const finalized = finalizeSuggestions(testModelWithSpacing, unfinalized, context, transform, /* verbose */ true);
expected.forEach((entry) => entry.transform.insert += testModelWithSpacing.punctuation.insertAfterWord);
expected.forEach((entry) => entry.appendedTransform = {
insert: testModelWithSpacing.punctuation.insertAfterWord,
deleteLeft: 0
});
assert.sameDeepOrderedMembers(finalized, expected);
});
@ -363,7 +375,10 @@ describe('finalizeSuggestions', () => {
const { unfinalized, expected } = build_its_is_set();
const finalized = finalizeSuggestions(testModelWithSpacing, unfinalized, context, transform, false);
expected.forEach((entry) => entry.transform.insert += testModelWithSpacing.punctuation.insertAfterWord);
expected.forEach((entry) => entry.appendedTransform = {
insert: testModelWithSpacing.punctuation.insertAfterWord,
deleteLeft: 0
});
assert.sameDeepOrderedMembers(finalized, expected);
});
});

2
web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-custom-punctuation.tests.ts
View file

@ -94,7 +94,7 @@ describe('Custom Punctuation', function () {
// Check that it has been changed:
for (var i = 0; i < dummySuggestions.length; i++) {
assert.isTrue(suggestions[i].transform.insert.endsWith(' '));
assert.isTrue(suggestions[i].appendedTransform.insert == ' ');
}
});
})

17
web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts
View file

@ -96,13 +96,14 @@ describe('ModelCompositor', function() {
});
assert.isDefined(keep);
assert.equal(keep.transform.insert, 'the ');
assert.equal(keep.transform.insert, 'the');
assert.isDefined(keep.appendedTransform?.insert, ' ');
// Expect an appended space.
let expectedEntries = ['they ', 'there ', 'their ', 'these ', 'themselves '];
let expectedEntries = ['they', 'there', 'their', 'these', 'themselves'];
expectedEntries.forEach(function(entry) {
assert.isDefined(suggestions.find(function(suggestion) {
return suggestion.transform.insert == entry;
return suggestion.transform.insert == entry && suggestion.appendedTransform?.insert == ' ';
}));
});
});
@ -813,10 +814,14 @@ describe('ModelCompositor', function() {
assert.equal(suggestions.length, 1);
let expectedTransform = {
insert: 'hi ', // Keeps current context the same, though it adds a wordbreak.
insert: 'hi', // Keeps current context the same, though it adds a wordbreak.
deleteLeft: 2
}
assert.deepEqual(suggestions[0].transform, expectedTransform);
assert.deepEqual(suggestions[0].appendedTransform, {
insert: ' ',
deleteLeft: 0
});
});
it('model with traversals: returns appropriate suggestions upon reversion', async function() {
@ -893,7 +898,7 @@ describe('ModelCompositor', function() {
assert.equal(compositor.contextTracker.count, 3);
// The replacement should be marked on the context-tracking token.
assert.isOk(suggestionContextState.tail.replacement);
assert.isAtLeast(suggestionContextState.tokenization.tail.appliedSuggestionId, 0);
let appliedContext = models.applyTransform(baseSuggestion.transform, baseContext);
compositor.applyReversion(reversion, appliedContext);
@ -903,7 +908,7 @@ describe('ModelCompositor', function() {
assert.equal(compositor.contextTracker.item(1), suggestionContextState);
// The replacement should no longer be marked for the context-tracking token.
assert.isNotOk(suggestionContextState.tail.replacement);
assert.isNotOk(suggestionContextState.tokenization.tail.appliedSuggestionId);
});
});
});