erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-11-05 18:44:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Merge branch 'change/web/adjust-tokenization-corrector-spec' into change/web/remove-unused-transition-edit-field
Some checks failed
Keyman Build Summary / Summarize build status checks (push) Has been cancelled
Details

Browse source
This commit is contained in:
Joshua Horton 2026-06-29 13:06:11 -05:00
commit 8bdebac13a
11 changed files with 103 additions and 99 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-corrector.ts
View file

@ -47,8 +47,16 @@ export type TokenResult = {
* range.
*/
export class TokenizationCorrector implements CorrectionSearchable<TokenizationResult, TokenizationResultMapping> {
/**
* The root ContextTokenization all generated corrections are based upon.
*/
public readonly tokenization: ContextTokenization;
private readonly tailCorrectionLength: number;
/**
* Indicates whether or not the correction range of this TokenizationCorrector
* started with any tokens considered "correctable".
*/
public readonly modelsCorrectables: boolean;
// public read-only via properties
private readonly _uncorrectables: QuotientNodeFinalizer[];
@ -60,7 +68,6 @@ export class TokenizationCorrector implements CorrectionSearchable<TokenizationR
private _correctablesMatched = 0;
// fully private
public readonly modelsCorrectables: boolean;
private selectionQueue: PriorityQueue<QuotientNodeFinalizer>;
private tokenCostMap: Map<number, number>;
private tokenLookupMap: Map<number, ContextToken>;
@ -68,6 +75,7 @@ export class TokenizationCorrector implements CorrectionSearchable<TokenizationR
private handleHasBeenCalled: boolean = false;
private predictableMatchFound: boolean = false;
private matchableTokenCount = 0;
private readonly tailCorrectionLength: number;
get currentCost(): number {
const correctable = this.selectionQueue.peek();

4
web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts
View file

@ -1,7 +1,7 @@
import * as models from '@keymanapp/models-templates';
import { LexicalModelTypes } from '@keymanapp/common-types';
import { applySuggestionCasing, compositeIntermediatePredictions, correctAndEnumerate, createDefaultKeep, dedupeSuggestions, finalizeSuggestions, predictionAutoSelect, processSimilarity, toAnnotatedSuggestion, tupleDisplayOrderSort } from './predict-helpers.js';
import { applySuggestionCasing, composeIntermediatePredictions, correctAndEnumerate, createDefaultKeep, dedupeSuggestions, finalizeSuggestions, predictionAutoSelect, processSimilarity, toAnnotatedSuggestion, tupleDisplayOrderSort } from './predict-helpers.js';
import { determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js';
import { ContextTracker } from './correction/context-tracker.js';
@ -151,7 +151,7 @@ export class ModelCompositor {
// We want to dedupe before trimming the list so that we can present a full set
// of viable distinct suggestions if available.
const deduplicatedSuggestionTuples = dedupeSuggestions(this.lexicalModel, compositeIntermediatePredictions(rawPredictions), context);
const deduplicatedSuggestionTuples = dedupeSuggestions(this.lexicalModel, composeIntermediatePredictions(rawPredictions), context);
// Needs "casing" to be applied first.
const postContext = postContextState?.context ?? models.applyTransform(inputTransform, context);

70
web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts
View file

@ -17,7 +17,6 @@ import { TokenResult } from './correction/tokenization-corrector.js';
import { TokenizationCorrector } from './correction/tokenization-corrector.js';
import { TokenizationResultMapping } from './correction/tokenization-result-mapping.js';
import CasingForm = LexicalModelTypes.CasingForm;
import Context = LexicalModelTypes.Context;
import Distribution = LexicalModelTypes.Distribution;
import Keep = LexicalModelTypes.Keep;
@ -189,7 +188,7 @@ export interface PredictionMetadata {
preservationTransform?: Transform;
}
export interface IntermediateTokenizedPrediction {
export interface TokenizedIntermediatePrediction {
/**
* Contains the tokenized components to be used to construct a full
* predictive-text Suggestion, as well as data about the source for each
@ -202,7 +201,7 @@ export interface IntermediateTokenizedPrediction {
metadata: PredictionMetadata;
}
export interface IntermediateCompositedPrediction {
export interface CompositedIntermediatePrediction {
/**
* Contains the fully composited predictive-text Suggestion and its underlying correction string.
*/
@ -213,7 +212,7 @@ export interface IntermediateCompositedPrediction {
metadata: PredictionMetadata;
}
type IntermediatePrediction = IntermediateCompositedPrediction | IntermediateTokenizedPrediction;
type IntermediatePrediction = CompositedIntermediatePrediction | TokenizedIntermediatePrediction;
/**
* An enum to be used when categorizing the level of similarity between
@ -528,7 +527,7 @@ export interface PredictionParameters {
* @param entry
* @returns
*/
applyInPost: (entry: IntermediateTokenizedPrediction) => void;
applyInPost: (entry: TokenizedIntermediatePrediction) => void;
}
export function buildCorrectionSequence(
@ -620,7 +619,7 @@ export function determineTokenizedCorrectionSequence(
return {
...suggestionParams,
applyInPost: (entry: IntermediateTokenizedPrediction) => {
applyInPost: (entry: TokenizedIntermediatePrediction) => {
entry.metadata.preservationTransform = tokenization.taillessTrueKeystroke;
// // Will need an extra lookup layer if the suggestion is generated from within a cluster.
// entry.baseTokenization = transition.final.tokenizationSourceMap.get(tokenization);
@ -655,7 +654,7 @@ export async function correctAndEnumerate(
/**
* The suggestions generated based on the user's input state.
*/
rawPredictions: IntermediateTokenizedPrediction[];
rawPredictions: TokenizedIntermediatePrediction[];
/**
* The id of a prior ContextTransition event that triggered a Suggestion found
@ -710,7 +709,7 @@ export async function correctAndEnumerate(
const searchModules = tokenizations.map(t => t.tail.searchModule);
// Only run the correction search when corrections are enabled.
let rawPredictions: IntermediateTokenizedPrediction[] = [];
let rawPredictions: TokenizedIntermediatePrediction[] = [];
let bestCorrectionCost: number;
for await(const match of getBestTokenMatches(searchModules, timer)) {
// Corrections obtained: now to predict from them!
@ -762,7 +761,7 @@ export async function correctAndEnumerate(
export function shouldStopSearchingEarly(
bestCorrectionCost: number,
currentCorrectionCost: number,
rawPredictions: IntermediateTokenizedPrediction[]
rawPredictions: TokenizedIntermediatePrediction[]
) {
if(currentCorrectionCost >= bestCorrectionCost + CORRECTION_SEARCH_THRESHOLDS.MAX_SEARCH_THRESHOLD) {
return true;
@ -795,18 +794,13 @@ export function shouldStopSearchingEarly(
* "rank" and select the best predictions once the search is complete. This is
* performed at later stages.
* @param lexicalModel
* @param corrections Each `correction` should insert a full token's text to be
* appended to the context resulting from all preceding corrections.
* @param rootContext This context should represent all portions of the
* post-context not represented by the entries of the `corrections` array.
* @param transitionId Indicates the unique ID of the transition that triggered
* prediction generation.
* @param predictionPrep
* @returns
*/
export function predictFromCorrectionSequence(
lexicalModel: LexicalModel,
predictionPrep: PredictionParameters
): IntermediateTokenizedPrediction[] {
): TokenizedIntermediatePrediction[] {
let successfulPredictions = 0;
const correctionTokens = predictionPrep.tokens;
@ -901,10 +895,10 @@ export function predictFromCorrectionSequence(
.slice(0, predictionComponents.length-1)
.map((p) => p[0]);
const completePredictionTuples: IntermediateTokenizedPrediction[] = predictionComponents[predictionComponents.length-1].map((tuple) => {
const completePredictionTuples: TokenizedIntermediatePrediction[] = predictionComponents[predictionComponents.length-1].map((tuple) => {
const predictionCost = tuple.predictionProb * prefixProb;
const returnVal: IntermediateTokenizedPrediction = {
const returnVal: TokenizedIntermediatePrediction = {
components: [...predictionPrefix, tuple],
metadata: {
probabilities: {
@ -930,33 +924,28 @@ export function predictFromCorrectionSequence(
/**
* Applies the specified casing-form to generated suggestions, leveraging the model's
* defined casing behaviors to do so.
* @param suggestion
* @param baseWord
* @param predictionToken A tuple representing a single context token's prediction and base correction
* @param lexicalModel
* @param casingForm
*/
export function applySuggestionCasing(predictionToken: TokenizedPredictionData, lexicalModel: LexicalModel) {
const suggestion = predictionToken.prediction;
// Step 0: our pattern for generating predictions and corrections already
// enforces them to encompass the whole word.
// Step 1: detect the original token's casing
let casingForm: CasingForm;
// enforces that they encompass the whole word.
const suggestion = predictionToken.prediction;
// If we are using the context-tracking engine (when traversals are enabled),
// we just leverage the context token's exampleInput to determine casing.
//
// If it's not available, the correction entry reflects a word-broken piece of
// the original context, with its original casing - so we use that instead.
let casingRoot = predictionToken.casingRoot ? predictionToken.casingRoot : predictionToken.correction;
const casingRoot = predictionToken.casingRoot ? predictionToken.casingRoot : predictionToken.correction;
if(!casingRoot) {
// There's no text in place to verify casing expectations; just leave it
// unchanged.
return;
}
casingForm = detectCurrentCasing(lexicalModel, {
// Step 1: detect the original token's casing
const casingForm = detectCurrentCasing(lexicalModel, {
left: casingRoot,
startOfBuffer: true,
endOfBuffer: true
@ -969,7 +958,14 @@ export function applySuggestionCasing(predictionToken: TokenizedPredictionData,
}
}
export function compositeIntermediatePredictions(predictions: IntermediateTokenizedPrediction[]): IntermediateCompositedPrediction[] {
/**
* Composes a set of `IntermediateTokenizedPrediction`s, merging the tokenized
* data into corresponding `CompositedIntermediatePrediction`s representing the
* full range of affected context.
* @param predictions
* @returns
*/
export function composeIntermediatePredictions(predictions: TokenizedIntermediatePrediction[]): CompositedIntermediatePrediction[] {
return predictions.map((predictionData) => {
const components = predictionData.components;
@ -1013,13 +1009,13 @@ export function compositeIntermediatePredictions(predictions: IntermediateTokeni
*/
export function dedupeSuggestions(
lexicalModel: LexicalModel,
rawPredictions: IntermediateCompositedPrediction[],
rawPredictions: CompositedIntermediatePrediction[],
context: Context
) {
const wordbreak = determineModelWordbreaker(lexicalModel);
let suggestionDistribMap: {[key: string]: IntermediateCompositedPrediction} = {};
let suggestionDistribution: IntermediateCompositedPrediction[] = [];
let suggestionDistribMap: {[key: string]: CompositedIntermediatePrediction} = {};
let suggestionDistribution: CompositedIntermediatePrediction[] = [];
// Deduplicator + annotator of 'keep' suggestions.
for(let tuple of rawPredictions) {
@ -1070,7 +1066,7 @@ export function dedupeSuggestions(
*/
export function processSimilarity(
lexicalModel: LexicalModel,
suggestionDistribution: IntermediateCompositedPrediction[],
suggestionDistribution: CompositedIntermediatePrediction[],
baseContext: Context,
finalContext: Context
): boolean {
@ -1141,7 +1137,7 @@ export function createDefaultKeep(
lexicalModel: LexicalModel,
context: Context,
trueInput: ProbabilityMass<Transform>
): IntermediateCompositedPrediction {
): CompositedIntermediatePrediction {
const { sample: inputTransform, p: inputTransformProb } = trueInput;
const wordbreak = determineModelWordbreaker(lexicalModel);
const tokenizer = determineModelTokenizer(lexicalModel);
@ -1223,7 +1219,7 @@ export function correctionValidForAutoSelect(correction: string) {
return false;
}
export function predictionAutoSelect(suggestionDistribution: IntermediateCompositedPrediction[]) {
export function predictionAutoSelect(suggestionDistribution: CompositedIntermediatePrediction[]) {
if(suggestionDistribution.length == 0) {
return;
}
@ -1317,7 +1313,7 @@ export function predictionAutoSelect(suggestionDistribution: IntermediateComposi
*/
export function finalizeSuggestions(
lexicalModel: LexicalModel,
deduplicatedSuggestionTuples: IntermediateCompositedPrediction[],
deduplicatedSuggestionTuples: CompositedIntermediatePrediction[],
context: Context,
inputTransform: Transform,
verbose?: boolean

4
web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/early-correction-search-stopping.tests.ts
View file

@ -1,6 +1,6 @@
import { assert } from 'chai';
import { CORRECTION_SEARCH_THRESHOLDS, IntermediateTokenizedPrediction, ModelCompositor, shouldStopSearchingEarly } from "@keymanapp/lm-worker/test-index";
import { CORRECTION_SEARCH_THRESHOLDS, TokenizedIntermediatePrediction, ModelCompositor, shouldStopSearchingEarly } from "@keymanapp/lm-worker/test-index";
function mockTokenizedPrediction(value: number) {
return {
@ -9,7 +9,7 @@ function mockTokenizedPrediction(value: number) {
total: value
}
}
} as IntermediateTokenizedPrediction
} as TokenizedIntermediatePrediction
}
describe('correction-search: shouldStopSearchingEarly', () => {

48
web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/auto-correct.tests.ts
View file

@ -1,6 +1,6 @@
import { assert } from 'chai';
import { AUTOSELECT_PROPORTION_THRESHOLD, IntermediateCompositedPrediction, predictionAutoSelect, SuggestionSimilarity, tupleDisplayOrderSort } from "@keymanapp/lm-worker/test-index";
import { AUTOSELECT_PROPORTION_THRESHOLD, CompositedIntermediatePrediction, predictionAutoSelect, SuggestionSimilarity, tupleDisplayOrderSort } from "@keymanapp/lm-worker/test-index";
/*
* Preconditions:
* - there should always be a 'keep' option. Now, whether or not that option
@ -9,7 +9,7 @@ import { AUTOSELECT_PROPORTION_THRESHOLD, IntermediateCompositedPrediction, pred
*/
describe('predictionAutoSelect', () => {
it(`does not throw when no suggestions are available`, () => {
const predictions: IntermediateCompositedPrediction[] = [];
const predictions: CompositedIntermediatePrediction[] = [];
const originalPredictions = [].concat(predictions);
assert.doesNotThrow(() => predictionAutoSelect(predictions));
@ -17,7 +17,7 @@ describe('predictionAutoSelect', () => {
});
it(`selects solitary 'keep' suggestion that does match the model`, () => {
const predictions: IntermediateCompositedPrediction[] = [
const predictions: CompositedIntermediatePrediction[] = [
{
components: {
prediction: {
@ -51,7 +51,7 @@ describe('predictionAutoSelect', () => {
});
it(`does not select suggestions if the root correction has no letters`, () => {
const predictions: IntermediateCompositedPrediction[] = [
const predictions: CompositedIntermediatePrediction[] = [
{
components: {
prediction: {
@ -106,7 +106,7 @@ describe('predictionAutoSelect', () => {
});
it(`does not select solitary 'keep' suggestion that doesn't match the model`, () => {
const predictions: IntermediateCompositedPrediction[] = [
const predictions: CompositedIntermediatePrediction[] = [
{
components: {
prediction: {
@ -140,7 +140,7 @@ describe('predictionAutoSelect', () => {
});
it(`selects 'keep' suggestion that does match the model over any alternatives`, () => {
const keepSuggestion: IntermediateCompositedPrediction = {
const keepSuggestion: CompositedIntermediatePrediction = {
components: {
prediction: {
tag: 'keep',
@ -163,7 +163,7 @@ describe('predictionAutoSelect', () => {
}
}
const highestNonKeepSuggestion: IntermediateCompositedPrediction = {
const highestNonKeepSuggestion: CompositedIntermediatePrediction = {
components: {
prediction: {
transform: { // can be null / "mocked out"
@ -184,7 +184,7 @@ describe('predictionAutoSelect', () => {
}
};
const predictions: IntermediateCompositedPrediction[] = [
const predictions: CompositedIntermediatePrediction[] = [
keepSuggestion,
highestNonKeepSuggestion,
{
@ -238,7 +238,7 @@ describe('predictionAutoSelect', () => {
});
it(`selects solitary non-'keep' suggestion when 'keep' does not match model`, () => {
const keepSuggestion: IntermediateCompositedPrediction = {
const keepSuggestion: CompositedIntermediatePrediction = {
components: {
prediction: {
tag: 'keep',
@ -265,7 +265,7 @@ describe('predictionAutoSelect', () => {
// This threshold may be subject to change.
//
// Refer to AUTOSELECT_PROPORTION_THRESHOLD in predict-helpers.ts.
const onlyNonKeepSuggestion: IntermediateCompositedPrediction = {
const onlyNonKeepSuggestion: CompositedIntermediatePrediction = {
components: {
prediction: {
transform: { // can be null / "mocked out"
@ -286,7 +286,7 @@ describe('predictionAutoSelect', () => {
}
};
const predictions: IntermediateCompositedPrediction[] = [
const predictions: CompositedIntermediatePrediction[] = [
keepSuggestion,
onlyNonKeepSuggestion
];
@ -305,7 +305,7 @@ describe('predictionAutoSelect', () => {
});
it(`does not select non-'keep' without sufficient winning probability`, () => {
const keepSuggestion: IntermediateCompositedPrediction = {
const keepSuggestion: CompositedIntermediatePrediction = {
components: {
prediction: {
tag: 'keep',
@ -332,7 +332,7 @@ describe('predictionAutoSelect', () => {
// This threshold may be subject to change.
//
// Refer to AUTOSELECT_PROPORTION_THRESHOLD in predict-helpers.ts.
const highestNonKeepSuggestion: IntermediateCompositedPrediction = {
const highestNonKeepSuggestion: CompositedIntermediatePrediction = {
components: {
prediction: {
transform: { // can be null / "mocked out"
@ -353,7 +353,7 @@ describe('predictionAutoSelect', () => {
}
};
const predictions: IntermediateCompositedPrediction[] = [
const predictions: CompositedIntermediatePrediction[] = [
keepSuggestion,
highestNonKeepSuggestion,
{
@ -412,7 +412,7 @@ describe('predictionAutoSelect', () => {
});
it(`does select non-'keep' with sufficient winning probability`, () => {
const keepSuggestion: IntermediateCompositedPrediction = {
const keepSuggestion: CompositedIntermediatePrediction = {
components: {
prediction: {
tag: 'keep',
@ -435,7 +435,7 @@ describe('predictionAutoSelect', () => {
}
}
const highestNonKeepSuggestion: IntermediateCompositedPrediction = {
const highestNonKeepSuggestion: CompositedIntermediatePrediction = {
components: {
prediction: {
transform: { // can be null / "mocked out"
@ -456,7 +456,7 @@ describe('predictionAutoSelect', () => {
}
};
const predictions: IntermediateCompositedPrediction[] = [
const predictions: CompositedIntermediatePrediction[] = [
keepSuggestion,
highestNonKeepSuggestion,
{
@ -513,7 +513,7 @@ describe('predictionAutoSelect', () => {
});
it('ignores non key-matched suggestions when key-matched suggestions exist', () => {
const keepSuggestion: IntermediateCompositedPrediction = {
const keepSuggestion: CompositedIntermediatePrediction = {
components: {
prediction: {
tag: 'keep',
@ -537,7 +537,7 @@ describe('predictionAutoSelect', () => {
}
}
const expectedSuggestion: IntermediateCompositedPrediction = {
const expectedSuggestion: CompositedIntermediatePrediction = {
components: {
prediction: {
transform: { // can be null / "mocked out"
@ -559,7 +559,7 @@ describe('predictionAutoSelect', () => {
}
};
const predictions: IntermediateCompositedPrediction[] = [
const predictions: CompositedIntermediatePrediction[] = [
keepSuggestion,
expectedSuggestion,
{
@ -597,7 +597,7 @@ describe('predictionAutoSelect', () => {
// The idea: avoid "over-correcting" when a potential correction has a
// super-high-frequency word.
it('does not auto-select suggestion if its root correction is not most likely', () => {
const keepSuggestion: IntermediateCompositedPrediction = {
const keepSuggestion: CompositedIntermediatePrediction = {
components: {
prediction: {
tag: 'keep',
@ -620,7 +620,7 @@ describe('predictionAutoSelect', () => {
}
}
const highestCorrectionSuggestion: IntermediateCompositedPrediction = {
const highestCorrectionSuggestion: CompositedIntermediatePrediction = {
components: {
prediction: {
transform: { // can be null / "mocked out"
@ -641,7 +641,7 @@ describe('predictionAutoSelect', () => {
}
};
const highestNonKeepSuggestion: IntermediateCompositedPrediction = {
const highestNonKeepSuggestion: CompositedIntermediatePrediction = {
components: {
prediction: {
transform: { // can be null / "mocked out"
@ -662,7 +662,7 @@ describe('predictionAutoSelect', () => {
}
};
const predictions: IntermediateCompositedPrediction[] = [
const predictions: CompositedIntermediatePrediction[] = [
keepSuggestion,
highestNonKeepSuggestion,
highestCorrectionSuggestion

16
web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts
View file

@ -12,7 +12,7 @@ import { assert } from 'chai';
import { LexicalModelTypes } from "@keymanapp/common-types";
import * as wordBreakers from '@keymanapp/models-wordbreakers';
import { IntermediateCompositedPrediction, createDefaultKeep, models, SuggestionSimilarity } from "@keymanapp/lm-worker/test-index";
import { CompositedIntermediatePrediction, createDefaultKeep, models, SuggestionSimilarity } from "@keymanapp/lm-worker/test-index";
import CasingFunction = LexicalModelTypes.CasingFunction;
import Context = LexicalModelTypes.Context;
@ -102,7 +102,7 @@ describe('createDefaultKeep', () => {
endOfBuffer: true
};
const expectedKeep: IntermediateCompositedPrediction = {
const expectedKeep: CompositedIntermediatePrediction = {
components: {
prediction: {
transform: {
@ -148,7 +148,7 @@ describe('createDefaultKeep', () => {
p: 1
};
const expectedKeep: IntermediateCompositedPrediction = {
const expectedKeep: CompositedIntermediatePrediction = {
components: {
prediction: {
transform: {
@ -192,7 +192,7 @@ describe('createDefaultKeep', () => {
p: 1
};
const expectedKeep: IntermediateCompositedPrediction = {
const expectedKeep: CompositedIntermediatePrediction = {
components: {
prediction: {
transform: {
@ -236,7 +236,7 @@ describe('createDefaultKeep', () => {
p: 1
};
const expectedKeep: IntermediateCompositedPrediction = {
const expectedKeep: CompositedIntermediatePrediction = {
components: {
prediction: {
transform: {
@ -280,7 +280,7 @@ describe('createDefaultKeep', () => {
p: 1
};
const expectedKeep: IntermediateCompositedPrediction = {
const expectedKeep: CompositedIntermediatePrediction = {
components: {
prediction: {
transform: {
@ -324,7 +324,7 @@ describe('createDefaultKeep', () => {
p: 1
};
const expectedKeep: IntermediateCompositedPrediction = {
const expectedKeep: CompositedIntermediatePrediction = {
components: {
prediction: {
transform: {
@ -368,7 +368,7 @@ describe('createDefaultKeep', () => {
p: 1
};
const expectedKeep: IntermediateCompositedPrediction = {
const expectedKeep: CompositedIntermediatePrediction = {
components: {
prediction: {
transform: {

4
web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts
View file

@ -21,7 +21,7 @@ import {
ContextState,
ContextToken,
ContextTokenization,
IntermediateTokenizedPrediction,
TokenizedIntermediatePrediction,
ModelCompositor,
TokenizationResultMapping
} from "@keymanapp/lm-worker/test-index";
@ -383,7 +383,7 @@ describe('determineTokenizedCorrectionSequence', () => {
autoSelectable: true
}]);
const dummiedTuple: IntermediateTokenizedPrediction = {
const dummiedTuple: TokenizedIntermediatePrediction = {
components: [{
prediction: {
transform: { insert: 'dog', deleteLeft: 0 },

4
web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts
View file

@ -13,7 +13,7 @@ import { LexicalModelTypes } from "@keymanapp/common-types";
import * as wordBreakers from '@keymanapp/models-wordbreakers';
import { KMWString } from 'keyman/common/web-utils';
import { determineTraversallessCorrectionSequences, IntermediateTokenizedPrediction, ModelCompositor, models } from "@keymanapp/lm-worker/test-index";
import { determineTraversallessCorrectionSequences, TokenizedIntermediatePrediction, ModelCompositor, models } from "@keymanapp/lm-worker/test-index";
import Context = LexicalModelTypes.Context;
import DummyModel = models.DummyModel;
@ -433,7 +433,7 @@ describe('determineTraversallessCorrectionSequences', () => {
autoSelectable: true
}]);
const dummiedTuple: IntermediateTokenizedPrediction = {
const dummiedTuple: TokenizedIntermediatePrediction = {
components: [{
prediction: {
transform: { insert: 'dog', deleteLeft: 0 },

10
web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-deduplication.tests.ts
View file

@ -4,7 +4,7 @@ import * as wordBreakers from '@keymanapp/models-wordbreakers';
import { deepCopy } from 'keyman/common/web-utils';
import { LexicalModelTypes } from '@keymanapp/common-types';
import { IntermediateCompositedPrediction, dedupeSuggestions, models } from "@keymanapp/lm-worker/test-index";
import { CompositedIntermediatePrediction, dedupeSuggestions, models } from "@keymanapp/lm-worker/test-index";
import Context = LexicalModelTypes.Context;
import DummyModel = models.DummyModel;
@ -24,7 +24,7 @@ const testModel = new DummyModel({
* @returns
*/
const build_its_is_set = () => {
const its: IntermediateCompositedPrediction = {
const its: CompositedIntermediatePrediction = {
components: {
prediction: {
transform: {
@ -46,7 +46,7 @@ const build_its_is_set = () => {
}
};
const it_is: IntermediateCompositedPrediction = {
const it_is: CompositedIntermediatePrediction = {
components: {
prediction: {
transform: {
@ -67,7 +67,7 @@ const build_its_is_set = () => {
}
};
const is: IntermediateCompositedPrediction = {
const is: CompositedIntermediatePrediction = {
components: {
prediction: {
transform: {
@ -88,7 +88,7 @@ const build_its_is_set = () => {
}
};
const is_not: IntermediateCompositedPrediction = {
const is_not: CompositedIntermediatePrediction = {
components: {
prediction: {
transform: {

10
web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-finalization.tests.ts
View file

@ -5,7 +5,7 @@ import { deepCopy } from 'keyman/common/web-utils';
import * as wordBreakers from '@keymanapp/models-wordbreakers';
import { LexicalModelTypes } from '@keymanapp/common-types';
import { IntermediateCompositedPrediction, finalizeSuggestions, models } from "@keymanapp/lm-worker/test-index";
import { CompositedIntermediatePrediction, finalizeSuggestions, models } from "@keymanapp/lm-worker/test-index";
import DummyModel = models.DummyModel;
import Outcome = LexicalModelTypes.Outcome;
@ -48,7 +48,7 @@ const testModelWithoutSpacing = new DummyModel({
*/
const build_its_is_set = (verbose?: string) => {
const verboseFlag = (verbose == 'verbose' ? true : false);
const its: IntermediateCompositedPrediction = {
const its: CompositedIntermediatePrediction = {
components: {
prediction: {
transform: {
@ -70,7 +70,7 @@ const build_its_is_set = (verbose?: string) => {
}
};
const it_is: IntermediateCompositedPrediction = {
const it_is: CompositedIntermediatePrediction = {
components: {
prediction: {
transform: {
@ -91,7 +91,7 @@ const build_its_is_set = (verbose?: string) => {
}
};
const is: IntermediateCompositedPrediction = {
const is: CompositedIntermediatePrediction = {
components: {
prediction: {
transform: {
@ -112,7 +112,7 @@ const build_its_is_set = (verbose?: string) => {
}
};
const is_not: IntermediateCompositedPrediction = {
const is_not: CompositedIntermediatePrediction = {
components: {
prediction: {
transform: {

20
web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts
View file

@ -5,7 +5,7 @@ import * as wordBreakers from '@keymanapp/models-wordbreakers';
import { deepCopy } from 'keyman/common/web-utils';
import { LexicalModelTypes } from '@keymanapp/common-types';
import { IntermediateCompositedPrediction, models, processSimilarity, SuggestionSimilarity, toAnnotatedSuggestion } from "@keymanapp/lm-worker/test-index";
import { CompositedIntermediatePrediction, models, processSimilarity, SuggestionSimilarity, toAnnotatedSuggestion } from "@keymanapp/lm-worker/test-index";
import CasingFunction = LexicalModelTypes.CasingFunction;
import Context = LexicalModelTypes.Context;
@ -109,7 +109,7 @@ const testModelWithCasing = new DummyModel({
* @returns
*/
const build_its_is_set = () => {
const its: IntermediateCompositedPrediction = {
const its: CompositedIntermediatePrediction = {
components: {
prediction: {
transform: {
@ -131,7 +131,7 @@ const build_its_is_set = () => {
}
};
const it_is: IntermediateCompositedPrediction = {
const it_is: CompositedIntermediatePrediction = {
components: {
prediction: {
transform: {
@ -152,7 +152,7 @@ const build_its_is_set = () => {
}
};
const is: IntermediateCompositedPrediction = {
const is: CompositedIntermediatePrediction = {
components: {
prediction: {
transform: {
@ -173,7 +173,7 @@ const build_its_is_set = () => {
}
};
const is_not: IntermediateCompositedPrediction = {
const is_not: CompositedIntermediatePrediction = {
components: {
prediction: {
transform: {
@ -222,7 +222,7 @@ describe('processSimilarity', () => {
const testSet = build_its_is_set();
const distribution = [...Object.values(testSet)];
const expectation: IntermediateCompositedPrediction[] = [...Object.values(testSet)];
const expectation: CompositedIntermediatePrediction[] = [...Object.values(testSet)];
expectation[0].metadata.matchLevel = SuggestionSimilarity.exact; // its
expectation[1].metadata.matchLevel = SuggestionSimilarity.sameKey; // it_is
expectation[2].metadata.matchLevel = SuggestionSimilarity.none; // is
@ -259,7 +259,7 @@ describe('processSimilarity', () => {
const testSet = build_its_is_set();
const distribution = [...Object.values(testSet)];
const expectation: IntermediateCompositedPrediction[] = [...Object.values(testSet)];
const expectation: CompositedIntermediatePrediction[] = [...Object.values(testSet)];
expectation[0].metadata.matchLevel = SuggestionSimilarity.sameKey; // its
expectation[1].metadata.matchLevel = SuggestionSimilarity.exact; // it_is
expectation[2].metadata.matchLevel = SuggestionSimilarity.none; // is
@ -287,7 +287,7 @@ describe('processSimilarity', () => {
endOfBuffer: true
};
const distribution: IntermediateCompositedPrediction[] = [
const distribution: CompositedIntermediatePrediction[] = [
{
components: {
prediction: {
@ -353,7 +353,7 @@ describe('processSimilarity', () => {
const distribution = [...Object.values(testSet)];
const expectation: IntermediateCompositedPrediction[] = [...Object.values(testSet)];
const expectation: CompositedIntermediatePrediction[] = [...Object.values(testSet)];
expectation[0].metadata.matchLevel = SuggestionSimilarity.sameKey; // its
expectation[1].metadata.matchLevel = SuggestionSimilarity.sameText; // it_is
expectation[2].metadata.matchLevel = SuggestionSimilarity.none; // is
@ -395,7 +395,7 @@ describe('processSimilarity', () => {
const distribution = [...Object.values(testSet)];
const expectation: IntermediateCompositedPrediction[] = [...Object.values(testSet)];
const expectation: CompositedIntermediatePrediction[] = [...Object.values(testSet)];
expectation.forEach((entry) => entry.metadata.matchLevel = SuggestionSimilarity.none);
processSimilarity(testModelWithoutCasing, distribution, context, models.applyTransform(trueInput.sample, context));