diff --git a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts index 77f1361a81..a518642d2c 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts @@ -3,7 +3,7 @@ import { LexicalModelTypes } from '@keymanapp/common-types'; import * as correction from './correction/index.js' import TransformUtils from './transformUtils.js'; -import { applySuggestionCasing, correctAndEnumerate, dedupeSuggestions, finalizeSuggestions, predictionAutoSelect, processSimilarity, toAnnotatedSuggestion, tupleDisplayOrderSort } from './predict-helpers.js'; +import { applySuggestionCasing, correctAndEnumerate, createDefaultKeep, dedupeSuggestions, finalizeSuggestions, predictionAutoSelect, processSimilarity, toAnnotatedSuggestion, tupleDisplayOrderSort } from './predict-helpers.js'; import { detectCurrentCasing, determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js'; import { ContextTracker } from './correction/context-tracker.js'; @@ -171,10 +171,18 @@ export class ModelCompositor { const deduplicatedSuggestionTuples = dedupeSuggestions(this.lexicalModel, rawPredictions, context); // Needs "casing" to be applied first. - // - // Will also add a 'keep' suggestion (with `.matchesModel = false`) matching - // the current state of context if there is no such matching prediction. - processSimilarity(this.lexicalModel, deduplicatedSuggestionTuples, context, transformDistribution[0]); + const hasExistingKeep = processSimilarity(this.lexicalModel, deduplicatedSuggestionTuples, context, transformDistribution[0]); + + // If no existing suggestion directly matches the user-visible version of + // the token, also add a 'keep' suggestion (with `.matchesModel = false`) + // matching it. + if(!hasExistingKeep) { + const baseTuple = createDefaultKeep(this.lexicalModel, context, transformDistribution[0]); + + // Will be re-sorted shortly after this; just use the simple O(1) method here + // and let sorting put it in place. + deduplicatedSuggestionTuples.push(baseTuple); + } // Section 3: Sort the suggestions in display priority order to determine // which are most optimal, then auto-select based on the results. diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 0a9df50685..e3ca0762c2 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -752,30 +752,29 @@ export function dedupeSuggestions( /** * This function checks for any suggestions that directly match the actual - * context in some manner and ranks suggestions accordingly. Additionally, if - * there is no such suggestion, a stand-in is generated and added to the list, - * though marked as "not matching the model". + * context in some manner and ranks suggestions accordingly. * * The suggestion "ranks", from highest to lowest: * - the suggestion produces an exact match for the user's current text * - the suggestion produces a case-insensitive match for the user's current * text - * - the suggestion produces a case + diacritic insensitive match for the - * user's current text + * - the suggestion produces a case + diacritic insensitive match for the user's + * current text * - any other suggestion * * @param suggestionDistribution * @param context * @param trueInput inputTransform + its assigned probability - * @returns + * @returns true if an existing suggestion fulfills the role of 'keep'; + * otherwise, false. */ export function processSimilarity( lexicalModel: LexicalModel, suggestionDistribution: CorrectionPredictionTuple[], context: Context, trueInput: ProbabilityMass -) { - const { sample: inputTransform, p: inputTransformProb } = trueInput; +): boolean { + const { sample: inputTransform } = trueInput; const wordbreak = determineModelWordbreaker(lexicalModel); const postContext = models.applyTransform(inputTransform, context); @@ -827,9 +826,33 @@ export function processSimilarity( } // If we already have a keep option, we're done! Return and move on. - if(keepOption || truePrefix == '') { - return; - } + // + // No actual 'keep' needed if the current context token is empty, so we say we + // have a 'keep' for that case, even though there isn't really one. + return !!(keepOption || truePrefix == ''); +} + +/** + * Generates metadata for a new 'keep' suggestion based solely upon the existing + * context. + * + * This method is designed for use when no appropriate 'keep' suggestion was + * generated by the correction-search process. + * @param lexicalModel + * @param context + * @param trueInput + * @returns + */ +export function createDefaultKeep( + lexicalModel: LexicalModel, + context: Context, + trueInput: ProbabilityMass +): CorrectionPredictionTuple { + const { sample: inputTransform, p: inputTransformProb } = trueInput; + const wordbreak = determineModelWordbreaker(lexicalModel); + + const postContext = models.applyTransform(inputTransform, context); + const truePrefix = wordbreak(postContext); // Generate a full-word 'keep' replacement like other suggestions when one is not otherwise // produced; we want to replace the full token in the same manner used for other suggestions. @@ -843,14 +866,14 @@ export function processSimilarity( // This is the one case where the transform doesn't insert the full word; we need to override the displayAs param. keepSuggestion.displayAs = truePrefix; - keepOption = toAnnotatedSuggestion(lexicalModel, keepSuggestion, 'keep'); + let keepOption = toAnnotatedSuggestion(lexicalModel, keepSuggestion, 'keep'); if(inputTransform.id !== undefined) { keepOption.transformId = inputTransform.id; } keepOption.matchesModel = false; - // Insert our synthetic keepOption as a prediction. - suggestionDistribution.unshift({ + // Insert our synthetic keepOption as a prediction tuple. + return { // Product of the two p's below. totalProb: inputTransformProb * MAX_PROB, prediction: { @@ -864,7 +887,7 @@ export function processSimilarity( p: inputTransformProb * MAX_PROB }, matchLevel: SuggestionSimilarity.exact - }); + }; } /** diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts new file mode 100644 index 0000000000..6b5981c834 --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts @@ -0,0 +1,135 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2026-03-17 + * + * This file tests the prediction helper-method responsible for constructing + * 'keep' suggestions when no lexicon-based suggestion fits the role. + */ + +import { assert } from 'chai'; + +import { LexicalModelTypes } from "@keymanapp/common-types"; +import * as wordBreakers from '@keymanapp/models-wordbreakers'; + +import { CorrectionPredictionTuple, createDefaultKeep, models, SuggestionSimilarity } from "@keymanapp/lm-worker/test-index"; + +import CasingFunction = LexicalModelTypes.CasingFunction; +import Context = LexicalModelTypes.Context; +import DummyModel = models.DummyModel; +import DummyOptions = models.DummyOptions; +import ProbabilityMass = LexicalModelTypes.ProbabilityMass; +import Transform = LexicalModelTypes.Transform; + + +/* + * This file's tests use these parts of a lexical model: + * - model.wordbreaker + * - model.toKey + * - model.applyCasing + * - model.punctuation + */ + +const DUMMY_MODEL_CONFIG: DummyOptions = { + punctuation: { + quotesForKeepSuggestion: { + open: '<', + close: '>' + }, + insertAfterWord: '\u00a0' // non-breaking space + }, + wordbreaker: wordBreakers.default +}; + +// See: developer/src/kmc-model/model-defaults.ts, defaultApplyCasing +const applyCasing: CasingFunction = (casing, text) => { + switch(casing) { + case 'lower': + return text.toLowerCase(); + case 'upper': + return text.toUpperCase(); + case 'initial': + var headCode = text.charCodeAt(0); + // The length of the first code unit, as measured in code points. + var headUnitLength = 1; + + // Is the first character a high surrogate, indicating possible use of UTF-16 + // surrogate pairs? Also, is the string long enough for there to BE a pair? + if(text.length > 1 && headCode >= 0xD800 && headCode <= 0xDBFF) { + // It's possible, so now we check for low surrogates. + var lowSurrogateCode = text.charCodeAt(1); + + if(lowSurrogateCode >= 0xDC00 && lowSurrogateCode <= 0xDFFF) { + // We have a surrogate pair; this pair is the 'first' character. + headUnitLength++; + } + } + + // Capitalizes the first code unit of the string, leaving the rest intact. + return text.substring(0, headUnitLength).toUpperCase() // head - uppercased + .concat(text.substring(headUnitLength)); // tail - lowercased + } +}; +const testModelWithCasing = new DummyModel({ + ...DUMMY_MODEL_CONFIG, + applyCasing: applyCasing, + toKey: (wordform) => { + // See: developer/src/kmc-model/model-defaults.ts, defaultCasedSearchTermToKey + return applyCasing('lower', wordform) + .normalize('NFKD') + // Remove any combining diacritics (if input is in NFKD) + .replace(/[\u0300-\u036F]/g, '') + // Replace directional quotation marks with plain apostrophes + .replace(/[‘’]/g, "'") + // Also double-quote marks. + .replace(/[“”]/g, '"') + // ** Difference from model-defaults here ** + // And finally, erase single-quotation marks. + .replace(/'/, ''); + }, + languageUsesCasing: true + // No suggestions needed here, so we don't define any. +}); + +describe('produceKeep', () => { + it(`creates an 'exact'-match suggestion based on primary input and current context`, () => { + const context: Context = { + left: 'iphon', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 'e', + deleteLeft: 0 + }, + p: 1 + }; + + const expectedKeep: CorrectionPredictionTuple = { + correction: { + sample: 'iphone', + p: 1 + }, + prediction: { + sample: { + transform: { + insert: 'iphone', + deleteLeft: 5 + }, + displayAs: '', + matchesModel: false, + tag: 'keep' + }, + p: 1 + }, + totalProb: 1, + matchLevel: SuggestionSimilarity.exact + }; + + const tuple = createDefaultKeep(testModelWithCasing, context, trueInput); + assert.deepEqual(tuple, expectedKeep); + }); +}); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts index ae5133cf4b..72911d1128 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts @@ -285,81 +285,6 @@ describe('processSimilarity', () => { assert.deepEqual(it_is.prediction.sample, keep_it_is); }); - it(`creates an 'exact'-match suggestion as 'keep' if no exact-match exists`, () => { - const context: Context = { - left: 'iphon', - right: '', - startOfBuffer: true, - endOfBuffer: true - }; - - const trueInput: ProbabilityMass = { - sample: { - insert: 'e', - deleteLeft: 0 - }, - p: 1 - }; - - const iPhone: CorrectionPredictionTuple = { - correction: { - sample: 'iphone', - p: 0.8 - }, - prediction: { - sample: { - transform: { - insert: 'iPhone', - deleteLeft: 5 - }, - displayAs: 'iPhone' - }, - p: 0.8 - }, - totalProb: 0.64 - // matchLevel does not yet exist. - }; - - const distribution: CorrectionPredictionTuple[] = [ - iPhone - ]; - - const keep_iphone: CorrectionPredictionTuple = { - correction: { - sample: 'iphone', - p: 1 - }, - prediction: { - sample: { - transform: { - insert: 'iphone', - deleteLeft: 5 - }, - displayAs: '', - matchesModel: false, - tag: 'keep' - }, - p: 1 - }, - totalProb: 1, - matchLevel: SuggestionSimilarity.exact - }; - - - const expectation: CorrectionPredictionTuple[] = [ - { - ...keep_iphone, - matchLevel: SuggestionSimilarity.exact - }, { - ...iPhone, - matchLevel: SuggestionSimilarity.sameText - } - ]; - - processSimilarity(testModelWithCasing, distribution, context, trueInput); - assert.sameDeepMembers(distribution, expectation); - }); - describe('with casing', () => { // If we ever add a mode that can force lowercase for certain words even // when the context is title-cased or upper-cased, this scenario would be @@ -415,18 +340,9 @@ describe('processSimilarity', () => { processSimilarity(testModelWithCasing, distribution, context, trueInput); - // Because we mucked with the casing here, a new 'keep' was generated. - // Find it, confirm it exists and meets basic expectations, then remove it - // for easy comparison to pre-existing entries. - // - // We'll be less thorough checking this 'keep', as the "creates an 'exact'..." - // test above is thorough enough and tests the behavior already. - + // Because we mucked with the casing here, there is no perfect 'keep' match. const keep = distribution.find((entry) => entry.prediction.sample.tag == 'keep'); - assert.isOk(keep); - assert.equal(keep.prediction.sample.displayAs, ''); - - distribution.splice(distribution.indexOf(keep), 1); + assert.isNotOk(keep); assert.sameDeepMembers(distribution, expectation); }); }); @@ -478,18 +394,9 @@ describe('processSimilarity', () => { processSimilarity(testModelWithoutCasing, distribution, context, trueInput); - // Because we mucked with the casing here, a new 'keep' was generated. - // Find it, confirm it exists and meets basic expectations, then remove it - // for easy comparison to pre-existing entries. - // - // We'll be less thorough checking this 'keep', as the "creates an 'exact'..." - // test above is thorough enough and tests the behavior already. - + // Because we mucked with the casing here, there is no perfect 'keep' match. const keep = distribution.find((entry) => entry.prediction.sample.tag == 'keep'); - assert.isOk(keep); - assert.equal(keep.prediction.sample.displayAs, ''); - - distribution.splice(distribution.indexOf(keep), 1); + assert.isNotOk(keep); assert.sameDeepMembers(distribution, expectation); }); });