Merge pull request #15766 from keymanapp/refactor/web/create-default-keep

refactor(web): spin off default keep-generation from suggestionSimilarity 🚂

web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts

@@ -3,7 +3,7 @@ import { LexicalModelTypes } from '@keymanapp/common-types';
 
 import * as correction from './correction/index.js'
 import TransformUtils from './transformUtils.js';
-import { applySuggestionCasing, correctAndEnumerate, dedupeSuggestions, finalizeSuggestions, predictionAutoSelect, processSimilarity, toAnnotatedSuggestion, tupleDisplayOrderSort } from './predict-helpers.js';
+import { applySuggestionCasing, correctAndEnumerate, createDefaultKeep, dedupeSuggestions, finalizeSuggestions, predictionAutoSelect, processSimilarity, toAnnotatedSuggestion, tupleDisplayOrderSort } from './predict-helpers.js';
 import { detectCurrentCasing, determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js';
 
 import { ContextTracker } from './correction/context-tracker.js';
@@ -171,10 +171,18 @@ export class ModelCompositor {
     const deduplicatedSuggestionTuples = dedupeSuggestions(this.lexicalModel, rawPredictions, context);
 
     // Needs "casing" to be applied first.
-    //
-    // Will also add a 'keep' suggestion (with `.matchesModel = false`) matching
-    // the current state of context if there is no such matching prediction.
-    processSimilarity(this.lexicalModel, deduplicatedSuggestionTuples, context, transformDistribution[0]);
+    const hasExistingKeep = processSimilarity(this.lexicalModel, deduplicatedSuggestionTuples, context, transformDistribution[0]);
+
+    // If no existing suggestion directly matches the user-visible version of
+    // the token, also add a 'keep' suggestion (with `.matchesModel = false`)
+    // matching it.
+    if(!hasExistingKeep) {
+      const baseTuple = createDefaultKeep(this.lexicalModel, context, transformDistribution[0]);
+
+      // Will be re-sorted shortly after this; just use the simple O(1) method here
+      // and let sorting put it in place.
+      deduplicatedSuggestionTuples.push(baseTuple);
+    }
 
     // Section 3:  Sort the suggestions in display priority order to determine
     // which are most optimal, then auto-select based on the results.

web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts

@@ -752,30 +752,29 @@ export function dedupeSuggestions(
 
 /**
  * This function checks for any suggestions that directly match the actual
- * context in some manner and ranks suggestions accordingly.  Additionally, if
- * there is no such suggestion, a stand-in is generated and added to the list,
- * though marked as "not matching the model".
+ * context in some manner and ranks suggestions accordingly.
  *
  * The suggestion "ranks", from highest to lowest:
  * - the suggestion produces an exact match for the user's current text
  * - the suggestion produces a case-insensitive match for the user's current
  *   text
- * - the suggestion produces a case + diacritic insensitive match for the
- *   user's current text
+ * - the suggestion produces a case + diacritic insensitive match for the user's
+ *   current text
  * - any other suggestion
  *
  * @param suggestionDistribution
  * @param context
  * @param trueInput inputTransform + its assigned probability
- * @returns
+ * @returns true if an existing suggestion fulfills the role of 'keep';
+ * otherwise, false.
  */
 export function processSimilarity(
   lexicalModel: LexicalModel,
   suggestionDistribution: CorrectionPredictionTuple[],
   context: Context,
   trueInput: ProbabilityMass<Transform>
-) {
-  const { sample: inputTransform, p: inputTransformProb } = trueInput;
+): boolean {
+  const { sample: inputTransform } = trueInput;
   const wordbreak = determineModelWordbreaker(lexicalModel);
 
   const postContext = models.applyTransform(inputTransform, context);
@@ -827,9 +826,33 @@ export function processSimilarity(
   }
 
   // If we already have a keep option, we're done!  Return and move on.
-  if(keepOption || truePrefix == '') {
-    return;
-  }
+  //
+  // No actual 'keep' needed if the current context token is empty, so we say we
+  // have a 'keep' for that case, even though there isn't really one.
+  return !!(keepOption || truePrefix == '');
+}
+
+/**
+ * Generates metadata for a new 'keep' suggestion based solely upon the existing
+ * context.
+ *
+ * This method is designed for use when no appropriate 'keep' suggestion was
+ * generated by the correction-search process.
+ * @param lexicalModel
+ * @param context
+ * @param trueInput
+ * @returns
+ */
+export function createDefaultKeep(
+  lexicalModel: LexicalModel,
+  context: Context,
+  trueInput: ProbabilityMass<Transform>
+): CorrectionPredictionTuple {
+  const { sample: inputTransform, p: inputTransformProb } = trueInput;
+  const wordbreak = determineModelWordbreaker(lexicalModel);
+
+  const postContext = models.applyTransform(inputTransform, context);
+  const truePrefix = wordbreak(postContext);
 
   // Generate a full-word 'keep' replacement like other suggestions when one is not otherwise
   // produced; we want to replace the full token in the same manner used for other suggestions.
@@ -843,14 +866,14 @@ export function processSimilarity(
   // This is the one case where the transform doesn't insert the full word; we need to override the displayAs param.
   keepSuggestion.displayAs = truePrefix;
 
-  keepOption = toAnnotatedSuggestion(lexicalModel, keepSuggestion, 'keep');
+  let keepOption = toAnnotatedSuggestion(lexicalModel, keepSuggestion, 'keep');
   if(inputTransform.id !== undefined) {
     keepOption.transformId = inputTransform.id;
   }
   keepOption.matchesModel = false;
 
-  // Insert our synthetic keepOption as a prediction.
-  suggestionDistribution.unshift({
+  // Insert our synthetic keepOption as a prediction tuple.
+  return {
     // Product of the two p's below.
     totalProb: inputTransformProb * MAX_PROB,
     prediction: {
@@ -864,7 +887,7 @@ export function processSimilarity(
       p: inputTransformProb * MAX_PROB
     },
     matchLevel: SuggestionSimilarity.exact
-  });
+  };
 }
 
 /**

web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts

@@ -0,0 +1,135 @@
+/*
+ * Keyman is copyright (C) SIL Global. MIT License.
+ *
+ * Created by jahorton on 2026-03-17
+ *
+ * This file tests the prediction helper-method responsible for constructing
+ * 'keep' suggestions when no lexicon-based suggestion fits the role.
+ */
+
+import { assert } from 'chai';
+
+import { LexicalModelTypes } from "@keymanapp/common-types";
+import * as wordBreakers from '@keymanapp/models-wordbreakers';
+
+import { CorrectionPredictionTuple, createDefaultKeep, models, SuggestionSimilarity } from "@keymanapp/lm-worker/test-index";
+
+import CasingFunction = LexicalModelTypes.CasingFunction;
+import Context = LexicalModelTypes.Context;
+import DummyModel = models.DummyModel;
+import DummyOptions = models.DummyOptions;
+import ProbabilityMass = LexicalModelTypes.ProbabilityMass;
+import Transform = LexicalModelTypes.Transform;
+
+
+/*
+ * This file's tests use these parts of a lexical model:
+ * - model.wordbreaker
+ * - model.toKey
+ * - model.applyCasing
+ * - model.punctuation
+ */
+
+const DUMMY_MODEL_CONFIG: DummyOptions = {
+  punctuation: {
+    quotesForKeepSuggestion: {
+      open: '<',
+      close: '>'
+    },
+    insertAfterWord: '\u00a0' // non-breaking space
+  },
+  wordbreaker: wordBreakers.default
+};
+
+// See: developer/src/kmc-model/model-defaults.ts, defaultApplyCasing
+const applyCasing: CasingFunction = (casing, text) => {
+  switch(casing) {
+    case 'lower':
+      return text.toLowerCase();
+    case 'upper':
+      return text.toUpperCase();
+    case 'initial':
+      var headCode = text.charCodeAt(0);
+      // The length of the first code unit, as measured in code points.
+      var headUnitLength = 1;
+
+      // Is the first character a high surrogate, indicating possible use of UTF-16
+      // surrogate pairs?  Also, is the string long enough for there to BE a pair?
+      if(text.length > 1 && headCode >= 0xD800 && headCode <= 0xDBFF) {
+        // It's possible, so now we check for low surrogates.
+        var lowSurrogateCode = text.charCodeAt(1);
+
+        if(lowSurrogateCode >= 0xDC00 && lowSurrogateCode <= 0xDFFF) {
+          // We have a surrogate pair; this pair is the 'first' character.
+          headUnitLength++;
+        }
+      }
+
+      // Capitalizes the first code unit of the string, leaving the rest intact.
+      return text.substring(0, headUnitLength).toUpperCase() // head - uppercased
+             .concat(text.substring(headUnitLength));        // tail - lowercased
+  }
+};
+const testModelWithCasing = new DummyModel({
+  ...DUMMY_MODEL_CONFIG,
+  applyCasing: applyCasing,
+  toKey: (wordform) => {
+    // See: developer/src/kmc-model/model-defaults.ts, defaultCasedSearchTermToKey
+    return applyCasing('lower', wordform)
+      .normalize('NFKD')
+      // Remove any combining diacritics (if input is in NFKD)
+      .replace(/[\u0300-\u036F]/g, '')
+      // Replace directional quotation marks with plain apostrophes
+      .replace(/[‘’]/g, "'")
+      // Also double-quote marks.
+      .replace(/[“”]/g, '"')
+      // ** Difference from model-defaults here **
+      // And finally, erase single-quotation marks.
+      .replace(/'/, '');
+  },
+  languageUsesCasing: true
+  // No suggestions needed here, so we don't define any.
+});
+
+describe('produceKeep', () => {
+  it(`creates an 'exact'-match suggestion based on primary input and current context`, () => {
+    const context: Context = {
+      left: 'iphon',
+      right: '',
+      startOfBuffer: true,
+      endOfBuffer: true
+    };
+
+    const trueInput: ProbabilityMass<Transform> = {
+      sample: {
+        insert: 'e',
+        deleteLeft: 0
+      },
+      p: 1
+    };
+
+    const expectedKeep: CorrectionPredictionTuple = {
+      correction: {
+        sample: 'iphone',
+        p: 1
+      },
+      prediction: {
+        sample: {
+          transform: {
+            insert: 'iphone',
+            deleteLeft: 5
+          },
+          displayAs: '<iphone>',
+          matchesModel: false,
+          tag: 'keep'
+        },
+        p: 1
+      },
+      totalProb: 1,
+      matchLevel: SuggestionSimilarity.exact
+    };
+
+    const tuple = createDefaultKeep(testModelWithCasing, context, trueInput);
+    assert.deepEqual(tuple, expectedKeep);
+  });
+});

web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/suggestion-similarity.tests.ts

@@ -285,81 +285,6 @@ describe('processSimilarity', () => {
     assert.deepEqual(it_is.prediction.sample, keep_it_is);
   });
 
-  it(`creates an 'exact'-match suggestion as 'keep' if no exact-match exists`, () => {
-    const context: Context = {
-      left: 'iphon',
-      right: '',
-      startOfBuffer: true,
-      endOfBuffer: true
-    };
-
-    const trueInput: ProbabilityMass<Transform> = {
-      sample: {
-        insert: 'e',
-        deleteLeft: 0
-      },
-      p: 1
-    };
-
-    const iPhone: CorrectionPredictionTuple = {
-      correction: {
-        sample: 'iphone',
-        p: 0.8
-      },
-      prediction: {
-        sample: {
-          transform: {
-            insert: 'iPhone',
-            deleteLeft: 5
-          },
-          displayAs: 'iPhone'
-        },
-        p: 0.8
-      },
-      totalProb: 0.64
-      // matchLevel does not yet exist.
-    };
-
-    const distribution: CorrectionPredictionTuple[] = [
-      iPhone
-    ];
-
-    const keep_iphone: CorrectionPredictionTuple = {
-      correction: {
-        sample: 'iphone',
-        p: 1
-      },
-      prediction: {
-        sample: {
-          transform: {
-            insert: 'iphone',
-            deleteLeft: 5
-          },
-          displayAs: '<iphone>',
-          matchesModel: false,
-          tag: 'keep'
-        },
-        p: 1
-      },
-      totalProb: 1,
-      matchLevel: SuggestionSimilarity.exact
-    };
-
-
-    const expectation: CorrectionPredictionTuple[] = [
-      {
-        ...keep_iphone,
-        matchLevel: SuggestionSimilarity.exact
-      }, {
-        ...iPhone,
-        matchLevel: SuggestionSimilarity.sameText
-      }
-    ];
-
-    processSimilarity(testModelWithCasing, distribution, context, trueInput);
-    assert.sameDeepMembers(distribution, expectation);
-  });
-
   describe('with casing', () => {
     // If we ever add a mode that can force lowercase for certain words even
     // when the context is title-cased or upper-cased, this scenario would be
@@ -415,18 +340,9 @@ describe('processSimilarity', () => {
 
       processSimilarity(testModelWithCasing, distribution, context, trueInput);
 
-      // Because we mucked with the casing here, a new 'keep' was generated.
-      // Find it, confirm it exists and meets basic expectations, then remove it
-      // for easy comparison to pre-existing entries.
-      //
-      // We'll be less thorough checking this 'keep', as the "creates an 'exact'..."
-      // test above is thorough enough and tests the behavior already.
-
+      // Because we mucked with the casing here, there is no perfect 'keep' match.
       const keep = distribution.find((entry) => entry.prediction.sample.tag == 'keep');
-      assert.isOk(keep);
-      assert.equal(keep.prediction.sample.displayAs, '<It\'s>');
-
-      distribution.splice(distribution.indexOf(keep), 1);
+      assert.isNotOk(keep);
       assert.sameDeepMembers(distribution, expectation);
     });
   });
@@ -478,18 +394,9 @@ describe('processSimilarity', () => {
 
       processSimilarity(testModelWithoutCasing, distribution, context, trueInput);
 
-      // Because we mucked with the casing here, a new 'keep' was generated.
-      // Find it, confirm it exists and meets basic expectations, then remove it
-      // for easy comparison to pre-existing entries.
-      //
-      // We'll be less thorough checking this 'keep', as the "creates an 'exact'..."
-      // test above is thorough enough and tests the behavior already.
-
+      // Because we mucked with the casing here, there is no perfect 'keep' match.
       const keep = distribution.find((entry) => entry.prediction.sample.tag == 'keep');
-      assert.isOk(keep);
-      assert.equal(keep.prediction.sample.displayAs, '<It\'s>');
-
-      distribution.splice(distribution.indexOf(keep), 1);
+      assert.isNotOk(keep);
       assert.sameDeepMembers(distribution, expectation);
     });
   });