diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index ff7a3faaa1..7ff7590a88 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -971,7 +971,7 @@ export function processSimilarity( /** * Generates metadata for a new 'keep' suggestion based solely upon the existing - * context. + * context and the most likely input. * * This method is designed for use when no appropriate 'keep' suggestion was * generated by the correction-search process. @@ -987,19 +987,28 @@ export function createDefaultKeep( ): CorrectionPredictionTuple { const { sample: inputTransform, p: inputTransformProb } = trueInput; const wordbreak = determineModelWordbreaker(lexicalModel); + const tokenizer = determineModelTokenizer(lexicalModel); const postContext = models.applyTransform(inputTransform, context); const truePrefix = wordbreak(postContext); - // Generate a full-word 'keep' replacement like other suggestions when one is not otherwise - // produced; we want to replace the full token in the same manner used for other suggestions. - const basePrefixLength = KMWString.length(truePrefix) - KMWString.length(inputTransform.insert) + inputTransform.deleteLeft; - const keepTransform = { - insert: truePrefix, - deleteLeft: basePrefixLength - }; + const tokenization = tokenizer(context); + const postTokenization = tokenizer(postContext); + + const tokenMapper = (t: models.Token) => { + return { + exampleInput: t.text, + codepointLength: KMWString.length(t.text) + } as ContextTokenLike; + } + const transitionEffects = determineSuggestionRange(tokenization.left.map(tokenMapper), postTokenization.left.map(tokenMapper), (a, b) => a.exampleInput == b.exampleInput); + + const keepTransition = { + insert: transitionEffects.tokensToPredict.reduce((accum, curr) => accum + curr.exampleInput, ''), + deleteLeft: transitionEffects.tokensToRemove.reduce((accum, curr) => accum + curr.codepointLength, 0) + } + let keepSuggestion = models.transformToSuggestion(keepTransition); - let keepSuggestion = models.transformToSuggestion(keepTransform); // This is the one case where the transform doesn't insert the full word; we need to override the displayAs param. keepSuggestion.displayAs = truePrefix; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts index 6b5981c834..052c349291 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/create-default-keep.tests.ts @@ -91,8 +91,8 @@ const testModelWithCasing = new DummyModel({ // No suggestions needed here, so we don't define any. }); -describe('produceKeep', () => { - it(`creates an 'exact'-match suggestion based on primary input and current context`, () => { +describe('createDefaultKeep', () => { + it(`creates an 'exact'-match suggestion based on simple primary input`, () => { const context: Context = { left: 'iphon', right: '', @@ -132,4 +132,209 @@ describe('produceKeep', () => { const tuple = createDefaultKeep(testModelWithCasing, context, trueInput); assert.deepEqual(tuple, expectedKeep); }); + + it(`creates an 'exact'-match suggestion based on full word after a backspace`, () => { + const context: Context = { + left: 'iphone ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: '', + deleteLeft: 1 + }, + p: 1 + }; + + const expectedKeep: CorrectionPredictionTuple = { + correction: { + sample: 'iphone', + p: 1 + }, + prediction: { + sample: { + transform: { + insert: 'iphone', + deleteLeft: 7 + }, + displayAs: '', + matchesModel: false, + tag: 'keep' + }, + p: 1 + }, + totalProb: 1, + matchLevel: SuggestionSimilarity.exact + }; + + const tuple = createDefaultKeep(testModelWithCasing, context, trueInput); + assert.deepEqual(tuple, expectedKeep); + }); + + it(`creates an 'exact'-match suggestion based on complex deletion`, () => { + const context: Context = { + left: 'iphone a', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 'e', + deleteLeft: 3 + }, + p: 1 + }; + + const expectedKeep: CorrectionPredictionTuple = { + correction: { + sample: 'iphone', + p: 1 + }, + prediction: { + sample: { + transform: { + insert: 'iphone', + deleteLeft: 8 + }, + displayAs: '', + matchesModel: false, + tag: 'keep' + }, + p: 1 + }, + totalProb: 1, + matchLevel: SuggestionSimilarity.exact + }; + + const tuple = createDefaultKeep(testModelWithCasing, context, trueInput); + assert.deepEqual(tuple, expectedKeep); + }); + + it(`creates an 'exact'-match suggestion based on complex insertion`, () => { + const context: Context = { + left: 'iphon', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 'es and', + deleteLeft: 0 + }, + p: 1 + }; + + const expectedKeep: CorrectionPredictionTuple = { + correction: { + sample: 'and', + p: 1 + }, + prediction: { + sample: { + transform: { + insert: 'iphones and', + deleteLeft: 5 + }, + displayAs: '', + matchesModel: false, + tag: 'keep' + }, + p: 1 + }, + totalProb: 1, + matchLevel: SuggestionSimilarity.exact + }; + + const tuple = createDefaultKeep(testModelWithCasing, context, trueInput); + assert.deepEqual(tuple, expectedKeep); + }); + + it(`creates an 'exact'-match suggestion based on complex replacement`, () => { + const context: Context = { + left: 'iphone ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 's', + deleteLeft: 1 + }, + p: 1 + }; + + const expectedKeep: CorrectionPredictionTuple = { + correction: { + sample: 'iphones', + p: 1 + }, + prediction: { + sample: { + transform: { + insert: 'iphones', + deleteLeft: 7 + }, + displayAs: '', + matchesModel: false, + tag: 'keep' + }, + p: 1 + }, + totalProb: 1, + matchLevel: SuggestionSimilarity.exact + }; + + const tuple = createDefaultKeep(testModelWithCasing, context, trueInput); + assert.deepEqual(tuple, expectedKeep); + }); + + it(`creates an empty 'exact'-match suggestion after adding a wordbreak`, () => { + const context: Context = { + left: 'iphon', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const trueInput: ProbabilityMass = { + sample: { + insert: 'e ', + deleteLeft: 0 + }, + p: 1 + }; + + const expectedKeep: CorrectionPredictionTuple = { + correction: { + sample: '', + p: 1 + }, + prediction: { + sample: { + transform: { + insert: 'iphone ', + deleteLeft: 5 + }, + displayAs: '<>', + matchesModel: false, + tag: 'keep' + }, + p: 1 + }, + totalProb: 1, + matchLevel: SuggestionSimilarity.exact + }; + + const tuple = createDefaultKeep(testModelWithCasing, context, trueInput); + assert.deepEqual(tuple, expectedKeep); + }); }); \ No newline at end of file