diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/alignment-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/alignment-helpers.ts index 62d62f3636..06ae26ed91 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/alignment-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/alignment-helpers.ts @@ -8,6 +8,7 @@ * edits for aligned context tokens. */ +import { SENTINEL_CODE_UNIT } from '@keymanapp/models-templates'; import { ClassicalDistanceCalculation, EditOperation } from "./classical-calculation.js"; /** @@ -18,7 +19,14 @@ export type ContextStateAlignment = { /** * Denotes whether or not alignment is possible between two contexts. */ - canAlign: false + canAlign: false, + + /** + * Indicates the edit path that could not be handled. (Useful for error reporting) + * + * The edit path does not include actual user text and is sanitized. + */ + editPath: EditOperation[]; } | { /** * Denotes whether or not alignment is possible between two contexts. @@ -201,10 +209,27 @@ export function computeAlignment( isSliding: boolean, noSubVerify?: boolean ): ContextStateAlignment { + const src = tokenizationToMatch.map(value => ({key: value})); + const dst = incomingTokenization.map(value => ({key: value})); + + // let changedEmptyTail = false; + if(dst[dst.length - 1].key == '') { + // Only allow matching if the tokenizations are identical, thus the empty + // token was unaffected. + if(src.length != dst.length || src[dst.length - 1].key != '') { + // Do not allow empty-token matches to match each other; this complicates + // things when applying zero-root suggestions. + // + // The SENTINEL char should never appear in raw text, thus should never + // match anything in the "tokenization to match". + dst[dst.length - 1].key = SENTINEL_CODE_UNIT; + } + } + // Inverted order, since 'match' existed before our new context. let mapping = ClassicalDistanceCalculation.computeDistance( - tokenizationToMatch.map(value => ({key: value})), - incomingTokenization.map(value => ({key: value})), + src, + dst, // Diagonal width to consider must be at least 2, as adding a single // whitespace after a token tends to add two tokens: one for whitespace, // one for the empty token to follow it. @@ -212,6 +237,12 @@ export function computeAlignment( ); let editPath = mapping.editPath(); + + const failure: ContextStateAlignment = { + canAlign: false, + editPath + }; + // Special case: new context bootstrapping - first token often substitutes. // The text length is small enough that no words should be able to rotate out the start of the context. // Special handling needed in case of no 'match'; the rest of the method assumes at least one 'match'. @@ -222,9 +253,7 @@ export function computeAlignment( if(editPath[i] == 'substitute') { subCount++; if(!noSubVerify && !isSubstitutionAlignable(incomingTokenization[i], tokenizationToMatch[i], true)) { - return { - canAlign: false - }; + return failure; } } else if(editPath[i] == 'match') { // If a substitution is already recorded, treat the 'match' as a substitution. @@ -254,16 +283,12 @@ export function computeAlignment( if(firstMatch == -1) { // If there are no matches, there's no alignment. - return { - canAlign: false - }; + return failure; } // Transpositions are not allowed at the token level during context alignment. if(editPath.find((entry) => entry.indexOf('transpose') > -1)) { - return { - canAlign: false - }; + return failure; } const lastMatch = getEditPathLastMatch(editPath); @@ -272,9 +297,7 @@ export function computeAlignment( // continuous block of 'match' entries. If there's anything else in // the middle, we have a context mismatch. if(lastMatch == -1) { - return { - canAlign: false - }; + return failure; } let matchLength = lastMatch - firstMatch + 1; @@ -290,9 +313,7 @@ export function computeAlignment( if(tailInsertLength > 0 && tailDeleteLength > 0) { // Something's gone weird if this happens; that should appear as a substitution instead. // Otherwise, we have a VERY niche edit scenario. - return { - canAlign: false - }; + return failure; } const tailSubstituteLength = (editPath.length - 1 - lastMatch) - tailInsertLength - tailDeleteLength; @@ -330,9 +351,7 @@ export function computeAlignment( // All deletions should appear at the sliding window edge; if a deletion appears // after the edge, but before the first match, something's wrong. if(priorEdit && priorEdit != 'delete') { - return { - canAlign: false - }; + return failure; } leadTokensRemoved++; break; @@ -342,9 +361,7 @@ export function computeAlignment( // Any extras in the front would be pure inserts, not substitutions, due to // the sliding context window and its implications. if(leadSubstitutions++ > 0) { - return { - canAlign: false - }; + return failure; } // Find the word before and after substitution. @@ -360,9 +377,7 @@ export function computeAlignment( // Exception: if the word is at the start of the context window and the // context window is likely sliding, don't check it. if(!noSubVerify && !atSlidePoint && !isSubstitutionAlignable(incomingSub, matchingSub)) { - return { - canAlign: false - }; + return failure; } // There's no major need to drop parts of a token being 'slid' out of the context window. @@ -372,9 +387,7 @@ export function computeAlignment( case 'insert': // Only allow an insert at the leading edge, as with 'delete's. if(priorEdit && priorEdit != 'insert') { - return { - canAlign: false - }; + return failure; } // In case of backspaces, it's also possible to 'insert' a 'new' // token - an old one that's slid back into view. @@ -383,9 +396,7 @@ export function computeAlignment( default: // No 'match' can exist before the first found index for a 'match'. // No 'transpose-' edits should exist within this section, either. - return { - canAlign: false - }; + return failure; } priorEdit = editPath[i]; } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts index 457c1349b2..f000bc8783 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts @@ -197,7 +197,10 @@ export class ContextState { // computeDistance 'radius' in the called function. const alignmentResults = this.tokenization.computeAlignment(tokenizedContext.map((token) => token.text), isSliding, isApplyingSuggestion); - if(!alignmentResults.canAlign) { + if(alignmentResults.canAlign == false) { // Needs to be explicit for TS type inference. + if(console && console.error) { + console.error(`Could not align contexts with edit path ${JSON.stringify(alignmentResults.editPath)}`); + } return null; } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 20d0733db5..26f43a3acb 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -107,7 +107,9 @@ export class ContextTokenization { // If we have a perfect match with a pre-existing tokenization, no mutations have // happened; just re-use the old context tokenization. if(tailEditLength == 0 && leadTokenShift == 0 && tailTokenShift == 0) { - return this; + // We must build a new instance in case the original did not have + // alignment data (like when it's the initial context!) + return new ContextTokenization(this.tokens, alignment); } else { // If we didn't get any input, we really should perfectly match // a previous context state. If such a state is out of our cache, diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/alignment-helpers.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/alignment-helpers.tests.ts index 16dd8327bc..44bba81c3c 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/alignment-helpers.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/alignment-helpers.tests.ts @@ -182,7 +182,7 @@ describe('computeAlignment', () => { ]; const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, {canAlign: false}); + assert.deepEqual(computedAlignment, {canAlign: false, editPath: ['substitute', 'substitute', 'substitute', 'substitute', 'substitute']}); }); it("detects unalignable contexts - too many mismatching tokens", () => { @@ -194,7 +194,7 @@ describe('computeAlignment', () => { ]; const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, {canAlign: false}); + assert.deepEqual(computedAlignment, {canAlign: false, editPath: ['substitute', 'substitute', 'match', 'match', 'match']}); }); it("fails alignment for leading-edge word substitutions", () => { @@ -206,7 +206,7 @@ describe('computeAlignment', () => { ]; const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, {canAlign: false}); + assert.deepEqual(computedAlignment, {canAlign: false, editPath: ['substitute', 'match', 'match', 'match', 'match']}); }); it("fails alignment for small leading-edge word substitutions", () => { @@ -218,7 +218,7 @@ describe('computeAlignment', () => { ]; const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, {canAlign: false}); + assert.deepEqual(computedAlignment, {canAlign: false, editPath: ['substitute', 'match', 'match', 'match', 'match']}); }); it("properly matches and aligns when lead token is modified", () => { @@ -428,7 +428,7 @@ describe('computeAlignment', () => { ]; const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, {canAlign: false}); + assert.deepEqual(computedAlignment, {canAlign: false, editPath: ["match", "delete", "match", "match", "match"]}); }); it("fails alignment for mid-head insertion", () => { @@ -440,7 +440,7 @@ describe('computeAlignment', () => { ]; const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, {canAlign: false}); + assert.deepEqual(computedAlignment, {canAlign: false, editPath: ["match", "insert", "match", "match", "match"]}); }); it("fails alignment for mid-tail deletion", () => { @@ -452,7 +452,7 @@ describe('computeAlignment', () => { ]; const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, {canAlign: false}); + assert.deepEqual(computedAlignment, {canAlign: false, editPath: ["match", "match", "match", "delete", "match"]}); }); it("fails alignment for mid-tail insertion", () => { @@ -464,7 +464,7 @@ describe('computeAlignment', () => { ]; const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, {canAlign: false}); + assert.deepEqual(computedAlignment, {canAlign: false, editPath: ["match", "match", "match", "match", "insert", "match"]}); }); it("handles late-context suggestion application after backspace", () => { @@ -485,6 +485,25 @@ describe('computeAlignment', () => { }); }); + it("handles late-context application of default suggestion", () => { + const baseContext = [ + 'quick', ' ', 'brown', ' ', 'fox', ' ', 'jumped', ' ', 'over', ' ', '' + ]; + const newContext = [ + 'quick', ' ', 'brown', ' ', 'fox', ' ', 'jumped', ' ', 'over', ' ', 'the', ' ', '' + ]; + + const computedAlignment = computeAlignment(baseContext, newContext, false); + + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: 0, + matchLength: 10, + tailEditLength: 1, + tailTokenShift: 2 + }); + }); + it("handles sliding context-window scenarios", () => { // // Explicitly-defined window, though it's not needed directly by the method. // const config = { diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts index c140fa83aa..87548e2096 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts @@ -80,7 +80,61 @@ describe('ContextState', () => { describe('analyzeTransition', function() { - it("properly matches and aligns when lead token is removed", function() { + it("properly matches and aligns when no context changes occur (end of word)", function() { + let existingContext = { + left: "an apple a day keeps the doctor", startOfBuffer: true, endOfBuffer: true + }; + let transform = { + insert: '', + deleteLeft: 0 + }; + let newContext = { + left: "an apple a day keeps the doctor", startOfBuffer: true, endOfBuffer: true + }; + let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"]; + + let baseState = new ContextState(existingContext, plainModel); + let newContextMatch = baseState.analyzeTransition(newContext, toWrapperDistribution(transform)); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + + // Phrased this way to facilitate TS type-inference; assert.isTrue() does + // NOT do this for us! + if(!newContextMatch.final.tokenization.alignment.canAlign) { + assert.fail("context alignment failed"); + } + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0); + }); + + it("properly matches and aligns when no context changes occur (after whitespace)", function() { + let existingContext = { + left: "an apple a day keeps the doctor ", startOfBuffer: true, endOfBuffer: true + }; + let transform = { + insert: '', + deleteLeft: 0 + }; + let newContext = { + left: "an apple a day keeps the doctor ", startOfBuffer: true, endOfBuffer: true + }; + let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""]; + + let baseState = new ContextState(existingContext, plainModel); + let newContextMatch = baseState.analyzeTransition(newContext, toWrapperDistribution(transform)); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + + // Phrased this way to facilitate TS type-inference; assert.isTrue() does + // NOT do this for us! + if(!newContextMatch.final.tokenization.alignment.canAlign) { + assert.fail("context alignment failed"); + } + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0); + }); + + it("properly matches and aligns when lead token is removed (end of word)", function() { let existingContext = { left: "an apple a day keeps the doctor", startOfBuffer: true, endOfBuffer: true }; @@ -105,6 +159,31 @@ describe('ContextState', () => { assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0); }); + it("properly matches and aligns when lead token is removed (after whitespace)", function() { + let existingContext = { + left: "an apple a day keeps the doctor ", startOfBuffer: true, endOfBuffer: true + }; + let transform = { + insert: '', + deleteLeft: 0 + }; + let newContext = { + left: " apple a day keeps the doctor ", startOfBuffer: true, endOfBuffer: true + }; + let rawTokens = [" ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""]; + + let baseState = new ContextState(existingContext, plainModel); + let newContextMatch = baseState.analyzeTransition(newContext, toWrapperDistribution(transform)); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + + if(!newContextMatch.final.tokenization.alignment.canAlign) { + assert.fail("context alignment failed"); + } + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, -1); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0); + }); + it("properly matches and aligns when lead token + following whitespace are removed", function() { let existingContext = { left: "an apple a day keeps the doctor", startOfBuffer: true, endOfBuffer: true diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index 17e335a8f9..1bdd33c8c2 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -14,7 +14,7 @@ import { default as defaultBreaker } from '@keymanapp/models-wordbreakers'; import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; import { LexicalModelTypes } from '@keymanapp/common-types'; -import { ContextToken, ContextTokenization, models } from '@keymanapp/lm-worker/test-index'; +import { ContextStateAlignment, ContextToken, ContextTokenization, models } from '@keymanapp/lm-worker/test-index'; import Transform = LexicalModelTypes.Transform; import TrieModel = models.TrieModel; @@ -43,7 +43,7 @@ describe('ContextTokenization', function() { it("constructs from a token array + alignment data", () => { const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; - let alignment = { + let alignment: ContextStateAlignment = { canAlign: true, leadTokenShift: 0, matchLength: 6, diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts index 2e7fc05a3f..674ed6abab 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts @@ -14,9 +14,6 @@ import { ModelCompositor, models } from '@keymanapp/lm-worker/test-index'; import TrieModel = models.TrieModel; - -var emptyInput = (id: number) => [{sample: {insert: '', deleteLeft: 0, id: id}, p: 1}]; - describe('ContextTracker', function() { describe('suggestion acceptance tracking', function() { let englishPunctuation = { @@ -62,7 +59,7 @@ describe('ContextTracker', function() { compositor.initContextTracker(baseContext, 0); const contextTracker = compositor.contextTracker; - let preAppliedTransition = contextTracker.analyzeState(model, baseContext, emptyInput(0)); + let preAppliedTransition = contextTracker.latest; // We'll ignore and overwrite the results. We do need the prediction round to occur, though. await compositor.predict([{sample: postTransform, p: 1}], baseContext); contextTracker.latest.final.suggestions = [baseSuggestion]; @@ -74,8 +71,7 @@ describe('ContextTracker', function() { contextTracker.unitTestEndPoints.cache().keys().forEach((key) => assert.isDefined(key)); // Next step - on the followup context, is the replacement still active? - let postContext = models.applyTransform(baseSuggestion.appendedTransform, models.applyTransform(baseSuggestion.transform, baseContext)); - let postContextMatch = compositor.contextTracker.analyzeState(model, postContext, emptyInput(2)); + let postContextMatch = contextTracker.unitTestEndPoints.cache().get(2); assert.equal(postContextMatch.final.appliedSuggestionId, baseSuggestion.id); // Penultimate token corresponds to whitespace, which does not have a 'raw' representation. diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts index 82bfcee60e..4affcfd8c0 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts @@ -19,8 +19,6 @@ import Suggestion = LexicalModelTypes.Suggestion; import Transform = LexicalModelTypes.Transform; import TrieModel = models.TrieModel; -var emptyInput = (id: number) => [{sample: {insert: '', deleteLeft: 0, id: id}, p: 1}]; - describe('ModelCompositor', function() { describe('Prediction with 14.0+ models', function() { describe('Basic suggestion generation', function() { @@ -115,6 +113,7 @@ describe('ModelCompositor', function() { let context = { left: '', startOfBuffer: true, endOfBuffer: true, }; + compositor.initContextTracker(context, 0); // Initialize context tracking first! // The 'weights' involved imply that we have an edge-case fat finger on the bottom of // the 'q' key, slightly in its favor. @@ -123,7 +122,6 @@ describe('ModelCompositor', function() { {sample: {insert: 'a', deleteLeft: 0, id: 1}, p: 0.4} // but at lower weight than 'and' (998). ]; - await compositor.predict({insert: '', deleteLeft: 0, id: 0}, context); // Initialize context tracking first! let suggestions = await compositor.predict(inputDistribution, context); // remove the keep suggestion; we're not testing that here. @@ -898,12 +896,11 @@ describe('ModelCompositor', function() { assert.equal(reversion.transformId, -baseSuggestion.transformId); assert.equal(reversion.id, -baseSuggestion.id); - let postContext = models.applyTransform(baseSuggestion.appendedTransform, models.applyTransform(baseSuggestion.transform, baseContext)); - const appliedContextState = compositor.contextTracker.analyzeState(model, postContext, emptyInput(15)); + const appliedContextState = compositor.contextTracker.unitTestEndPoints.cache().get(13); // Accepting the suggestion rewrites the latest context transition. - assert.equal(compositor.contextTracker.unitTestEndPoints.cache().size, 2); - assert.sameMembers(compositor.contextTracker.unitTestEndPoints.cache().keys(), [15, baseSuggestion.transformId]); + assert.equal(compositor.contextTracker.unitTestEndPoints.cache().size, 1); + assert.sameMembers(compositor.contextTracker.unitTestEndPoints.cache().keys(), [baseSuggestion.transformId]); // The replacement should be marked on the context-tracking token for the applied version of the results. assert.equal(suggestionContextState.final.appliedSuggestionId, undefined);