From 4680295108aa2dcf9fa2f762ad3bbe737965de59 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 25 Aug 2025 15:22:45 -0500 Subject: [PATCH 1/2] fix(web): fix error that occurs when applying suggestion to empty token --- .../src/main/correction/context-state.ts | 5 +- .../main/correction/context-tokenization.ts | 83 +++++++++++-------- .../context/context-state.tests.ts | 81 +++++++++++++++++- .../context/context-tokenization.tests.ts | 20 +++++ .../context/context-tracker.tests.ts | 8 +- .../worker-model-compositor.tests.ts | 11 +-- 6 files changed, 157 insertions(+), 51 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts index bc98787ed7..cac2c7a9d3 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts @@ -197,7 +197,10 @@ export class ContextState { // computeDistance 'radius' in the called function. const alignmentResults = this.tokenization.computeAlignment(tokenizedContext.map((token) => token.text), isSliding, isApplyingSuggestion); - if(!alignmentResults.canAlign) { + if(alignmentResults.canAlign == false) { // Needs to be explicit for TS type inference. + if(console && console.error) { + console.error(`Could not align contexts with edit path ${JSON.stringify(alignmentResults.editPath)}`); + } return null; } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index bcfd3b9efe..71891f91ba 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -8,9 +8,9 @@ */ import { ContextToken } from './context-token.js'; -import { ClassicalDistanceCalculation } from './classical-calculation.js'; +import { ClassicalDistanceCalculation, EditOperation } from './classical-calculation.js'; import { getEditPathLastMatch, isSubstitutionAlignable } from './alignment-helpers.js'; -import { Token } from '@keymanapp/models-templates'; +import { SENTINEL_CODE_UNIT, Token } from '@keymanapp/models-templates'; import { LexicalModelTypes } from '@keymanapp/common-types'; import Distribution = LexicalModelTypes.Distribution; @@ -26,7 +26,14 @@ export type ContextStateAlignment = { /** * Denotes whether or not alignment is possible between two contexts. */ - canAlign: false + canAlign: false, + + /** + * Indicates the edit path that could not be handled. (Useful for error reporting) + * + * The edit path does not include actual user text and is sanitized. + */ + editPath: EditOperation[]; } | { /** * Denotes whether or not alignment is possible between two contexts. @@ -112,10 +119,27 @@ export class ContextTokenization { // Map the tokenized state to an edit-distance friendly version. const tokenizationToMatch = this.exampleInput; + const src = tokenizationToMatch.map(value => ({key: value})); + const dst = incomingTokenization.map(value => ({key: value})); + + // let changedEmptyTail = false; + if(dst[dst.length - 1].key == '') { + // Only allow matching if the tokenizations are identical, thus the empty + // token was unaffected. + if(src.length != dst.length || src[dst.length - 1].key != '') { + // Do not allow empty-token matches to match each other; this complicates + // things when applying zero-root suggestions. + // + // The SENTINEL char should never appear in raw text, thus should never + // match anything in the "tokenization to match". + dst[dst.length - 1].key = SENTINEL_CODE_UNIT; + } + } + // Inverted order, since 'match' existed before our new context. let mapping = ClassicalDistanceCalculation.computeDistance( - tokenizationToMatch.map(value => ({key: value})), - incomingTokenization.map(value => ({key: value})), + src, + dst, // Diagonal width to consider must be at least 2, as adding a single // whitespace after a token tends to add two tokens: one for whitespace, // one for the empty token to follow it. @@ -123,6 +147,11 @@ export class ContextTokenization { ); let editPath = mapping.editPath(); + const failure: ContextStateAlignment = { + canAlign: false, + editPath + }; + // Special case: new context bootstrapping - first token often substitutes. // The text length is small enough that no words should be able to rotate out the start of the context. // Special handling needed in case of no 'match'; the rest of the method assumes at least one 'match'. @@ -133,9 +162,7 @@ export class ContextTokenization { if(editPath[i] == 'substitute') { subCount++; if(!noSubVerify && !isSubstitutionAlignable(incomingTokenization[i], tokenizationToMatch[i], true)) { - return { - canAlign: false - }; + return failure; } } else if(editPath[i] == 'match') { // If a substitution is already recorded, treat the 'match' as a substitution. @@ -165,16 +192,12 @@ export class ContextTokenization { if(firstMatch == -1) { // If there are no matches, there's no alignment. - return { - canAlign: false - }; + return failure; } // Transpositions are not allowed at the token level during context alignment. if(editPath.find((entry) => entry.indexOf('transpose') > -1)) { - return { - canAlign: false - }; + return failure; } const lastMatch = getEditPathLastMatch(editPath); @@ -183,9 +206,7 @@ export class ContextTokenization { // continuous block of 'match' entries. If there's anything else in // the middle, we have a context mismatch. if(lastMatch == -1) { - return { - canAlign: false - }; + return failure; } let matchLength = lastMatch - firstMatch + 1; @@ -201,9 +222,7 @@ export class ContextTokenization { if(tailInsertLength > 0 && tailDeleteLength > 0) { // Something's gone weird if this happens; that should appear as a substitution instead. // Otherwise, we have a VERY niche edit scenario. - return { - canAlign: false - }; + return failure; } const tailSubstituteLength = (editPath.length - 1 - lastMatch) - tailInsertLength - tailDeleteLength; @@ -241,9 +260,7 @@ export class ContextTokenization { // All deletions should appear at the sliding window edge; if a deletion appears // after the edge, but before the first match, something's wrong. if(priorEdit && priorEdit != 'delete') { - return { - canAlign: false - }; + failure; } leadTokensRemoved++; break; @@ -253,9 +270,7 @@ export class ContextTokenization { // Any extras in the front would be pure inserts, not substitutions, due to // the sliding context window and its implications. if(leadSubstitutions++ > 0) { - return { - canAlign: false - }; + failure; } // Find the word before and after substitution. @@ -271,9 +286,7 @@ export class ContextTokenization { // Exception: if the word is at the start of the context window and the // context window is likely sliding, don't check it. if(!noSubVerify && !atSlidePoint && !isSubstitutionAlignable(incomingSub, matchingSub)) { - return { - canAlign: false - }; + failure; } // There's no major need to drop parts of a token being 'slid' out of the context window. @@ -283,9 +296,7 @@ export class ContextTokenization { case 'insert': // Only allow an insert at the leading edge, as with 'delete's. if(priorEdit && priorEdit != 'insert') { - return { - canAlign: false - }; + failure; } // In case of backspaces, it's also possible to 'insert' a 'new' // token - an old one that's slid back into view. @@ -294,9 +305,7 @@ export class ContextTokenization { default: // No 'match' can exist before the first found index for a 'match'. // No 'transpose-' edits should exist within this section, either. - return { - canAlign: false - }; + failure; } priorEdit = editPath[i]; } @@ -350,7 +359,9 @@ export class ContextTokenization { // If we have a perfect match with a pre-existing tokenization, no mutations have // happened; just re-use the old context tokenization. if(tailEditLength == 0 && leadTokenShift == 0 && tailTokenShift == 0) { - return this; + // We must build a new instance in case the original did not have + // alignment data (like when it's the initial context!) + return new ContextTokenization(this.tokens, alignment); } else { // If we didn't get any input, we really should perfectly match // a previous context state. If such a state is out of our cache, diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts index c140fa83aa..87548e2096 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts @@ -80,7 +80,61 @@ describe('ContextState', () => { describe('analyzeTransition', function() { - it("properly matches and aligns when lead token is removed", function() { + it("properly matches and aligns when no context changes occur (end of word)", function() { + let existingContext = { + left: "an apple a day keeps the doctor", startOfBuffer: true, endOfBuffer: true + }; + let transform = { + insert: '', + deleteLeft: 0 + }; + let newContext = { + left: "an apple a day keeps the doctor", startOfBuffer: true, endOfBuffer: true + }; + let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"]; + + let baseState = new ContextState(existingContext, plainModel); + let newContextMatch = baseState.analyzeTransition(newContext, toWrapperDistribution(transform)); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + + // Phrased this way to facilitate TS type-inference; assert.isTrue() does + // NOT do this for us! + if(!newContextMatch.final.tokenization.alignment.canAlign) { + assert.fail("context alignment failed"); + } + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0); + }); + + it("properly matches and aligns when no context changes occur (after whitespace)", function() { + let existingContext = { + left: "an apple a day keeps the doctor ", startOfBuffer: true, endOfBuffer: true + }; + let transform = { + insert: '', + deleteLeft: 0 + }; + let newContext = { + left: "an apple a day keeps the doctor ", startOfBuffer: true, endOfBuffer: true + }; + let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""]; + + let baseState = new ContextState(existingContext, plainModel); + let newContextMatch = baseState.analyzeTransition(newContext, toWrapperDistribution(transform)); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + + // Phrased this way to facilitate TS type-inference; assert.isTrue() does + // NOT do this for us! + if(!newContextMatch.final.tokenization.alignment.canAlign) { + assert.fail("context alignment failed"); + } + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0); + }); + + it("properly matches and aligns when lead token is removed (end of word)", function() { let existingContext = { left: "an apple a day keeps the doctor", startOfBuffer: true, endOfBuffer: true }; @@ -105,6 +159,31 @@ describe('ContextState', () => { assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0); }); + it("properly matches and aligns when lead token is removed (after whitespace)", function() { + let existingContext = { + left: "an apple a day keeps the doctor ", startOfBuffer: true, endOfBuffer: true + }; + let transform = { + insert: '', + deleteLeft: 0 + }; + let newContext = { + left: " apple a day keeps the doctor ", startOfBuffer: true, endOfBuffer: true + }; + let rawTokens = [" ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""]; + + let baseState = new ContextState(existingContext, plainModel); + let newContextMatch = baseState.analyzeTransition(newContext, toWrapperDistribution(transform)); + assert.isNotNull(newContextMatch?.final); + assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens); + + if(!newContextMatch.final.tokenization.alignment.canAlign) { + assert.fail("context alignment failed"); + } + assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, -1); + assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0); + }); + it("properly matches and aligns when lead token + following whitespace are removed", function() { let existingContext = { left: "an apple a day keeps the doctor", startOfBuffer: true, endOfBuffer: true diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index d8287decee..ef328a5258 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -512,6 +512,26 @@ describe('ContextTokenization', function() { }); }); + it("handles late-context application of default suggestion", () => { + const baseContext = [ + 'quick', ' ', 'brown', ' ', 'fox', ' ', 'jumped', ' ', 'over', ' ', '' + ]; + const newContext = [ + 'quick', ' ', 'brown', ' ', 'fox', ' ', 'jumped', ' ', 'over', ' ', 'the', ' ', '' + ]; + + const baseTokenization = buildBaseTokenization(baseContext); + const computedAlignment = baseTokenization.computeAlignment(newContext, false); + + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: 0, + matchLength: 10, + tailEditLength: 1, + tailTokenShift: 2 + }); + }); + it("handles sliding context-window scenarios", () => { // // Explicitly-defined window, though it's not needed directly by the method. // const config = { diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts index 2e7fc05a3f..674ed6abab 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tracker.tests.ts @@ -14,9 +14,6 @@ import { ModelCompositor, models } from '@keymanapp/lm-worker/test-index'; import TrieModel = models.TrieModel; - -var emptyInput = (id: number) => [{sample: {insert: '', deleteLeft: 0, id: id}, p: 1}]; - describe('ContextTracker', function() { describe('suggestion acceptance tracking', function() { let englishPunctuation = { @@ -62,7 +59,7 @@ describe('ContextTracker', function() { compositor.initContextTracker(baseContext, 0); const contextTracker = compositor.contextTracker; - let preAppliedTransition = contextTracker.analyzeState(model, baseContext, emptyInput(0)); + let preAppliedTransition = contextTracker.latest; // We'll ignore and overwrite the results. We do need the prediction round to occur, though. await compositor.predict([{sample: postTransform, p: 1}], baseContext); contextTracker.latest.final.suggestions = [baseSuggestion]; @@ -74,8 +71,7 @@ describe('ContextTracker', function() { contextTracker.unitTestEndPoints.cache().keys().forEach((key) => assert.isDefined(key)); // Next step - on the followup context, is the replacement still active? - let postContext = models.applyTransform(baseSuggestion.appendedTransform, models.applyTransform(baseSuggestion.transform, baseContext)); - let postContextMatch = compositor.contextTracker.analyzeState(model, postContext, emptyInput(2)); + let postContextMatch = contextTracker.unitTestEndPoints.cache().get(2); assert.equal(postContextMatch.final.appliedSuggestionId, baseSuggestion.id); // Penultimate token corresponds to whitespace, which does not have a 'raw' representation. diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts index 82bfcee60e..4affcfd8c0 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-model-compositor.tests.ts @@ -19,8 +19,6 @@ import Suggestion = LexicalModelTypes.Suggestion; import Transform = LexicalModelTypes.Transform; import TrieModel = models.TrieModel; -var emptyInput = (id: number) => [{sample: {insert: '', deleteLeft: 0, id: id}, p: 1}]; - describe('ModelCompositor', function() { describe('Prediction with 14.0+ models', function() { describe('Basic suggestion generation', function() { @@ -115,6 +113,7 @@ describe('ModelCompositor', function() { let context = { left: '', startOfBuffer: true, endOfBuffer: true, }; + compositor.initContextTracker(context, 0); // Initialize context tracking first! // The 'weights' involved imply that we have an edge-case fat finger on the bottom of // the 'q' key, slightly in its favor. @@ -123,7 +122,6 @@ describe('ModelCompositor', function() { {sample: {insert: 'a', deleteLeft: 0, id: 1}, p: 0.4} // but at lower weight than 'and' (998). ]; - await compositor.predict({insert: '', deleteLeft: 0, id: 0}, context); // Initialize context tracking first! let suggestions = await compositor.predict(inputDistribution, context); // remove the keep suggestion; we're not testing that here. @@ -898,12 +896,11 @@ describe('ModelCompositor', function() { assert.equal(reversion.transformId, -baseSuggestion.transformId); assert.equal(reversion.id, -baseSuggestion.id); - let postContext = models.applyTransform(baseSuggestion.appendedTransform, models.applyTransform(baseSuggestion.transform, baseContext)); - const appliedContextState = compositor.contextTracker.analyzeState(model, postContext, emptyInput(15)); + const appliedContextState = compositor.contextTracker.unitTestEndPoints.cache().get(13); // Accepting the suggestion rewrites the latest context transition. - assert.equal(compositor.contextTracker.unitTestEndPoints.cache().size, 2); - assert.sameMembers(compositor.contextTracker.unitTestEndPoints.cache().keys(), [15, baseSuggestion.transformId]); + assert.equal(compositor.contextTracker.unitTestEndPoints.cache().size, 1); + assert.sameMembers(compositor.contextTracker.unitTestEndPoints.cache().keys(), [baseSuggestion.transformId]); // The replacement should be marked on the context-tracking token for the applied version of the results. assert.equal(suggestionContextState.final.appliedSuggestionId, undefined); From d35bd80a909975929e2c9b934a7cf7c9fef2e558 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 25 Aug 2025 15:48:05 -0500 Subject: [PATCH 2/2] fix(web): fix last commit's fix - missed the `return` keyword a few times --- .../main/correction/context-tokenization.ts | 10 +++++----- .../worker-thread/src/main/test-index.ts | 2 +- .../context/context-tokenization.tests.ts | 20 +++++++++---------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 71891f91ba..943a042289 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -260,7 +260,7 @@ export class ContextTokenization { // All deletions should appear at the sliding window edge; if a deletion appears // after the edge, but before the first match, something's wrong. if(priorEdit && priorEdit != 'delete') { - failure; + return failure; } leadTokensRemoved++; break; @@ -270,7 +270,7 @@ export class ContextTokenization { // Any extras in the front would be pure inserts, not substitutions, due to // the sliding context window and its implications. if(leadSubstitutions++ > 0) { - failure; + return failure; } // Find the word before and after substitution. @@ -286,7 +286,7 @@ export class ContextTokenization { // Exception: if the word is at the start of the context window and the // context window is likely sliding, don't check it. if(!noSubVerify && !atSlidePoint && !isSubstitutionAlignable(incomingSub, matchingSub)) { - failure; + return failure; } // There's no major need to drop parts of a token being 'slid' out of the context window. @@ -296,7 +296,7 @@ export class ContextTokenization { case 'insert': // Only allow an insert at the leading edge, as with 'delete's. if(priorEdit && priorEdit != 'insert') { - failure; + return failure; } // In case of backspaces, it's also possible to 'insert' a 'new' // token - an old one that's slid back into view. @@ -305,7 +305,7 @@ export class ContextTokenization { default: // No 'match' can exist before the first found index for a 'match'. // No 'transpose-' edits should exist within this section, either. - failure; + return failure; } priorEdit = editPath[i]; } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts b/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts index b65d566197..01d5d05c5a 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts @@ -1,7 +1,7 @@ export { ClassicalDistanceCalculation } from './correction/classical-calculation.js'; export { ContextState } from './correction/context-state.js'; export { ContextToken } from './correction/context-token.js'; -export { ContextTokenization } from './correction/context-tokenization.js'; +export { ContextStateAlignment, ContextTokenization } from './correction/context-tokenization.js'; export { ContextTracker } from './correction/context-tracker.js'; export { ContextTransition } from './correction/context-transition.js'; export { EditOperation } from './correction/classical-calculation.js'; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index ef328a5258..7b2619abda 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -13,7 +13,7 @@ import { assert } from 'chai'; import { default as defaultBreaker } from '@keymanapp/models-wordbreakers'; import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; -import { ContextToken, ContextTokenization, models } from '@keymanapp/lm-worker/test-index'; +import { ContextStateAlignment, ContextToken, ContextTokenization, models } from '@keymanapp/lm-worker/test-index'; import TrieModel = models.TrieModel; @@ -46,7 +46,7 @@ describe('ContextTokenization', function() { it("constructs from a token array + alignment data", () => { const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; - let alignment = { + let alignment: ContextStateAlignment = { canAlign: true, leadTokenShift: 0, matchLength: 6, @@ -171,7 +171,7 @@ describe('ContextTokenization', function() { const baseTokenization = buildBaseTokenization(baseContext); const computedAlignment = baseTokenization.computeAlignment(newContext, false); - assert.deepEqual(computedAlignment, {canAlign: false}); + assert.deepEqual(computedAlignment, {canAlign: false, editPath: ['substitute', 'substitute', 'substitute', 'substitute', 'substitute']}); }); it("detects unalignable contexts - too many mismatching tokens", () => { @@ -185,7 +185,7 @@ describe('ContextTokenization', function() { const baseTokenization = buildBaseTokenization(baseContext); const computedAlignment = baseTokenization.computeAlignment(newContext, false); - assert.deepEqual(computedAlignment, {canAlign: false}); + assert.deepEqual(computedAlignment, {canAlign: false, editPath: ['substitute', 'substitute', 'match', 'match', 'match']}); }); it("fails alignment for leading-edge word substitutions", () => { @@ -199,7 +199,7 @@ describe('ContextTokenization', function() { const baseTokenization = buildBaseTokenization(baseContext); const computedAlignment = baseTokenization.computeAlignment(newContext, false); - assert.deepEqual(computedAlignment, {canAlign: false}); + assert.deepEqual(computedAlignment, {canAlign: false, editPath: ['substitute', 'match', 'match', 'match', 'match']}); }); it("fails alignment for small leading-edge word substitutions", () => { @@ -213,7 +213,7 @@ describe('ContextTokenization', function() { const baseTokenization = buildBaseTokenization(baseContext); const computedAlignment = baseTokenization.computeAlignment(newContext, false); - assert.deepEqual(computedAlignment, {canAlign: false}); + assert.deepEqual(computedAlignment, {canAlign: false, editPath: ['substitute', 'match', 'match', 'match', 'match']}); }); it("properly matches and aligns when lead token is modified", () => { @@ -447,7 +447,7 @@ describe('ContextTokenization', function() { const baseTokenization = buildBaseTokenization(baseContext); const computedAlignment = baseTokenization.computeAlignment(newContext, false); - assert.deepEqual(computedAlignment, {canAlign: false}); + assert.deepEqual(computedAlignment, {canAlign: false, editPath: ["match", "delete", "match", "match", "match"]}); }); it("fails alignment for mid-head insertion", () => { @@ -461,7 +461,7 @@ describe('ContextTokenization', function() { const baseTokenization = buildBaseTokenization(baseContext); const computedAlignment = baseTokenization.computeAlignment(newContext, false); - assert.deepEqual(computedAlignment, {canAlign: false}); + assert.deepEqual(computedAlignment, {canAlign: false, editPath: ["match", "insert", "match", "match", "match"]}); }); it("fails alignment for mid-tail deletion", () => { @@ -475,7 +475,7 @@ describe('ContextTokenization', function() { const baseTokenization = buildBaseTokenization(baseContext); const computedAlignment = baseTokenization.computeAlignment(newContext, false); - assert.deepEqual(computedAlignment, {canAlign: false}); + assert.deepEqual(computedAlignment, {canAlign: false, editPath: ["match", "match", "match", "delete", "match"]}); }); it("fails alignment for mid-tail insertion", () => { @@ -489,7 +489,7 @@ describe('ContextTokenization', function() { const baseTokenization = buildBaseTokenization(baseContext); const computedAlignment = baseTokenization.computeAlignment(newContext, false); - assert.deepEqual(computedAlignment, {canAlign: false}); + assert.deepEqual(computedAlignment, {canAlign: false, editPath: ["match", "match", "match", "match", "insert", "match"]}); }); it("handles late-context suggestion application after backspace", () => {