diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/alignment-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/alignment-helpers.ts index 4f133da119..2fab233135 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/alignment-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/alignment-helpers.ts @@ -8,120 +8,7 @@ * edits for aligned context tokens. */ -import { SENTINEL_CODE_UNIT } from '@keymanapp/models-templates'; -import { ClassicalDistanceCalculation, computeDistance, EditTuple } from "./classical-calculation.js"; -import { ExtendedEditOperation } from './segmentable-calculation.js'; - -/** - * Represents token-count values resulting from an alignment attempt between two - * different modeled context states. - */ -export type ContextStateAlignment = { - /** - * Denotes whether or not alignment is possible between two contexts. - */ - canAlign: false, - - /** - * Indicates the edit path that could not be handled. (Useful for error reporting) - * - * The edit path does not include actual user text and is sanitized. - */ - editPath: EditTuple[]; -} | { - /** - * Denotes whether or not alignment is possible between two contexts. - */ - canAlign: true, - - /** - * Indicates the edit path that could not be handled. (Useful for error reporting) - * - * The edit path does not include actual user text and is sanitized. - */ - editPath: EditTuple[]; - - /** - * Notes the number of tokens added to the head of the 'incoming'/'new' context - * of the contexts being aligned. If negative, the incoming context deleted - * a token found in the 'original' / base context. - * - * For the alignment, [base context index] + leadTokenShift = [incoming context index]. - */ - leadTokenShift: number, - /** - * Notes the number of tokens at the head of the 'incoming'/'new' context, - * perfectly aligned but edited for two successfully-alignable contexts. These - * tokens directly precede those that need no edits. - * - * When a token could be considered as either 'lead' or 'tail' edit, it will - * only be reported as a 'tail' edit. - */ - leadEditLength: number, - /** - * The count of tokens perfectly aligned, with no need for edits, for two successfully- - * alignable contexts. - */ - matchLength: number, - /** - * The count of tokens at the tail perfectly aligned (existing in both contexts) but - * edited for two successfully-alignable contexts. These tokens directly follow those - * that need no edits. - */ - tailEditLength: number, - /** - * The count of new tokens added at the end of the incoming context for two aligned contexts. - * If negative, the incoming context deleted a previously-existing token from the original. - */ - tailTokenShift: number -}; - - -/** - * Determines the proper 'last match' index for a tokenized sequence based on its edit path. - * - * In particular, this method is designed to handle the following cases: - * - ['to', ' ', 'apple', ' ', ''] => ['to', ' ', 'apply', ' ', ''] - * - ['to', ' ', 'apple', ' ', ''] => ['to', ' ', 'apply', ' ', 'n'] - * - * Edit path for these example cases: - * - ['match', 'match', 'substitute', 'match', 'match'] - * - ['match', 'match', 'substitute', 'match', 'substitute'] - * - * In cases such as these, the late whitespace match should be considered 'edited'. While the - * ' ' is unedited, it follows the edited 'apple' => 'apply', so it must have been deleted and - * then re-inserted. As a result, the whitespace after 'to' is the true "last matched" token. - * - * Returns -1 if an unexpected edit other than 'substitute' occurs in the middle of the big - * 'match' block. - * @param editPath - * @returns - */ -export function getEditPathLastMatch(editPath: ExtendedEditOperation[], forAppliedSuggestion?: boolean) { - // Assertion: for a long context, the bulk of the edit path should be a - // continuous block of 'match' entries. If there's anything but a substitution - // in the middle, we have a context mismatch. - // - // That said, it is possible to apply a suggestion after a backspace. Anything - // after the substitution needs to be treated as a substitution rather than - // a match. - const firstMatch = editPath.indexOf('match'); - const lastMatch = editPath.lastIndexOf('match'); - if(firstMatch > -1) { - for(let i = firstMatch+1; i <= lastMatch; i++) { - if(editPath[i] != 'match') { - // fun case: ' ' + ' applied ' has an unusual edit path. - // we get an 'insert'. - return ( - (editPath[i] == 'substitute') - || (forAppliedSuggestion && editPath[i] == 'insert') - ) ? (i - 1) : -1; - } - } - } - - return lastMatch; -} +import { ClassicalDistanceCalculation, computeDistance } from "./classical-calculation.js"; /** * Aligns two tokens on a character-by-character basis as needed for higher, token-level alignment @@ -214,229 +101,4 @@ export function isSubstitutionAlignable( } return true; -} - -/** - * Determines the alignment between a new, incoming tokenization source and the - * tokenization modeled by the current instance. - * @param tokenizationToMatch Raw strings corresponding to the tokenization of the original context - * @param incomingTokenization Raw strings corresponding to the tokenization of the incoming context - * @param isSliding Notes if the context window is full (and sliding-alignment is particularly needed) - * @param forAppliedSuggestion When true, this asserts that the contexts are alignable and loosens - * alignment requirements accordingly. - * @returns Alignment data that details if and how the incoming tokenization aligns with - * the tokenization modeled by this instance. - */ -export function computeAlignment( - tokenizationToMatch: string[], - incomingTokenization: string[], - isSliding: boolean, - forAppliedSuggestion?: boolean -): ContextStateAlignment { - const src = tokenizationToMatch; - const dst = incomingTokenization; - - // let changedEmptyTail = false; - if(dst[dst.length - 1] == '') { - // Only allow matching if the tokenizations are identical, thus the empty - // token was unaffected. - if(src.length != dst.length || src[dst.length - 1] != '') { - // Do not allow empty-token matches to match each other; this complicates - // things when applying zero-root suggestions. - // - // The SENTINEL char should never appear in raw text, thus should never - // match anything in the "tokenization to match". - dst[dst.length - 1] = SENTINEL_CODE_UNIT; - } - } - - // Inverted order, since 'match' existed before our new context. - const mapping = computeDistance( - // Diagonal width allows asymmetric edits and is also needed to cover - // difference in length for the inputs. We should try to cover at least 2 - // edits on one side in addition to potential length asymmetry. - new ClassicalDistanceCalculation({diagonalWidth: Math.abs(src.length - dst.length) + 3}), - src, - dst - ); - - // Later iteration: we could return this itself directly for use in alignment - // operations, rather than relying solely on the edit-op names. - let editPaths = mapping.editPath(); - if(editPaths.length == 0) { - console.error(`Could not compute edit path for aligning contexts of length ${src.length}, ${dst.length}`); - } - let editPath = editPaths[0].map(t => t.op); - - const failure: ContextStateAlignment = { - canAlign: false, - editPath: editPaths[0] - }; - - // Special case: new context bootstrapping - first token often substitutes. - // The text length is small enough that no words should be able to rotate out the start of the context. - // Special handling needed in case of no 'match'; the rest of the method assumes at least one 'match'. - if(editPath.length <= 3 && (editPath[0] == 'substitute' || editPath[0] == 'match')) { - let matchCount = 0; - let subCount = 0; - for(let i = 0; i < editPath.length; i++) { - if(editPath[i] == 'substitute') { - subCount++; - if(!forAppliedSuggestion && !isSubstitutionAlignable(incomingTokenization[i], tokenizationToMatch[i], true)) { - return failure; - } - } else if(editPath[i] == 'match') { - // If a substitution is already recorded, treat the 'match' as a substitution. - if(subCount > 0) { - subCount++; - } else { - matchCount++; - } - } - } - - const insertCount = editPath.filter((entry) => entry == 'insert').length; - const deleteCount = editPath.filter((entry) => entry == 'delete').length; - - return { - canAlign: true, - editPath: editPaths[0], - matchLength: matchCount, - leadTokenShift: 0, - leadEditLength: 0, - tailEditLength: subCount, - tailTokenShift: insertCount - deleteCount - } - } - - // From here on assumes that at least one 'match' exists on the path. - // It all works great... once the context is long enough for at least one stable token. - const firstMatch = editPath.indexOf('match'); - - if(firstMatch == -1) { - // If there are no matches, there's no alignment. - return failure; - } - - // Transpositions are not allowed at the token level during context alignment. - if(editPath.find((entry) => entry.indexOf('transpose') > -1)) { - return failure; - } - - const lastMatch = getEditPathLastMatch(editPath, forAppliedSuggestion); - - // Assertion: for a long context, the bulk of the edit path should be a - // continuous block of 'match' entries. If there's anything else in - // the middle, we have a context mismatch. - if(lastMatch == -1) { - return failure; - } - - let matchLength = lastMatch - firstMatch + 1; - let tailInsertLength = 0; - let tailDeleteLength = 0; - for(let i = lastMatch; i < editPath.length; i++) { - if(editPath[i] == 'insert') { - tailInsertLength++; - } else if(editPath[i] == 'delete') { - tailDeleteLength++; - } - } - if(tailInsertLength > 0 && tailDeleteLength > 0) { - // Something's gone weird if this happens; that should appear as a substitution instead. - // Otherwise, we have a VERY niche edit scenario. - return failure; - } - const tailSubstituteLength = (editPath.length - 1 - lastMatch) - tailInsertLength - tailDeleteLength; - - // If we have a perfect match with a pre-existing context, no mutations have - // happened; we have a 100% perfect match. - if(firstMatch == 0 && lastMatch == editPath.length - 1) { - return { - canAlign: true, - editPath: editPaths[0], - leadTokenShift: 0, - leadEditLength: 0, - matchLength, - tailEditLength: tailSubstituteLength, - tailTokenShift: tailInsertLength - tailDeleteLength - }; - } - - // The edit path calc tries to put substitutes first, before inserts. - // We don't want that on the leading edge. - const lastEarlyInsert = editPath.lastIndexOf('insert', firstMatch); - const firstSubstitute = editPath.indexOf('substitute'); - if(firstSubstitute > -1 && firstSubstitute < firstMatch && firstSubstitute < lastEarlyInsert) { - editPath[firstSubstitute] = 'insert'; - editPath[lastEarlyInsert] = 'substitute'; - } - - // If mutations HAVE happened, we need to double-check the context-state alignment. - let priorEdit: typeof editPath[0]; - let leadTokensRemoved = 0; - let leadSubstitutions = 0; - - // The `i` index below aligns based upon the index within the `tokenizationToMatch` sequence - // and how it would have to be edited to align to the `incomingTokenization` sequence. - for(let i = 0; i < firstMatch; i++) { - switch(editPath[i]) { - case 'delete': - // All deletions should appear at the sliding window edge; if a deletion appears - // after the edge, but before the first match, something's wrong. - if(priorEdit && priorEdit != 'delete') { - return failure; - } - leadTokensRemoved++; - break; - case 'substitute': - // Find the word before and after substitution. - const incomingIndex = i - (leadTokensRemoved > 0 ? leadTokensRemoved : 0); - const matchingIndex = i + (leadTokensRemoved < 0 ? leadTokensRemoved : 0); - const incomingSub = incomingTokenization[incomingIndex]; - const matchingSub = tokenizationToMatch[matchingIndex]; - - const atSlidePoint = isSliding && (incomingIndex == 0 || matchingIndex == 0); - - // Double-check the word - does the 'substituted' word itself align? - // - // Exception: if the word is at the start of the context window and the - // context window is likely sliding, don't check it. - if(!forAppliedSuggestion && !atSlidePoint && !isSubstitutionAlignable(incomingSub, matchingSub)) { - return failure; - } - - leadSubstitutions++; - break; - case 'insert': - // Only allow an insert at the leading edge, as with 'delete's. - if(priorEdit && priorEdit != 'insert') { - return failure; - } - // In case of backspaces, it's also possible to 'insert' a 'new' - // token - an old one that's slid back into view. - leadTokensRemoved--; - break; - default: - // No 'match' can exist before the first found index for a 'match'. - // No 'transpose-' edits should exist within this section, either. - return failure; - } - priorEdit = editPath[i]; - } - - // If we need some form of tail-token substitution verification, add that here. - - return { - canAlign: true, - editPath: editPaths[0], - // leadTokensRemoved represents the number of tokens that must be removed from the base context - // when aligning the contexts. Externally, it's more helpful to think in terms of the count added - // to the incoming context. - leadTokenShift: -leadTokensRemoved + 0, // add 0 in case of a 'negative zero', which affects unit tests. - leadEditLength: leadSubstitutions, - matchLength, - tailEditLength: tailSubstituteLength, - tailTokenShift: tailInsertLength - tailDeleteLength - }; } \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/alignment-helpers.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/alignment-helpers.tests.ts index d25528a7fe..b7c0052d62 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/alignment-helpers.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/alignment-helpers.tests.ts @@ -9,39 +9,7 @@ */ import { assert } from 'chai'; -import { computeAlignment, EditOperation, getEditPathLastMatch, isSubstitutionAlignable } from '@keymanapp/lm-worker/test-index'; - -describe('getEditPathLastMatch', () => { - it('returns the last match when no substitutions exist', () => { - const path: EditOperation[] = ['delete', 'delete', 'match', 'match', 'match', 'match', 'insert']; - assert.equal(path.lastIndexOf('match'), 5); - assert.equal(getEditPathLastMatch(path), 5); - }); - - it('returns the last match when no substitutions exist left of a "match"', () => { - const path: EditOperation[] = ['delete', 'delete', 'match', 'match', 'match', 'match', 'substitute', 'insert']; - assert.equal(path.lastIndexOf('match'), 5); - assert.equal(getEditPathLastMatch(path), 5); - }); - - // is intended to handle application of suggestions. - it('returns the last match before a substitute occurring after the first match', () => { - // limitation: if there is _anything_ after that last match, the first assertion will fail. - // 0 1 2 3 4 5 6 - const path: EditOperation[] = ['delete', 'delete', 'match', 'match', 'substitute', 'match', 'match']; - assert.notEqual(getEditPathLastMatch(path), 6); - assert.equal(getEditPathLastMatch(path), 3); - }); - - // is intended to handle complex transforms that include a whitespace and affect prior tokens. - it('returns the last match before a substitute occurring after the first match', () => { - // limitation: if there is _anything_ after that last match, the first assertion will fail. - // 0 1 2 3 4 5 6 - const path: EditOperation[] = ['delete', 'delete', 'match', 'match', 'substitute', 'match', 'substitute']; - assert.notEqual(getEditPathLastMatch(path), 5); - assert.equal(getEditPathLastMatch(path), 3); - }); -}); +import { isSubstitutionAlignable } from '@keymanapp/lm-worker/test-index'; describe('isSubstitutionAlignable', () => { it(`returns true: 'ca' => 'can'`, () => { @@ -120,773 +88,4 @@ describe('isSubstitutionAlignable', () => { // The double-p adds a fun complication once the first gets dropped. assert.isTrue(isSubstitutionAlignable('applesauce', 'plesauce')); }); -}); - - -describe('computeAlignment', () => { - it("properly matches and aligns when contexts match", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - const newContext = [...baseContext]; - - const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, { - canAlign: true, - editPath: [ - {op: 'match', input: 0, match: 0}, - {op: 'match', input: 1, match: 1}, - {op: 'match', input: 2, match: 2}, - {op: 'match', input: 3, match: 3}, - {op: 'match', input: 4, match: 4} - ], - leadTokenShift: 0, - leadEditLength: 0, - matchLength: 5, - tailEditLength: 0, - tailTokenShift: 0 - }); - }); - - it("properly matches and aligns with applied-suggestion contexts", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'o' - ]; - const newContext = [...baseContext]; - newContext[4] = 'over'; - - const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, { - canAlign: true, - editPath: [ - {op: 'match', input: 0, match: 0}, - {op: 'match', input: 1, match: 1}, - {op: 'match', input: 2, match: 2}, - {op: 'match', input: 3, match: 3}, - {op: 'substitute', input: 4, match: 4} - ], - leadTokenShift: 0, - leadEditLength: 0, - matchLength: 4, - tailEditLength: 1, - tailTokenShift: 0 - }); - }); - - it("properly matches and aligns with applied-suggestion at start of context", () => { - const baseContext = [ - 'te' - ]; - const newContext = [ - 'testing', - ' ', - '' - ]; - - const computedAlignment = computeAlignment(baseContext, newContext, false, true); - assert.deepEqual(computedAlignment, { - canAlign: true, - editPath: [ - {op: 'substitute', input: 0, match: 0}, - {op: 'insert', match: 1}, - {op: 'insert', match: 2} - ], - leadTokenShift: 0, - leadEditLength: 0, - matchLength: 0, - tailEditLength: 1, - tailTokenShift: 2 - }); - }); - - it("detects unalignable contexts - no matching tokens", () => { - const baseContext = [ - 'swift', 'tan', 'wolf', 'leaped', 'across' - ]; - const newContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - - const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, { - canAlign: false, - editPath: [ - { op: 'substitute', input: 0, match: 0 }, - { op: 'substitute', input: 1, match: 1 }, - { op: 'substitute', input: 2, match: 2 }, - { op: 'substitute', input: 3, match: 3 }, - { op: 'substitute', input: 4, match: 4 } - ] - }); - }); - - it("detects unalignable contexts - too many mismatching tokens", () => { - const baseContext = [ - 'swift', 'tan', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - - const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, { - canAlign: false, - editPath: [ - {op: 'substitute', input: 0, match: 0}, - {op: 'substitute', input: 1, match: 1}, - {op: 'match', input: 2, match: 2}, - {op: 'match', input: 3, match: 3}, - {op: 'match', input: 4, match: 4} - ], - }); - }); - - it("fails alignment for leading-edge word substitutions", () => { - const baseContext = [ - 'swift', 'brown', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - - const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, { - canAlign: false, - editPath: [ - {op: 'substitute', input: 0, match: 0}, - {op: 'match', input: 1, match: 1}, - {op: 'match', input: 2, match: 2}, - {op: 'match', input: 3, match: 3}, - {op: 'match', input: 4, match: 4} - ] - }); - }); - - it("fails alignment for small leading-edge word substitutions", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'sick', 'brown', 'fox', 'jumped', 'over' - ]; - - const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, { - canAlign: false, - editPath: [ - {op: 'substitute', input: 0, match: 0}, - {op: 'match', input: 1, match: 1}, - {op: 'match', input: 2, match: 2}, - {op: 'match', input: 3, match: 3}, - {op: 'match', input: 4, match: 4} - ] - }); - }); - - it("properly matches and aligns when lead token is modified", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'uick', 'brown', 'fox', 'jumped', 'over' - ]; - - const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, { - canAlign: true, - editPath: [ - {op: 'substitute', input: 0, match: 0}, - {op: 'match', input: 1, match: 1}, - {op: 'match', input: 2, match: 2}, - {op: 'match', input: 3, match: 3}, - {op: 'match', input: 4, match: 4} - ], - leadTokenShift: 0, - leadEditLength: 1, - matchLength: 4, - tailEditLength: 0, - tailTokenShift: 0 - }); - }); - - it("properly matches and aligns when lead token is removed", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'brown', 'fox', 'jumped', 'over' - ]; - - const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, { - canAlign: true, - editPath: [ - {op: 'delete', input: 0}, - {op: 'match', input: 1, match: 0}, - {op: 'match', input: 2, match: 1}, - {op: 'match', input: 3, match: 2}, - {op: 'match', input: 4, match: 3} - ], - leadTokenShift: -1, - leadEditLength: 0, - matchLength: 4, - tailEditLength: 0, - tailTokenShift: 0 - }); - }); - - it("properly matches and aligns when lead token is added", () => { - const baseContext = [ - 'brown', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - - const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, { - canAlign: true, - editPath: [ - {op: 'insert', match: 0}, - {op: 'match', input: 0, match: 1}, - {op: 'match', input: 1, match: 2}, - {op: 'match', input: 2, match: 3}, - {op: 'match', input: 3, match: 4} - ], - leadTokenShift: 1, - leadEditLength: 0, - matchLength: 4, - tailEditLength: 0, - tailTokenShift: 0 - }); - }); - - it("properly matches and aligns when lead tokens are removed and modified", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'ox', 'jumped', 'over' - ]; - - const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, { - canAlign: true, - editPath: [ - {op: 'delete', input: 0}, - {op: 'delete', input: 1}, - {op: 'substitute', input: 2, match: 0}, - {op: 'match', input: 3, match: 1}, - {op: 'match', input: 4, match: 2}, - ], - leadTokenShift: -2, - leadEditLength: 1, - matchLength: 2, - tailEditLength: 0, - tailTokenShift: 0 - }); - }); - - it("properly matches and aligns when lead tokens are added and modified", () => { - const baseContext = [ - 'rown', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - - const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, { - canAlign: true, - editPath: [ - {op: 'insert', match: 0}, - {op: 'substitute', input: 0, match: 1}, - {op: 'match', input: 1, match: 2}, - {op: 'match', input: 2, match: 3}, - {op: 'match', input: 3, match: 4}, - ], - leadTokenShift: 1, - leadEditLength: 1, - matchLength: 3, - tailEditLength: 0, - tailTokenShift: 0 - }); - }); - - it("properly matches and aligns when lead token is removed and tail token is added", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'brown', 'fox', 'jumped', 'over', 'the' - ]; - - const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, { - canAlign: true, - editPath: [ - {op: 'delete', input: 0}, - {op: 'match', input: 1, match: 0}, - {op: 'match', input: 2, match: 1}, - {op: 'match', input: 3, match: 2}, - {op: 'match', input: 4, match: 3}, - {op: 'insert', match: 4} - ], - leadTokenShift: -1, - leadEditLength: 0, - matchLength: 4, - tailEditLength: 0, - tailTokenShift: 1 - }); - }); - - it("properly matches and aligns when lead token and tail token are modified", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'ove' - ]; - const newContext = [ - 'uick', 'brown', 'fox', 'jumped', 'over' - ]; - - const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, { - canAlign: true, - editPath: [ - {op: 'substitute', input: 0, match: 0}, - {op: 'match', input: 1, match: 1}, - {op: 'match', input: 2, match: 2}, - {op: 'match', input: 3, match: 3}, - {op: 'substitute', input: 4, match: 4} - ], - leadTokenShift: 0, - leadEditLength: 1, - matchLength: 3, - tailEditLength: 1, - tailTokenShift: 0 - }); - }); - - it("properly matches and aligns when lead token and tail token are modified + new token appended", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'ove' - ]; - const newContext = [ - 'uick', 'brown', 'fox', 'jumped', 'over', 't' - ]; - - const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, { - canAlign: true, - editPath: [ - {op: 'substitute', input: 0, match: 0}, - {op: 'match', input: 1, match: 1}, - {op: 'match', input: 2, match: 2}, - {op: 'match', input: 3, match: 3}, - {op: 'substitute', input: 4, match: 4}, - {op: 'insert', match: 5} - ], - leadTokenShift: 0, - leadEditLength: 1, - matchLength: 3, - tailEditLength: 1, - tailTokenShift: 1 - }); - }); - - it("properly handles context window sliding backward", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'e', 'quick', 'brown', 'fox', 'jumped', 'ove' - ]; - - const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, { - canAlign: true, - editPath: [ - {op: 'insert', match: 0}, - {op: 'match', input: 0, match: 1}, - {op: 'match', input: 1, match: 2}, - {op: 'match', input: 2, match: 3}, - {op: 'match', input: 3, match: 4}, - {op: 'substitute', input: 4, match: 5} - ], - leadTokenShift: 1, - leadEditLength: 0, - matchLength: 4, - tailEditLength: 1, - tailTokenShift: 0 - }); - }); - - it("properly handles context window sliding far backward", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'the', 'quick', 'brown', 'fox', 'jumped' - ]; - - const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, { - canAlign: true, - editPath: [ - {op: 'insert', match: 0}, - {op: 'match', input: 0, match: 1}, - {op: 'match', input: 1, match: 2}, - {op: 'match', input: 2, match: 3}, - {op: 'match', input: 3, match: 4}, - {op: 'delete', input: 4} - ], - leadTokenShift: 1, - leadEditLength: 0, - matchLength: 4, - tailEditLength: 0, - tailTokenShift: -1 - }); - }); - - it("properly handles context window sliding farther backward", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'the', 'quick', 'brown', 'fox', 'jumpe' - ]; - - const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, { - canAlign: true, - editPath: [ - {op: 'insert', match: 0}, - {op: 'match', input: 0, match: 1}, - {op: 'match', input: 1, match: 2}, - {op: 'match', input: 2, match: 3}, - {op: 'substitute', input: 3, match: 4}, - {op: 'delete', input: 4} - ], - leadTokenShift: 1, - leadEditLength: 0, - matchLength: 3, - tailEditLength: 1, - tailTokenShift: -1 - }); - }); - - it("fails alignment for mid-head deletion", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'quick', 'fox', 'jumped', 'over' - ]; - - const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, { - canAlign: false, - editPath: [ - {op: 'match', input: 0, match: 0}, - {op: 'delete', input: 1}, - {op: 'match', input: 2, match: 1}, - {op: 'match', input: 3, match: 2}, - {op: 'match', input: 4, match: 3} - ] - }); - }); - - it("fails alignment for mid-head insertion", () => { - const baseContext = [ - 'quick', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - - const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, { - canAlign: false, - editPath: [ - {op: 'match', input: 0, match: 0}, - {op: 'insert', match: 1}, - {op: 'match', input: 1, match: 2}, - {op: 'match', input: 2, match: 3}, - {op: 'match', input: 3, match: 4} - ] - }); - }); - - it("fails alignment for mid-tail deletion", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'quick', 'brown', 'fox', 'over' - ]; - - const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, { - canAlign: false, - editPath: [ - {op: 'match', input: 0, match: 0}, - {op: 'match', input: 1, match: 1}, - {op: 'match', input: 2, match: 2}, - {op: 'delete', input: 3}, - {op: 'match', input: 4, match: 3} - ] - }); - }); - - it("fails alignment for mid-tail insertion", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'quick', 'brown', 'fox', 'jumped', 'far', 'over' - ]; - - const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, { - canAlign: false, - editPath: [ - {op: 'match', input: 0, match: 0}, - {op: 'match', input: 1, match: 1}, - {op: 'match', input: 2, match: 2}, - {op: 'match', input: 3, match: 3}, - {op: 'insert', match: 4}, - {op: 'match', input: 4, match: 5} - ] - }); - }); - - it("handles late-context suggestion application after backspace", () => { - const baseContext = [ - 'quick', ' ', 'brown', ' ', 'fox', ' ', 'jumped', ' ', 'oven', ' ', '' - ]; - const newContext = [ - 'quick', ' ', 'brown', ' ', 'fox', ' ', 'jumped', ' ', 'over', ' ', '' - ]; - - const computedAlignment = computeAlignment(baseContext, newContext, false); - assert.deepEqual(computedAlignment, { - canAlign: true, - editPath: [ - {op: 'match', input: 0, match: 0}, - {op: 'match', input: 1, match: 1}, - {op: 'match', input: 2, match: 2}, - {op: 'match', input: 3, match: 3}, - {op: 'match', input: 4, match: 4}, - {op: 'match', input: 5, match: 5}, - {op: 'match', input: 6, match: 6}, - {op: 'match', input: 7, match: 7}, - {op: 'substitute', input: 8, match: 8}, - {op: 'match', input: 9, match: 9}, - {op: 'match', input: 10, match: 10} - ], - leadTokenShift: 0, - leadEditLength: 0, - matchLength: 8, - tailEditLength: 3, - tailTokenShift: 0 - }); - }); - - it("handles late-context application of default suggestion", () => { - const baseContext = [ - 'quick', ' ', 'brown', ' ', 'fox', ' ', 'jumped', ' ', 'over', ' ', '' - ]; - const newContext = [ - 'quick', ' ', 'brown', ' ', 'fox', ' ', 'jumped', ' ', 'over', ' ', 'the', ' ', '' - ]; - - const computedAlignment = computeAlignment(baseContext, newContext, false); - - assert.deepEqual(computedAlignment, { - canAlign: true, - editPath: [ - {op: 'match', input: 0, match: 0}, - {op: 'match', input: 1, match: 1}, - {op: 'match', input: 2, match: 2}, - {op: 'match', input: 3, match: 3}, - {op: 'match', input: 4, match: 4}, - {op: 'match', input: 5, match: 5}, - {op: 'match', input: 6, match: 6}, - {op: 'match', input: 7, match: 7}, - {op: 'match', input: 8, match: 8}, - {op: 'match', input: 9, match: 9}, - {op: 'substitute', input: 10, match: 10}, - {op: 'insert', match: 11}, - {op: 'insert', match: 12} - ], - leadTokenShift: 0, - leadEditLength: 0, - matchLength: 10, - tailEditLength: 1, - tailTokenShift: 2 - }); - }); - - it("handles sliding context-window scenarios", () => { - // // Explicitly-defined window, though it's not needed directly by the method. - // const config = { - // leftContextCodePoints: 64, - // rightContextCodePoints: 64 - // }; - - const baseContext1 = [ - // "ap" prefix not in actual view, but preserved by prior tokenization rounds. - "applesauce", " ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ", - "like", " ", "they'd", " ", "make", " ", "for", " ", "the", " ", "be" - ]; - - const incomingContext1 = [ - "plesauce", " ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ", - "like", " ", "they'd", " ", "make", " ", "for", " ", "the", " ", "bes" - ]; - - // 66 chars above, vs a sliding window of length 64. - assert.equal(baseContext1.reduce((accum, curr) => accum + curr.length, 0), 66); - // Actual window + one newly-typed character - assert.equal(incomingContext1.reduce((accum, curr) => accum + curr.length, 0), 65); - - assert.deepEqual(computeAlignment(baseContext1, incomingContext1, true), { - canAlign: true, - editPath: [ - {op: 'substitute', input: 0, match: 0}, - {op: 'match', input: 1, match: 1}, - {op: 'match', input: 2, match: 2}, - {op: 'match', input: 3, match: 3}, - {op: 'match', input: 4, match: 4}, - {op: 'match', input: 5, match: 5}, - {op: 'match', input: 6, match: 6}, - {op: 'match', input: 7, match: 7}, - {op: 'match', input: 8, match: 8}, - {op: 'match', input: 9, match: 9}, - {op: 'match', input: 10, match: 10}, - {op: 'match', input: 11, match: 11}, - {op: 'match', input: 12, match: 12}, - {op: 'match', input: 13, match: 13}, - {op: 'match', input: 14, match: 14}, - {op: 'match', input: 15, match: 15}, - {op: 'match', input: 16, match: 16}, - {op: 'match', input: 17, match: 17}, - {op: 'match', input: 18, match: 18}, - {op: 'match', input: 19, match: 19}, - {op: 'match', input: 20, match: 20}, - {op: 'match', input: 21, match: 21}, - {op: 'substitute', input: 22, match: 22} - ], - leadTokenShift: 0, - leadEditLength: 1, - matchLength: 21, - tailEditLength: 1, - tailTokenShift: 0 - }); - - // Our tokenization scheme remembers the full original word before any of it slid out of - // the context window. - const baseContext2 = [ - "applesauce", " ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ", - // +2 +1 +4 - "like", " ", "they'd", " ", "make", " ", "for", " ", "the", " ", "best", " ", "brea" - ]; - - const incomingContext2 = [ - // "plesauce" => "e": -7 chars. - "e", " ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ", - "like", " ", "they'd", " ", "make", " ", "for", " ", "the", " ", "best", " ", "break" - ]; - - // 73 chars above, vs a sliding window of length 64. - assert.equal(baseContext2.reduce((accum, curr) => accum + curr.length, 0), 73); - // Actual window + one newly-typed character - assert.equal(incomingContext2.reduce((accum, curr) => accum + curr.length, 0), 65); - - assert.deepEqual(computeAlignment(baseContext2, incomingContext2, true), { - canAlign: true, - editPath: [ - {op: 'substitute', input: 0, match: 0}, - {op: 'match', input: 1, match: 1}, - {op: 'match', input: 2, match: 2}, - {op: 'match', input: 3, match: 3}, - {op: 'match', input: 4, match: 4}, - {op: 'match', input: 5, match: 5}, - {op: 'match', input: 6, match: 6}, - {op: 'match', input: 7, match: 7}, - {op: 'match', input: 8, match: 8}, - {op: 'match', input: 9, match: 9}, - {op: 'match', input: 10, match: 10}, - {op: 'match', input: 11, match: 11}, - {op: 'match', input: 12, match: 12}, - {op: 'match', input: 13, match: 13}, - {op: 'match', input: 14, match: 14}, - {op: 'match', input: 15, match: 15}, - {op: 'match', input: 16, match: 16}, - {op: 'match', input: 17, match: 17}, - {op: 'match', input: 18, match: 18}, - {op: 'match', input: 19, match: 19}, - {op: 'match', input: 20, match: 20}, - {op: 'match', input: 21, match: 21}, - {op: 'match', input: 22, match: 22}, - {op: 'match', input: 23, match: 23}, - {op: 'substitute', input: 24, match: 24} - ], - leadTokenShift: 0, - leadEditLength: 1, - matchLength: 23, - tailEditLength: 1, - tailTokenShift: 0 - }); - - const baseContext3 = [ - "applesauce", " ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ", "like", " ", - "they'd", " ", "make", " ", "for", " ", "the", " ", "best", " ", "break" - ]; - - const incomingContext3 = [ - " ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ", "like", " ", - "they'd", " ", "make", " ", "for", " ", "the", " ", "best", " ", "breakf" - ]; - - // 74 chars above, vs a sliding window of length 64. - assert.equal(baseContext3.reduce((accum, curr) => accum + curr.length, 0), 74); - // Actual window + one newly-typed character - assert.equal(incomingContext3.reduce((accum, curr) => accum + curr.length, 0), 65); - - assert.deepEqual(computeAlignment(baseContext3, incomingContext3, true), { - canAlign: true, - editPath: [ - {op: 'delete', input: 0}, - {op: 'match', input: 1, match: 0}, - {op: 'match', input: 2, match: 1}, - {op: 'match', input: 3, match: 2}, - {op: 'match', input: 4, match: 3}, - {op: 'match', input: 5, match: 4}, - {op: 'match', input: 6, match: 5}, - {op: 'match', input: 7, match: 6}, - {op: 'match', input: 8, match: 7}, - {op: 'match', input: 9, match: 8}, - {op: 'match', input: 10, match: 9}, - {op: 'match', input: 11, match: 10}, - {op: 'match', input: 12, match: 11}, - {op: 'match', input: 13, match: 12}, - {op: 'match', input: 14, match: 13}, - {op: 'match', input: 15, match: 14}, - {op: 'match', input: 16, match: 15}, - {op: 'match', input: 17, match: 16}, - {op: 'match', input: 18, match: 17}, - {op: 'match', input: 19, match: 18}, - {op: 'match', input: 20, match: 19}, - {op: 'match', input: 21, match: 20}, - {op: 'match', input: 22, match: 21}, - {op: 'match', input: 23, match: 22}, - {op: 'substitute', input: 24, match: 23} - ], - leadTokenShift: -1, - leadEditLength: 0, - matchLength: 23, - tailEditLength: 1, - tailTokenShift: 0 - }); - }); }); \ No newline at end of file