diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/alignment-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/alignment-helpers.ts index 5c72a833f0..62d62f3636 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/alignment-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/alignment-helpers.ts @@ -10,6 +10,47 @@ import { ClassicalDistanceCalculation, EditOperation } from "./classical-calculation.js"; +/** + * Represents token-count values resulting from an alignment attempt between two + * different modeled context states. + */ +export type ContextStateAlignment = { + /** + * Denotes whether or not alignment is possible between two contexts. + */ + canAlign: false +} | { + /** + * Denotes whether or not alignment is possible between two contexts. + */ + canAlign: true, + /** + * Notes the number of tokens added to the head of the 'incoming'/'new' context + * of the contexts being aligned. If negative, the incoming context deleted + * a token found in the 'original' / base context. + * + * For the alignment, [base context index] + leadTokenShift = [incoming context index]. + */ + leadTokenShift: number, + /** + * The count of tokens perfectly aligned, with no need for edits, for two successfully- + * alignable contexts. + */ + matchLength: number, + /** + * The count of tokens at the tail perfectly aligned (existing in both contexts) but + * edited for two successfully-alignable contexts. These tokens directly follow those + * that need no edits. + */ + tailEditLength: number, + /** + * The count of new tokens added at the end of the incoming context for two aligned contexts. + * If negative, the incoming context deleted a previously-existing token from the original. + */ + tailTokenShift: number +}; + + /** * Determines the proper 'last match' index for a tokenized sequence based on its edit path. * @@ -141,4 +182,224 @@ export function isSubstitutionAlignable( } return true; +} + +/** + * Determines the alignment between a new, incoming tokenization source and the + * tokenization modeled by the current instance. + * @param tokenizationToMatch Raw strings corresponding to the tokenization of the original context + * @param incomingTokenization Raw strings corresponding to the tokenization of the incoming context + * @param isSliding Notes if the context window is full (and sliding-alignment is particularly needed) + * @param noSubVerify When true, this disables inspection of 'substitute' transitions that avoids + * wholesale replacement of the original token. + * @returns Alignment data that details if and how the incoming tokenization aligns with + * the tokenization modeled by this instance. + */ +export function computeAlignment( + tokenizationToMatch: string[], + incomingTokenization: string[], + isSliding: boolean, + noSubVerify?: boolean +): ContextStateAlignment { + // Inverted order, since 'match' existed before our new context. + let mapping = ClassicalDistanceCalculation.computeDistance( + tokenizationToMatch.map(value => ({key: value})), + incomingTokenization.map(value => ({key: value})), + // Diagonal width to consider must be at least 2, as adding a single + // whitespace after a token tends to add two tokens: one for whitespace, + // one for the empty token to follow it. + 3 + ); + + let editPath = mapping.editPath(); + // Special case: new context bootstrapping - first token often substitutes. + // The text length is small enough that no words should be able to rotate out the start of the context. + // Special handling needed in case of no 'match'; the rest of the method assumes at least one 'match'. + if(editPath.length <= 3 && (editPath[0] == 'substitute' || editPath[0] == 'match')) { + let matchCount = 0; + let subCount = 0; + for(let i = 0; i < editPath.length; i++) { + if(editPath[i] == 'substitute') { + subCount++; + if(!noSubVerify && !isSubstitutionAlignable(incomingTokenization[i], tokenizationToMatch[i], true)) { + return { + canAlign: false + }; + } + } else if(editPath[i] == 'match') { + // If a substitution is already recorded, treat the 'match' as a substitution. + if(subCount > 0) { + subCount++; + } else { + matchCount++; + } + } + } + + const insertCount = editPath.filter((entry) => entry == 'insert').length; + const deleteCount = editPath.filter((entry) => entry == 'delete').length; + + return { + canAlign: true, + matchLength: matchCount, + leadTokenShift: 0, + tailEditLength: subCount, + tailTokenShift: insertCount - deleteCount + } + } + + // From here on assumes that at least one 'match' exists on the path. + // It all works great... once the context is long enough for at least one stable token. + const firstMatch = editPath.indexOf('match'); + + if(firstMatch == -1) { + // If there are no matches, there's no alignment. + return { + canAlign: false + }; + } + + // Transpositions are not allowed at the token level during context alignment. + if(editPath.find((entry) => entry.indexOf('transpose') > -1)) { + return { + canAlign: false + }; + } + + const lastMatch = getEditPathLastMatch(editPath); + + // Assertion: for a long context, the bulk of the edit path should be a + // continuous block of 'match' entries. If there's anything else in + // the middle, we have a context mismatch. + if(lastMatch == -1) { + return { + canAlign: false + }; + } + + let matchLength = lastMatch - firstMatch + 1; + let tailInsertLength = 0; + let tailDeleteLength = 0; + for(let i = lastMatch; i < editPath.length; i++) { + if(editPath[i] == 'insert') { + tailInsertLength++; + } else if(editPath[i] == 'delete') { + tailDeleteLength++; + } + } + if(tailInsertLength > 0 && tailDeleteLength > 0) { + // Something's gone weird if this happens; that should appear as a substitution instead. + // Otherwise, we have a VERY niche edit scenario. + return { + canAlign: false + }; + } + const tailSubstituteLength = (editPath.length - 1 - lastMatch) - tailInsertLength - tailDeleteLength; + + // If we have a perfect match with a pre-existing context, no mutations have + // happened; we have a 100% perfect match. + if(firstMatch == 0 && lastMatch == editPath.length - 1) { + return { + canAlign: true, + leadTokenShift: 0, + matchLength, + tailEditLength: tailSubstituteLength, + tailTokenShift: tailInsertLength - tailDeleteLength + }; + } + + // The edit path calc tries to put substitutes first, before inserts. + // We don't want that on the leading edge. + const lastEarlyInsert = editPath.lastIndexOf('insert', firstMatch); + const firstSubstitute = editPath.indexOf('substitute'); + if(firstSubstitute > -1 && firstSubstitute < firstMatch && firstSubstitute < lastEarlyInsert) { + editPath[firstSubstitute] = 'insert'; + editPath[lastEarlyInsert] = 'substitute'; + } + + // If mutations HAVE happened, we need to double-check the context-state alignment. + let priorEdit: typeof editPath[0]; + let leadTokensRemoved = 0; + let leadSubstitutions = 0; + + // The `i` index below aligns based upon the index within the `tokenizationToMatch` sequence + // and how it would have to be edited to align to the `incomingTokenization` sequence. + for(let i = 0; i < firstMatch; i++) { + switch(editPath[i]) { + case 'delete': + // All deletions should appear at the sliding window edge; if a deletion appears + // after the edge, but before the first match, something's wrong. + if(priorEdit && priorEdit != 'delete') { + return { + canAlign: false + }; + } + leadTokensRemoved++; + break; + case 'substitute': + // We only allow for one leading token to be substituted. + // + // Any extras in the front would be pure inserts, not substitutions, due to + // the sliding context window and its implications. + if(leadSubstitutions++ > 0) { + return { + canAlign: false + }; + } + + // Find the word before and after substitution. + const incomingIndex = i - (leadTokensRemoved > 0 ? leadTokensRemoved : 0); + const matchingIndex = i + (leadTokensRemoved < 0 ? leadTokensRemoved : 0); + const incomingSub = incomingTokenization[incomingIndex]; + const matchingSub = tokenizationToMatch[matchingIndex]; + + const atSlidePoint = isSliding && (incomingIndex == 0 || matchingIndex == 0); + + // Double-check the word - does the 'substituted' word itself align? + // + // Exception: if the word is at the start of the context window and the + // context window is likely sliding, don't check it. + if(!noSubVerify && !atSlidePoint && !isSubstitutionAlignable(incomingSub, matchingSub)) { + return { + canAlign: false + }; + } + + // There's no major need to drop parts of a token being 'slid' out of the context window. + // We'll leave it intact and treat it as a 'match' + matchLength++; + break; + case 'insert': + // Only allow an insert at the leading edge, as with 'delete's. + if(priorEdit && priorEdit != 'insert') { + return { + canAlign: false + }; + } + // In case of backspaces, it's also possible to 'insert' a 'new' + // token - an old one that's slid back into view. + leadTokensRemoved--; + break; + default: + // No 'match' can exist before the first found index for a 'match'. + // No 'transpose-' edits should exist within this section, either. + return { + canAlign: false + }; + } + priorEdit = editPath[i]; + } + + // If we need some form of tail-token substitution verification, add that here. + + return { + canAlign: true, + // leadTokensRemoved represents the number of tokens that must be removed from the base context + // when aligning the contexts. Externally, it's more helpful to think in terms of the count added + // to the incoming context. + leadTokenShift: -leadTokensRemoved + 0, // add 0 in case of a 'negative zero', which affects unit tests. + matchLength, + tailEditLength: tailSubstituteLength, + tailTokenShift: tailInsertLength - tailDeleteLength + }; } \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index bcfd3b9efe..618dbb795d 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -8,8 +8,7 @@ */ import { ContextToken } from './context-token.js'; -import { ClassicalDistanceCalculation } from './classical-calculation.js'; -import { getEditPathLastMatch, isSubstitutionAlignable } from './alignment-helpers.js'; +import { computeAlignment, ContextStateAlignment } from './alignment-helpers.js'; import { Token } from '@keymanapp/models-templates'; import { LexicalModelTypes } from '@keymanapp/common-types'; @@ -18,46 +17,6 @@ import LexicalModel = LexicalModelTypes.LexicalModel; import Transform = LexicalModelTypes.Transform; import TransformUtils from '../transformUtils.js'; -/** - * Represents token-count values resulting from an alignment attempt between two - * different modeled context states. - */ -export type ContextStateAlignment = { - /** - * Denotes whether or not alignment is possible between two contexts. - */ - canAlign: false -} | { - /** - * Denotes whether or not alignment is possible between two contexts. - */ - canAlign: true, - /** - * Notes the number of tokens added to the head of the 'incoming'/'new' context - * of the contexts being aligned. If negative, the incoming context deleted - * a token found in the 'original' / base context. - * - * For the alignment, [base context index] + leadTokenShift = [incoming context index]. - */ - leadTokenShift: number, - /** - * The count of tokens perfectly aligned, with no need for edits, for two successfully- - * alignable contexts. - */ - matchLength: number, - /** - * The count of tokens at the tail perfectly aligned (existing in both contexts) but - * edited for two successfully-alignable contexts. These tokens directly follow those - * that need no edits. - */ - tailEditLength: number, - /** - * The count of new tokens added at the end of the incoming context for two aligned contexts. - * If negative, the incoming context deleted a previously-existing token from the original. - */ - tailTokenShift: number -}; - /** * This class represents the sequence of tokens (words and whitespace blocks) * held within the active sliding context-window at a single point in time. @@ -109,210 +68,7 @@ export class ContextTokenization { * the tokenization modeled by this instance. */ computeAlignment(incomingTokenization: string[], isSliding: boolean, noSubVerify?: boolean): ContextStateAlignment { - // Map the tokenized state to an edit-distance friendly version. - const tokenizationToMatch = this.exampleInput; - - // Inverted order, since 'match' existed before our new context. - let mapping = ClassicalDistanceCalculation.computeDistance( - tokenizationToMatch.map(value => ({key: value})), - incomingTokenization.map(value => ({key: value})), - // Diagonal width to consider must be at least 2, as adding a single - // whitespace after a token tends to add two tokens: one for whitespace, - // one for the empty token to follow it. - 3 - ); - - let editPath = mapping.editPath(); - // Special case: new context bootstrapping - first token often substitutes. - // The text length is small enough that no words should be able to rotate out the start of the context. - // Special handling needed in case of no 'match'; the rest of the method assumes at least one 'match'. - if(editPath.length <= 3 && (editPath[0] == 'substitute' || editPath[0] == 'match')) { - let matchCount = 0; - let subCount = 0; - for(let i = 0; i < editPath.length; i++) { - if(editPath[i] == 'substitute') { - subCount++; - if(!noSubVerify && !isSubstitutionAlignable(incomingTokenization[i], tokenizationToMatch[i], true)) { - return { - canAlign: false - }; - } - } else if(editPath[i] == 'match') { - // If a substitution is already recorded, treat the 'match' as a substitution. - if(subCount > 0) { - subCount++; - } else { - matchCount++; - } - } - } - - const insertCount = editPath.filter((entry) => entry == 'insert').length; - const deleteCount = editPath.filter((entry) => entry == 'delete').length; - - return { - canAlign: true, - matchLength: matchCount, - leadTokenShift: 0, - tailEditLength: subCount, - tailTokenShift: insertCount - deleteCount - } - } - - // From here on assumes that at least one 'match' exists on the path. - // It all works great... once the context is long enough for at least one stable token. - const firstMatch = editPath.indexOf('match'); - - if(firstMatch == -1) { - // If there are no matches, there's no alignment. - return { - canAlign: false - }; - } - - // Transpositions are not allowed at the token level during context alignment. - if(editPath.find((entry) => entry.indexOf('transpose') > -1)) { - return { - canAlign: false - }; - } - - const lastMatch = getEditPathLastMatch(editPath); - - // Assertion: for a long context, the bulk of the edit path should be a - // continuous block of 'match' entries. If there's anything else in - // the middle, we have a context mismatch. - if(lastMatch == -1) { - return { - canAlign: false - }; - } - - let matchLength = lastMatch - firstMatch + 1; - let tailInsertLength = 0; - let tailDeleteLength = 0; - for(let i = lastMatch; i < editPath.length; i++) { - if(editPath[i] == 'insert') { - tailInsertLength++; - } else if(editPath[i] == 'delete') { - tailDeleteLength++; - } - } - if(tailInsertLength > 0 && tailDeleteLength > 0) { - // Something's gone weird if this happens; that should appear as a substitution instead. - // Otherwise, we have a VERY niche edit scenario. - return { - canAlign: false - }; - } - const tailSubstituteLength = (editPath.length - 1 - lastMatch) - tailInsertLength - tailDeleteLength; - - // If we have a perfect match with a pre-existing context, no mutations have - // happened; we have a 100% perfect match. - if(firstMatch == 0 && lastMatch == editPath.length - 1) { - return { - canAlign: true, - leadTokenShift: 0, - matchLength, - tailEditLength: tailSubstituteLength, - tailTokenShift: tailInsertLength - tailDeleteLength - }; - } - - // The edit path calc tries to put substitutes first, before inserts. - // We don't want that on the leading edge. - const lastEarlyInsert = editPath.lastIndexOf('insert', firstMatch); - const firstSubstitute = editPath.indexOf('substitute'); - if(firstSubstitute > -1 && firstSubstitute < firstMatch && firstSubstitute < lastEarlyInsert) { - editPath[firstSubstitute] = 'insert'; - editPath[lastEarlyInsert] = 'substitute'; - } - - // If mutations HAVE happened, we need to double-check the context-state alignment. - let priorEdit: typeof editPath[0]; - let leadTokensRemoved = 0; - let leadSubstitutions = 0; - - // The `i` index below aligns based upon the index within the `tokenizationToMatch` sequence - // and how it would have to be edited to align to the `incomingTokenization` sequence. - for(let i = 0; i < firstMatch; i++) { - switch(editPath[i]) { - case 'delete': - // All deletions should appear at the sliding window edge; if a deletion appears - // after the edge, but before the first match, something's wrong. - if(priorEdit && priorEdit != 'delete') { - return { - canAlign: false - }; - } - leadTokensRemoved++; - break; - case 'substitute': - // We only allow for one leading token to be substituted. - // - // Any extras in the front would be pure inserts, not substitutions, due to - // the sliding context window and its implications. - if(leadSubstitutions++ > 0) { - return { - canAlign: false - }; - } - - // Find the word before and after substitution. - const incomingIndex = i - (leadTokensRemoved > 0 ? leadTokensRemoved : 0); - const matchingIndex = i + (leadTokensRemoved < 0 ? leadTokensRemoved : 0); - const incomingSub = incomingTokenization[incomingIndex]; - const matchingSub = tokenizationToMatch[matchingIndex]; - - const atSlidePoint = isSliding && (incomingIndex == 0 || matchingIndex == 0); - - // Double-check the word - does the 'substituted' word itself align? - // - // Exception: if the word is at the start of the context window and the - // context window is likely sliding, don't check it. - if(!noSubVerify && !atSlidePoint && !isSubstitutionAlignable(incomingSub, matchingSub)) { - return { - canAlign: false - }; - } - - // There's no major need to drop parts of a token being 'slid' out of the context window. - // We'll leave it intact and treat it as a 'match' - matchLength++; - break; - case 'insert': - // Only allow an insert at the leading edge, as with 'delete's. - if(priorEdit && priorEdit != 'insert') { - return { - canAlign: false - }; - } - // In case of backspaces, it's also possible to 'insert' a 'new' - // token - an old one that's slid back into view. - leadTokensRemoved--; - break; - default: - // No 'match' can exist before the first found index for a 'match'. - // No 'transpose-' edits should exist within this section, either. - return { - canAlign: false - }; - } - priorEdit = editPath[i]; - } - - // If we need some form of tail-token substitution verification, add that here. - - return { - canAlign: true, - // leadTokensRemoved represents the number of tokens that must be removed from the base context - // when aligning the contexts. Externally, it's more helpful to think in terms of the count added - // to the incoming context. - leadTokenShift: -leadTokensRemoved + 0, // add 0 in case of a 'negative zero', which affects unit tests. - matchLength, - tailEditLength: tailSubstituteLength, - tailTokenShift: tailInsertLength - tailDeleteLength - }; + return computeAlignment(this.exampleInput, incomingTokenization, isSliding, noSubVerify); } /** diff --git a/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts b/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts index b65d566197..3a4dd6737a 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/test-index.ts @@ -10,7 +10,7 @@ export * as correction from './correction/index.js'; export * from './model-helpers.js'; export * as models from './models/index.js'; export { ModelCompositor } from './model-compositor.js'; -export { tokenizeTransform, tokenizeTransformDistribution } from './correction/transform-tokenization.js'; +export * from './correction/transform-tokenization.js'; export * from './predict-helpers.js'; export { default as TransformUtils } from './transformUtils.js' export { default as LMLayerWorker } from './index.js' \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/alignment-helpers.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/alignment-helpers.tests.ts index 90cff2ccda..16dd8327bc 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/alignment-helpers.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/alignment-helpers.tests.ts @@ -9,7 +9,7 @@ */ import { assert } from 'chai'; -import { EditOperation, getEditPathLastMatch, isSubstitutionAlignable } from '@keymanapp/lm-worker/test-index'; +import { computeAlignment, EditOperation, getEditPathLastMatch, isSubstitutionAlignable } from '@keymanapp/lm-worker/test-index'; describe('getEditPathLastMatch', () => { it('returns the last match when no substitutions exist', () => { @@ -116,4 +116,454 @@ describe('isSubstitutionAlignable', () => { // The double-p adds a fun complication once the first gets dropped. assert.isTrue(isSubstitutionAlignable('applesauce', 'plesauce')); }); +}); + + +describe('computeAlignment', () => { + it("properly matches and aligns when contexts match", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + const newContext = [...baseContext]; + + const computedAlignment = computeAlignment(baseContext, newContext, false); + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: 0, + matchLength: 5, + tailEditLength: 0, + tailTokenShift: 0 + }); + }); + + it("properly matches and aligns with applied-suggestion contexts", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'o' + ]; + const newContext = [...baseContext]; + newContext[4] = 'over'; + + const computedAlignment = computeAlignment(baseContext, newContext, false); + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: 0, + matchLength: 4, + tailEditLength: 1, + tailTokenShift: 0 + }); + }); + + it("properly matches and aligns with applied-suggestion at start of context", () => { + const baseContext = [ + 'te' + ]; + const newContext = [ + 'testing', + ' ', + '' + ]; + + const computedAlignment = computeAlignment(baseContext, newContext, false, true); + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: 0, + matchLength: 0, + tailEditLength: 1, + tailTokenShift: 2 + }); + }); + + it("detects unalignable contexts - no matching tokens", () => { + const baseContext = [ + 'swift', 'tan', 'wolf', 'leaped', 'across' + ]; + const newContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + + const computedAlignment = computeAlignment(baseContext, newContext, false); + assert.deepEqual(computedAlignment, {canAlign: false}); + }); + + it("detects unalignable contexts - too many mismatching tokens", () => { + const baseContext = [ + 'swift', 'tan', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + + const computedAlignment = computeAlignment(baseContext, newContext, false); + assert.deepEqual(computedAlignment, {canAlign: false}); + }); + + it("fails alignment for leading-edge word substitutions", () => { + const baseContext = [ + 'swift', 'brown', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + + const computedAlignment = computeAlignment(baseContext, newContext, false); + assert.deepEqual(computedAlignment, {canAlign: false}); + }); + + it("fails alignment for small leading-edge word substitutions", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'sick', 'brown', 'fox', 'jumped', 'over' + ]; + + const computedAlignment = computeAlignment(baseContext, newContext, false); + assert.deepEqual(computedAlignment, {canAlign: false}); + }); + + it("properly matches and aligns when lead token is modified", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'uick', 'brown', 'fox', 'jumped', 'over' + ]; + + const computedAlignment = computeAlignment(baseContext, newContext, false); + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: 0, + matchLength: 5, + tailEditLength: 0, + tailTokenShift: 0 + }); + }); + + it("properly matches and aligns when lead token is removed", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'brown', 'fox', 'jumped', 'over' + ]; + + const computedAlignment = computeAlignment(baseContext, newContext, false); + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: -1, + matchLength: 4, + tailEditLength: 0, + tailTokenShift: 0 + }); + }); + + it("properly matches and aligns when lead token is added", () => { + const baseContext = [ + 'brown', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + + const computedAlignment = computeAlignment(baseContext, newContext, false); + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: 1, + matchLength: 4, + tailEditLength: 0, + tailTokenShift: 0 + }); + }); + + it("properly matches and aligns when lead tokens are removed and modified", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'ox', 'jumped', 'over' + ]; + + const computedAlignment = computeAlignment(baseContext, newContext, false); + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: -2, + matchLength: 3, + tailEditLength: 0, + tailTokenShift: 0 + }); + }); + + it("properly matches and aligns when lead tokens are added and modified", () => { + const baseContext = [ + 'rown', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + + const computedAlignment = computeAlignment(baseContext, newContext, false); + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: 1, + matchLength: 4, + tailEditLength: 0, + tailTokenShift: 0 + }); + }); + + it("properly matches and aligns when lead token is removed and tail token is added", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'brown', 'fox', 'jumped', 'over', 'the' + ]; + + const computedAlignment = computeAlignment(baseContext, newContext, false); + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: -1, + matchLength: 4, + tailEditLength: 0, + tailTokenShift: 1 + }); + }); + + it("properly matches and aligns when lead token and tail token are modified", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'ove' + ]; + const newContext = [ + 'uick', 'brown', 'fox', 'jumped', 'over' + ]; + + const computedAlignment = computeAlignment(baseContext, newContext, false); + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: 0, + matchLength: 4, // we treat 'quick' and 'uick' as the same + tailEditLength: 1, + tailTokenShift: 0 + }); + }); + + it("properly matches and aligns when lead token and tail token are modified + new token appended", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'ove' + ]; + const newContext = [ + 'uick', 'brown', 'fox', 'jumped', 'over', 't' + ]; + + const computedAlignment = computeAlignment(baseContext, newContext, false); + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: 0, + matchLength: 4, // we treat 'quick' and 'uick' as the same + tailEditLength: 1, + tailTokenShift: 1 + }); + }); + + it("properly handles context window sliding backward", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'e', 'quick', 'brown', 'fox', 'jumped', 'ove' + ]; + + const computedAlignment = computeAlignment(baseContext, newContext, false); + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: 1, + matchLength: 4, // we treat 'quick' and 'uick' as the same + tailEditLength: 1, + tailTokenShift: 0 + }); + }); + + it("properly handles context window sliding far backward", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'the', 'quick', 'brown', 'fox', 'jumped' + ]; + + const computedAlignment = computeAlignment(baseContext, newContext, false); + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: 1, + matchLength: 4, // we treat 'quick' and 'uick' as the same + tailEditLength: 0, + tailTokenShift: -1 + }); + }); + + it("properly handles context window sliding farther backward", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'the', 'quick', 'brown', 'fox', 'jumpe' + ]; + + const computedAlignment = computeAlignment(baseContext, newContext, false); + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: 1, + matchLength: 3, // we treat 'quick' and 'uick' as the same + tailEditLength: 1, + tailTokenShift: -1 + }); + }); + + it("fails alignment for mid-head deletion", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'quick', 'fox', 'jumped', 'over' + ]; + + const computedAlignment = computeAlignment(baseContext, newContext, false); + assert.deepEqual(computedAlignment, {canAlign: false}); + }); + + it("fails alignment for mid-head insertion", () => { + const baseContext = [ + 'quick', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + + const computedAlignment = computeAlignment(baseContext, newContext, false); + assert.deepEqual(computedAlignment, {canAlign: false}); + }); + + it("fails alignment for mid-tail deletion", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'quick', 'brown', 'fox', 'over' + ]; + + const computedAlignment = computeAlignment(baseContext, newContext, false); + assert.deepEqual(computedAlignment, {canAlign: false}); + }); + + it("fails alignment for mid-tail insertion", () => { + const baseContext = [ + 'quick', 'brown', 'fox', 'jumped', 'over' + ]; + const newContext = [ + 'quick', 'brown', 'fox', 'jumped', 'far', 'over' + ]; + + const computedAlignment = computeAlignment(baseContext, newContext, false); + assert.deepEqual(computedAlignment, {canAlign: false}); + }); + + it("handles late-context suggestion application after backspace", () => { + const baseContext = [ + 'quick', ' ', 'brown', ' ', 'fox', ' ', 'jumped', ' ', 'oven', ' ', '' + ]; + const newContext = [ + 'quick', ' ', 'brown', ' ', 'fox', ' ', 'jumped', ' ', 'over', ' ', '' + ]; + + const computedAlignment = computeAlignment(baseContext, newContext, false); + assert.deepEqual(computedAlignment, { + canAlign: true, + leadTokenShift: 0, + matchLength: 8, + tailEditLength: 3, + tailTokenShift: 0 + }); + }); + + it("handles sliding context-window scenarios", () => { + // // Explicitly-defined window, though it's not needed directly by the method. + // const config = { + // leftContextCodePoints: 64, + // rightContextCodePoints: 64 + // }; + + const baseContext1 = [ + // "ap" prefix not in actual view, but preserved by prior tokenization rounds. + "applesauce", " ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ", + "like", " ", "they'd", " ", "make", " ", "for", " ", "the", " ", "be" + ]; + + const incomingContext1 = [ + "plesauce", " ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ", + "like", " ", "they'd", " ", "make", " ", "for", " ", "the", " ", "bes" + ]; + + // 66 chars above, vs a sliding window of length 64. + assert.equal(baseContext1.reduce((accum, curr) => accum + curr.length, 0), 66); + // Actual window + one newly-typed character + assert.equal(incomingContext1.reduce((accum, curr) => accum + curr.length, 0), 65); + + assert.deepEqual(computeAlignment(baseContext1, incomingContext1, true), { + canAlign: true, + leadTokenShift: 0, + matchLength: 22, + tailEditLength: 1, + tailTokenShift: 0 + }); + + // Our tokenization scheme remembers the full original word before any of it slid out of + // the context window. + const baseContext2 = [ + "applesauce", " ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ", + // +2 +1 +4 + "like", " ", "they'd", " ", "make", " ", "for", " ", "the", " ", "best", " ", "brea" + ]; + + const incomingContext2 = [ + // "plesauce" => "e": -7 chars. + "e", " ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ", + "like", " ", "they'd", " ", "make", " ", "for", " ", "the", " ", "best", " ", "break" + ]; + + // 73 chars above, vs a sliding window of length 64. + assert.equal(baseContext2.reduce((accum, curr) => accum + curr.length, 0), 73); + // Actual window + one newly-typed character + assert.equal(incomingContext2.reduce((accum, curr) => accum + curr.length, 0), 65); + + assert.deepEqual(computeAlignment(baseContext2, incomingContext2, true), { + canAlign: true, + leadTokenShift: 0, + matchLength: 24, + tailEditLength: 1, + tailTokenShift: 0 + }); + + const baseContext3 = [ + "applesauce", " ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ", "like", " ", + "they'd", " ", "make", " ", "for", " ", "the", " ", "best", " ", "break" + ]; + + const incomingContext3 = [ + " ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ", "like", " ", + "they'd", " ", "make", " ", "for", " ", "the", " ", "best", " ", "breakf" + ]; + + // 74 chars above, vs a sliding window of length 64. + assert.equal(baseContext3.reduce((accum, curr) => accum + curr.length, 0), 74); + // Actual window + one newly-typed character + assert.equal(incomingContext3.reduce((accum, curr) => accum + curr.length, 0), 65); + + assert.deepEqual(computeAlignment(baseContext3, incomingContext3, true), { + canAlign: true, + leadTokenShift: -1, + matchLength: 23, + tailEditLength: 1, + tailTokenShift: 0 + }); + }); }); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index d8287decee..173dd10b1d 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -20,11 +20,6 @@ import TrieModel = models.TrieModel; var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'), {wordBreaker: defaultBreaker}); -function buildBaseTokenization(textTokens: string[]) { - const tokens = textTokens.map((entry) => toToken(entry)); - return new ContextTokenization(tokens); -} - function toToken(text: string) { let isWhitespace = text == ' '; let token = new ContextToken(plainModel, text); @@ -100,505 +95,6 @@ describe('ContextTokenization', function() { assert.deepEqual(tokenization.exampleInput, rawTextTokens); }); - describe('computeAlignment', () => { - it("properly matches and aligns when contexts match", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - const newContext = [...baseContext]; - - const baseTokenization = buildBaseTokenization(baseContext); - const computedAlignment = baseTokenization.computeAlignment(newContext, false); - - assert.deepEqual(computedAlignment, { - canAlign: true, - leadTokenShift: 0, - matchLength: 5, - tailEditLength: 0, - tailTokenShift: 0 - }); - }); - - it("properly matches and aligns with applied-suggestion contexts", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'o' - ]; - const newContext = [...baseContext]; - newContext[4] = 'over'; - - const baseTokenization = buildBaseTokenization(baseContext); - const computedAlignment = baseTokenization.computeAlignment(newContext, false); - - assert.deepEqual(computedAlignment, { - canAlign: true, - leadTokenShift: 0, - matchLength: 4, - tailEditLength: 1, - tailTokenShift: 0 - }); - }); - - it("properly matches and aligns with applied-suggestion at start of context", () => { - const baseContext = [ - 'te' - ]; - const newContext = [ - 'testing', - ' ', - '' - ]; - - const baseTokenization = buildBaseTokenization(baseContext); - const computedAlignment = baseTokenization.computeAlignment(newContext, false, true); - - assert.deepEqual(computedAlignment, { - canAlign: true, - leadTokenShift: 0, - matchLength: 0, - tailEditLength: 1, - tailTokenShift: 2 - }); - }); - - it("detects unalignable contexts - no matching tokens", () => { - const baseContext = [ - 'swift', 'tan', 'wolf', 'leaped', 'across' - ]; - const newContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - - const baseTokenization = buildBaseTokenization(baseContext); - const computedAlignment = baseTokenization.computeAlignment(newContext, false); - - assert.deepEqual(computedAlignment, {canAlign: false}); - }); - - it("detects unalignable contexts - too many mismatching tokens", () => { - const baseContext = [ - 'swift', 'tan', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - - const baseTokenization = buildBaseTokenization(baseContext); - const computedAlignment = baseTokenization.computeAlignment(newContext, false); - - assert.deepEqual(computedAlignment, {canAlign: false}); - }); - - it("fails alignment for leading-edge word substitutions", () => { - const baseContext = [ - 'swift', 'brown', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - - const baseTokenization = buildBaseTokenization(baseContext); - const computedAlignment = baseTokenization.computeAlignment(newContext, false); - - assert.deepEqual(computedAlignment, {canAlign: false}); - }); - - it("fails alignment for small leading-edge word substitutions", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'sick', 'brown', 'fox', 'jumped', 'over' - ]; - - const baseTokenization = buildBaseTokenization(baseContext); - const computedAlignment = baseTokenization.computeAlignment(newContext, false); - - assert.deepEqual(computedAlignment, {canAlign: false}); - }); - - it("properly matches and aligns when lead token is modified", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'uick', 'brown', 'fox', 'jumped', 'over' - ]; - - const baseTokenization = buildBaseTokenization(baseContext); - const computedAlignment = baseTokenization.computeAlignment(newContext, false); - - assert.deepEqual(computedAlignment, { - canAlign: true, - leadTokenShift: 0, - matchLength: 5, - tailEditLength: 0, - tailTokenShift: 0 - }); - }); - - it("properly matches and aligns when lead token is removed", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'brown', 'fox', 'jumped', 'over' - ]; - - const baseTokenization = buildBaseTokenization(baseContext); - const computedAlignment = baseTokenization.computeAlignment(newContext, false); - - assert.deepEqual(computedAlignment, { - canAlign: true, - leadTokenShift: -1, - matchLength: 4, - tailEditLength: 0, - tailTokenShift: 0 - }); - }); - - it("properly matches and aligns when lead token is added", () => { - const baseContext = [ - 'brown', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - - const baseTokenization = buildBaseTokenization(baseContext); - const computedAlignment = baseTokenization.computeAlignment(newContext, false); - - assert.deepEqual(computedAlignment, { - canAlign: true, - leadTokenShift: 1, - matchLength: 4, - tailEditLength: 0, - tailTokenShift: 0 - }); - }); - - it("properly matches and aligns when lead tokens are removed and modified", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'ox', 'jumped', 'over' - ]; - - const baseTokenization = buildBaseTokenization(baseContext); - const computedAlignment = baseTokenization.computeAlignment(newContext, false); - - assert.deepEqual(computedAlignment, { - canAlign: true, - leadTokenShift: -2, - matchLength: 3, - tailEditLength: 0, - tailTokenShift: 0 - }); - }); - - it("properly matches and aligns when lead tokens are added and modified", () => { - const baseContext = [ - 'rown', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - - const baseTokenization = buildBaseTokenization(baseContext); - const computedAlignment = baseTokenization.computeAlignment(newContext, false); - - assert.deepEqual(computedAlignment, { - canAlign: true, - leadTokenShift: 1, - matchLength: 4, - tailEditLength: 0, - tailTokenShift: 0 - }); - }); - - it("properly matches and aligns when lead token is removed and tail token is added", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'brown', 'fox', 'jumped', 'over', 'the' - ]; - - const baseTokenization = buildBaseTokenization(baseContext); - const computedAlignment = baseTokenization.computeAlignment(newContext, false); - - assert.deepEqual(computedAlignment, { - canAlign: true, - leadTokenShift: -1, - matchLength: 4, - tailEditLength: 0, - tailTokenShift: 1 - }); - }); - - it("properly matches and aligns when lead token and tail token are modified", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'ove' - ]; - const newContext = [ - 'uick', 'brown', 'fox', 'jumped', 'over' - ]; - - const baseTokenization = buildBaseTokenization(baseContext); - const computedAlignment = baseTokenization.computeAlignment(newContext, false); - - assert.deepEqual(computedAlignment, { - canAlign: true, - leadTokenShift: 0, - matchLength: 4, // we treat 'quick' and 'uick' as the same - tailEditLength: 1, - tailTokenShift: 0 - }); - }); - - it("properly matches and aligns when lead token and tail token are modified + new token appended", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'ove' - ]; - const newContext = [ - 'uick', 'brown', 'fox', 'jumped', 'over', 't' - ]; - - const baseTokenization = buildBaseTokenization(baseContext); - const computedAlignment = baseTokenization.computeAlignment(newContext, false); - - assert.deepEqual(computedAlignment, { - canAlign: true, - leadTokenShift: 0, - matchLength: 4, // we treat 'quick' and 'uick' as the same - tailEditLength: 1, - tailTokenShift: 1 - }); - }); - - it("properly handles context window sliding backward", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'e', 'quick', 'brown', 'fox', 'jumped', 'ove' - ]; - - const baseTokenization = buildBaseTokenization(baseContext); - const computedAlignment = baseTokenization.computeAlignment(newContext, false); - - assert.deepEqual(computedAlignment, { - canAlign: true, - leadTokenShift: 1, - matchLength: 4, // we treat 'quick' and 'uick' as the same - tailEditLength: 1, - tailTokenShift: 0 - }); - }); - - it("properly handles context window sliding far backward", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'the', 'quick', 'brown', 'fox', 'jumped' - ]; - - const baseTokenization = buildBaseTokenization(baseContext); - const computedAlignment = baseTokenization.computeAlignment(newContext, false); - - assert.deepEqual(computedAlignment, { - canAlign: true, - leadTokenShift: 1, - matchLength: 4, // we treat 'quick' and 'uick' as the same - tailEditLength: 0, - tailTokenShift: -1 - }); - }); - - it("properly handles context window sliding farther backward", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'the', 'quick', 'brown', 'fox', 'jumpe' - ]; - - const baseTokenization = buildBaseTokenization(baseContext); - const computedAlignment = baseTokenization.computeAlignment(newContext, false); - - assert.deepEqual(computedAlignment, { - canAlign: true, - leadTokenShift: 1, - matchLength: 3, // we treat 'quick' and 'uick' as the same - tailEditLength: 1, - tailTokenShift: -1 - }); - }); - - it("fails alignment for mid-head deletion", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'quick', 'fox', 'jumped', 'over' - ]; - - const baseTokenization = buildBaseTokenization(baseContext); - const computedAlignment = baseTokenization.computeAlignment(newContext, false); - - assert.deepEqual(computedAlignment, {canAlign: false}); - }); - - it("fails alignment for mid-head insertion", () => { - const baseContext = [ - 'quick', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - - const baseTokenization = buildBaseTokenization(baseContext); - const computedAlignment = baseTokenization.computeAlignment(newContext, false); - - assert.deepEqual(computedAlignment, {canAlign: false}); - }); - - it("fails alignment for mid-tail deletion", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'quick', 'brown', 'fox', 'over' - ]; - - const baseTokenization = buildBaseTokenization(baseContext); - const computedAlignment = baseTokenization.computeAlignment(newContext, false); - - assert.deepEqual(computedAlignment, {canAlign: false}); - }); - - it("fails alignment for mid-tail insertion", () => { - const baseContext = [ - 'quick', 'brown', 'fox', 'jumped', 'over' - ]; - const newContext = [ - 'quick', 'brown', 'fox', 'jumped', 'far', 'over' - ]; - - const baseTokenization = buildBaseTokenization(baseContext); - const computedAlignment = baseTokenization.computeAlignment(newContext, false); - - assert.deepEqual(computedAlignment, {canAlign: false}); - }); - - it("handles late-context suggestion application after backspace", () => { - const baseContext = [ - 'quick', ' ', 'brown', ' ', 'fox', ' ', 'jumped', ' ', 'oven', ' ', '' - ]; - const newContext = [ - 'quick', ' ', 'brown', ' ', 'fox', ' ', 'jumped', ' ', 'over', ' ', '' - ]; - - const baseTokenization = buildBaseTokenization(baseContext); - const computedAlignment = baseTokenization.computeAlignment(newContext, false); - - assert.deepEqual(computedAlignment, { - canAlign: true, - leadTokenShift: 0, - matchLength: 8, - tailEditLength: 3, - tailTokenShift: 0 - }); - }); - - it("handles sliding context-window scenarios", () => { - // // Explicitly-defined window, though it's not needed directly by the method. - // const config = { - // leftContextCodePoints: 64, - // rightContextCodePoints: 64 - // }; - - const baseContext1 = [ - // "ap" prefix not in actual view, but preserved by prior tokenization rounds. - "applesauce", " ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ", - "like", " ", "they'd", " ", "make", " ", "for", " ", "the", " ", "be" - ]; - - const incomingContext1 = [ - "plesauce", " ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ", - "like", " ", "they'd", " ", "make", " ", "for", " ", "the", " ", "bes" - ]; - - // 66 chars above, vs a sliding window of length 64. - assert.equal(baseContext1.reduce((accum, curr) => accum + curr.length, 0), 66); - // Actual window + one newly-typed character - assert.equal(incomingContext1.reduce((accum, curr) => accum + curr.length, 0), 65); - - const tokenization1 = new ContextTokenization(buildBaseTokenization(baseContext1)); - - assert.deepEqual(tokenization1.computeAlignment(incomingContext1, true), { - canAlign: true, - leadTokenShift: 0, - matchLength: 22, - tailEditLength: 1, - tailTokenShift: 0 - }); - - // Our tokenization scheme remembers the full original word before any of it slid out of - // the context window. - const baseContext2 = [ - "applesauce", " ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ", - // +2 +1 +4 - "like", " ", "they'd", " ", "make", " ", "for", " ", "the", " ", "best", " ", "brea" - ]; - - const incomingContext2 = [ - // "plesauce" => "e": -7 chars. - "e", " ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ", - "like", " ", "they'd", " ", "make", " ", "for", " ", "the", " ", "best", " ", "break" - ]; - - // 73 chars above, vs a sliding window of length 64. - assert.equal(baseContext2.reduce((accum, curr) => accum + curr.length, 0), 73); - // Actual window + one newly-typed character - assert.equal(incomingContext2.reduce((accum, curr) => accum + curr.length, 0), 65); - - const tokenization2 = new ContextTokenization(buildBaseTokenization(baseContext2)); - assert.deepEqual(tokenization2.computeAlignment(incomingContext2, true), { - canAlign: true, - leadTokenShift: 0, - matchLength: 24, - tailEditLength: 1, - tailTokenShift: 0 - }); - - const baseContext3 = [ - "applesauce", " ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ", "like", " ", - "they'd", " ", "make", " ", "for", " ", "the", " ", "best", " ", "break" - ]; - - const incomingContext3 = [ - " ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ", "like", " ", - "they'd", " ", "make", " ", "for", " ", "the", " ", "best", " ", "breakf" - ]; - - // 74 chars above, vs a sliding window of length 64. - assert.equal(baseContext3.reduce((accum, curr) => accum + curr.length, 0), 74); - // Actual window + one newly-typed character - assert.equal(incomingContext3.reduce((accum, curr) => accum + curr.length, 0), 65); - - const tokenization3 = new ContextTokenization(buildBaseTokenization(baseContext3)); - assert.deepEqual(tokenization3.computeAlignment(incomingContext3, true), { - canAlign: true, - leadTokenShift: -1, - matchLength: 23, - tailEditLength: 1, - tailTokenShift: 0 - }); - }); - }); - describe('transitionTo', function() { it('simple case - new whitespace + new empty token', () => { const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];