diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts index bc98787ed7..457c1349b2 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts @@ -224,6 +224,8 @@ export class ContextState { // Used to construct and represent the part of the incoming transform that // does not land as part of the final token in the resulting context. This // component should be preserved by any suggestions that get applied. + // + // undefined by default, which asserts we're still affecting the same token. let preservationTransform: Transform; // Handling for non-whitespace word boundaries - for example, @@ -235,10 +237,15 @@ export class ContextState { preservationTransform = { insert: '', deleteLeft: 0 }; } - // Leave out the final entry! - for(let i = 0; i < transformSequenceDistribution?.[0].sample.length - 1; i++) { - const primaryInput = transformSequenceDistribution[0].sample[i]; - preservationTransform = preservationTransform ? buildMergedTransform(preservationTransform, primaryInput): primaryInput; + if(transformSequenceDistribution) { + const transformKeys = [...transformSequenceDistribution[0].sample.keys()]; + // Leave out the final entry - that part is replaceable by suggestions. + transformKeys.pop(); + + for(let i of transformKeys) { + const primaryInput = transformSequenceDistribution[0].sample.get(i); + preservationTransform = preservationTransform ? buildMergedTransform(preservationTransform, primaryInput): primaryInput; + } } const state = new ContextState(postContext, lexicalModel); diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 618dbb795d..d847f1bde1 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -14,6 +14,7 @@ import { Token } from '@keymanapp/models-templates'; import { LexicalModelTypes } from '@keymanapp/common-types'; import Distribution = LexicalModelTypes.Distribution; import LexicalModel = LexicalModelTypes.LexicalModel; +import ProbabilityMass = LexicalModelTypes.ProbabilityMass; import Transform = LexicalModelTypes.Transform; import TransformUtils from '../transformUtils.js'; @@ -89,7 +90,7 @@ export class ContextTokenization { lexicalModel: LexicalModel, // FUTURE NOTE: especially for epic-dict-breaker, we'll want an array of these - to align across multiple transitions // in case word boundaries shift back and forth. - alignedTransformDistribution: Distribution + alignedTransformDistribution: Distribution> ): ContextTokenization { if(!alignment.canAlign) { return null; @@ -156,17 +157,26 @@ export class ContextTokenization { // first index: original sample's tokenization // second index: token index within original sample const tokenDistribution = alignedTransformDistribution.map((entry) => { - return entry.sample.map((sample) => { - return { - sample: sample, + const remap: Map> = new Map(); + + for(let pair of entry.sample.entries()) { + remap.set(pair[0], { + sample: pair[1], p: entry.p - } - }); + }) + } + + return remap; }); - // Using these as base indices... - let tailIndex = 0; + // The original tail token should match index 0. If tokens have been deleted, that + // left-shifts our base indices; we start left of 0. If more than one token was + // edited, those edits occur to the left as well - and further left of whatever + // the new tail token is *if* tokens were removed. + const firstTailEditIndex = Math.min((1 - tailEditLength), 0) + Math.min(tailTokenShift, 0); for(let i = 0; i < tailEditLength; i++) { + const tailIndex = firstTailEditIndex + i; + // do tail edits const incomingIndex = i + incomingTailUpdateIndex; const matchingIndex = i + matchingTailUpdateIndex; @@ -174,7 +184,7 @@ export class ContextTokenization { const incomingToken = tokenizedContext[incomingIndex]; const matchedToken = this.tokens[matchingIndex]; - let primaryInput = hasDistribution ? tokenizedPrimaryInput[i] : null; + let primaryInput = hasDistribution ? tokenizedPrimaryInput.get(tailIndex) : null; const isBackspace = primaryInput && TransformUtils.isBackspace(primaryInput); let token: ContextToken; @@ -187,11 +197,10 @@ export class ContextTokenization { token = new ContextToken(matchedToken); // Erase any applied-suggestion transition ID; it is no longer valid. token.appliedTransitionId = undefined; - token.searchSpace.addInput(tokenDistribution.map((seq) => seq[tailIndex] ?? { sample: { insert: '', deleteLeft: 0 }, p: 1 })); + token.searchSpace.addInput(tokenDistribution.map((seq) => seq.get(tailIndex) ?? { sample: { insert: '', deleteLeft: 0 }, p: 1 })); } tokenization[incomingIndex] = token; - tailIndex++; } if(tailTokenShift < 0) { @@ -205,14 +214,16 @@ export class ContextTokenization { tokenization.pop(); } } else { - for(let i = tailEditLength; i < tailEditLength + tailTokenShift; i++) { + // First tail insertion index within the tokenized-transform map is always 1. + for(let i = 1; i <= tailTokenShift; i++) { // create tail tokens - const incomingIndex = i + incomingTailUpdateIndex; + // same original base after all edited + const incomingIndex = incomingTailUpdateIndex + tailEditLength + (i - 1); const incomingToken = tokenizedContext[incomingIndex]; // // Assertion: there should be no matching token; this should be a newly-appended token. // const matchingIndex = i + tailEditLength + matchingTailUpdateIndex; - const primaryInput = hasDistribution ? tokenizedPrimaryInput[i] : null; + const primaryInput = hasDistribution ? tokenizedPrimaryInput.get(i) : null; let pushedToken = new ContextToken(lexicalModel); // TODO: assumes that there was no shift in wordbreaking from the @@ -228,7 +239,7 @@ export class ContextTokenization { // tokenized `butter` `fli`. (e.g: `fli` leads to `flight`.) How do // we know to properly relocate the `f` and `l` transforms? let tokenDistribComponent = tokenDistribution.map((seq) => { - const entry = seq[tailIndex]; + const entry = seq.get(i); if(!entry || TransformUtils.isEmpty(entry.sample)) { return null; } else { @@ -250,7 +261,6 @@ export class ContextTokenization { // Auto-replaces the search space to correspond with the new token. tokenization.push(pushedToken); - tailIndex++; } } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/transform-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/transform-tokenization.ts index ed3e76ba42..3b705cfa89 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/transform-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/transform-tokenization.ts @@ -1,26 +1,34 @@ import { LexicalModelTypes } from '@keymanapp/common-types'; +import { applyTransform, type Tokenization } from "@keymanapp/models-templates"; + +import { determineModelTokenizer } from '../model-helpers.js'; + import Context = LexicalModelTypes.Context; import Distribution = LexicalModelTypes.Distribution; import LexicalModel = LexicalModelTypes.LexicalModel; import Transform = LexicalModelTypes.Transform; -import { applyTransform, type Tokenization } from "@keymanapp/models-templates"; -import { determineModelTokenizer } from '#./model-helpers.js'; +import { computeAlignment } from './alignment-helpers.js'; +import { KMWString } from '@keymanapp/web-utils'; /** * Determines a tokenization-aware sequence of (`Transform`) edits, one per * token that would result after applying the incoming keystroke's `Transform` * to its base `Context`. This sequence reproduces the same net effect as the - * original incoming `transform` when applied in sequence. + * original incoming `transform` when applied in sequence. The component + * transforms are then indexed relative to the position of the corresponding + * token in the base `Context`, with 0 matching the original token left of + * the text insertion point, negative indices affecting previous tokens, and + * positive indicies affecting new tokens. * * For example, using English and standard whitespace-based tokenization: * - context: `the quick blue` * - transform: `{ insert: 'rown fox', deleteLeft: 3 }` * - Resulting context: `the quick brown fox` - * - Output: `[ - * { insert: 'rown', deleteLeft: 3 }, - * { insert: ' ', deleteLeft: 0 }, - * { insert: 'fox', deleteLeft: 0 } - * ]` + * - Output: `Map { + * 0 => { insert: 'rown', deleteLeft: 3 }, + * 1 => { insert: ' ', deleteLeft: 0 }, + * 2 => { insert: 'fox', deleteLeft: 0 } + * }` * @param tokenize The tokenization function to utilize, as determined by the * active lexical model or its settings. * @param context The original, unmodified context @@ -31,38 +39,82 @@ export function tokenizeTransform( tokenize: (context: Context) => Tokenization, context: Context, transform: Transform -): Transform[] { +): Map { + if(transform.insert == '' && transform.deleteLeft == 0) { + const map = new Map(); + map.set(0, { insert: '', deleteLeft: 0 }); + return map; + } + // Context does not slide within this function. const postContext = applyTransform(transform, context); + const preTokenization = tokenize(context).left; const postTokenization = tokenize(postContext).left; + if(preTokenization.length == 0) { + preTokenization.push({text: ''}); + } if(postTokenization.length == 0) { postTokenization.push({text: ''}); } - let insert = transform.insert; + const alignment = computeAlignment( + preTokenization.map(t => t.text), + postTokenization.map(t => t.text), + false, + true + ); + if(!alignment.canAlign || alignment.leadTokenShift) { + throw new Error(`Could not align context ${JSON.stringify(context)} before and after transform ${JSON.stringify(transform)}`); + } + + const baseIndex = Math.min(1, 1 - alignment.tailEditLength) + Math.min(0, alignment.tailTokenShift); + + let deleteLeft = transform.deleteLeft; const tokenizedTransforms: Transform[] = []; - for(let index = postTokenization.length - 1; index >= 0; index--) { - const currentToken = postTokenization[index]; - const textLen = currentToken.text.length; + // Create deletion transforms for deleted ... + for(let index = -alignment.tailTokenShift; index > 0; index--) { + const deletedToken = preTokenization.pop(); + const srcLen = KMWString.length(deletedToken.text); + deleteLeft -= KMWString.length(deletedToken.text); - if(textLen < insert.length) { - tokenizedTransforms.unshift({ + tokenizedTransforms.push({ insert: '', deleteLeft: srcLen }); + } + + let insert = transform.insert; + // Avoid emitting an empty transform if we land right at the end of a previous + // token (say, after a backspace) + if(insert.length > 0 || deleteLeft > 0) { + for(let index = postTokenization.length - 1; index >= 0; index--) { + const currentToken = postTokenization[index]; + const srcLen = KMWString.length(preTokenization[index]?.text ?? ''); + const curLen = KMWString.length(currentToken.text); + + if(srcLen >= deleteLeft && curLen >= KMWString.length(insert)) { + tokenizedTransforms.push({ + insert: insert, + deleteLeft: deleteLeft + }); + break; + } + + insert = insert.substring(0, insert.length - currentToken.text.length); + deleteLeft = Math.max(0, deleteLeft - srcLen); + tokenizedTransforms.push({ insert: currentToken.text, - deleteLeft: 0 + deleteLeft: srcLen }); - - insert = insert.substring(0, insert.length - textLen); - } else { - tokenizedTransforms.unshift({ - insert: insert, - deleteLeft: transform.deleteLeft - }); - break; } } - return tokenizedTransforms; + const returnedMap = new Map(); + let pushedCount: number = 0; + while(tokenizedTransforms.length > 0) { + returnedMap.set(baseIndex + pushedCount, tokenizedTransforms.pop()); + pushedCount++; + } + + return returnedMap; } /** @@ -83,7 +135,7 @@ export function tokenizeTransformDistribution( tokenize: (context: Context) => Tokenization, context: Context, transformDistribution: Distribution -): Distribution { +): Distribution> { return transformDistribution.map((transform) => { return { sample: tokenizeTransform(tokenize, context, transform.sample), @@ -97,7 +149,7 @@ export function tokenizeTransformDistribution( * `tokenizeTransform` for each, mapping each transform to its tokenized form in * the returned distribution. * - * It then filters out all incoming Transforms that do not result in the same + * It then filters out all incoming Transforms that do not result in the same final * number of tokens as the "primary input" when applied, as the context-tracker * and predictive-text engine cannot handle word-breaking divergence well at * this time. @@ -110,34 +162,23 @@ export function tokenizeAndFilterDistribution( context: Context, model: LexicalModel, transformDistribution?: Distribution -) { +): Distribution> { let tokenize = determineModelTokenizer(model); const inputTransform = transformDistribution?.[0]; - let transformTokenLength = 0; - let tokenizedDistribution: Distribution = null; - if(inputTransform) { - // These two methods apply transforms internally; do not mutate context here. - // This particularly matters for the 'distribution' variant. - - // What if a pre-whitespace token has a final substitution as PART of an edit? - // Say, ['apple', ' ', ''] => ['apply', ' ', 'n'] - // For now... we can't really handle that case well - modeling the 'e' => 'y' part. - // Will likely require improvements to tokenizeTransform(), which doesn't yet handle - // deleteLeft tokenization for transforms spanning tokens & whitespace. - // - // See: #14361. - // There's a good shot attemptTokenizedAlignment would be useful for it. - transformTokenLength = tokenizeTransform(tokenize, context, inputTransform.sample).length; - tokenizedDistribution = tokenizeTransformDistribution(tokenize, context, transformDistribution); - - // Now we update the context used for context-state management based upon our input. - context = applyTransform(inputTransform.sample, context); - - // While we lack phrase-based / phrase-oriented prediction support, we'll just extract the - // set that matches the token length that results from our input. - tokenizedDistribution = tokenizedDistribution.filter((entry) => entry.sample.length == transformTokenLength); + if(!inputTransform) { + return null; } - return tokenizedDistribution; + // These two methods apply transforms internally; do not mutate context here. + // This particularly matters for the 'distribution' variant. + const tokenizedInputTransform = tokenizeTransform(tokenize, context, inputTransform.sample); + const lastTokenizedInputIndex = [...tokenizedInputTransform.keys()].reverse()[0]; + const tokenizedDistribution = tokenizeTransformDistribution(tokenize, context, transformDistribution); + + // While we lack phrase-based / phrase-oriented prediction support, we'll just extract the + // set that matches the token length that results from our input. + return tokenizedDistribution.filter((entry) => + entry.sample.has(lastTokenizedInputIndex) && !entry.sample.has(lastTokenizedInputIndex + 1) + ); } \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index 173dd10b1d..17e335a8f9 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -12,9 +12,11 @@ import { assert } from 'chai'; import { default as defaultBreaker } from '@keymanapp/models-wordbreakers'; import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; +import { LexicalModelTypes } from '@keymanapp/common-types'; import { ContextToken, ContextTokenization, models } from '@keymanapp/lm-worker/test-index'; +import Transform = LexicalModelTypes.Transform; import TrieModel = models.TrieModel; var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'), @@ -101,6 +103,10 @@ describe('ContextTokenization', function() { const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t)), null); const targetTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day', ' ', ''].map((t) => ({text: t, isWhitespace: t == ' '})); + const inputTransformMap: Map = new Map(); + inputTransformMap.set(1, { insert: ' ', deleteLeft: 0 }); + inputTransformMap.set(2, { insert: '', deleteLeft: 0 }); + const tokenization = baseTokenization.transitionTo( targetTokens, { canAlign: true, @@ -110,7 +116,7 @@ describe('ContextTokenization', function() { tailTokenShift: 2 }, plainModel, - [{ sample: [{ insert: ' ', deleteLeft: 0 }, { insert: '', deleteLeft: 0 }], p: 1}] + [{ sample: inputTransformMap, p: 1}] ); assert.isOk(tokenization); @@ -125,6 +131,9 @@ describe('ContextTokenization', function() { const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t)), null); const targetTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'].map((t) => ({text: t, isWhitespace: t == ' '})); + const inputTransformMap: Map = new Map(); + inputTransformMap.set(0, { insert: 'y', deleteLeft: 0 }); + const tokenization = baseTokenization.transitionTo( targetTokens, { canAlign: true, @@ -134,7 +143,7 @@ describe('ContextTokenization', function() { tailTokenShift: 0 }, plainModel, - [{ sample: [{ insert: 'y', deleteLeft: 0 }], p: 1}] + [{ sample: inputTransformMap, p: 1}] ); assert.isOk(tokenization); @@ -160,6 +169,9 @@ describe('ContextTokenization', function() { "seem", " ", "like", " ", "they'd", " ", "make", " ", "for", " ", "the", " ", "b" ]; const targetTokens = targetTexts.map((t) => ({text: t, isWhitespace: t == ' '})); + const inputTransformMap: Map = new Map(); + inputTransformMap.set(0, { insert: 'b', deleteLeft: 0 }); + const tokenization = baseTokenization.transitionTo( targetTokens, { canAlign: true, @@ -169,7 +181,7 @@ describe('ContextTokenization', function() { tailTokenShift: 0 }, plainModel, - [{ sample: [{ insert: 'b', deleteLeft: 0 }], p: 1}] + [{ sample: inputTransformMap, p: 1}] ); assert.isOk(tokenization); @@ -192,6 +204,9 @@ describe('ContextTokenization', function() { "like", " ", "they'd", " ", "make", " ", "for", " ", "the", " ", "best", " ", "break" ]; const targetTokens = targetTexts.map((t) => ({text: t, isWhitespace: t == ' '})); + const inputTransformMap: Map = new Map(); + inputTransformMap.set(0, { insert: 'k', deleteLeft: 0 }); + const tokenization = baseTokenization.transitionTo( targetTokens, { canAlign: true, @@ -201,7 +216,7 @@ describe('ContextTokenization', function() { tailTokenShift: 0 }, plainModel, - [{ sample: [{ insert: 'k', deleteLeft: 0 }], p: 1}] + [{ sample: inputTransformMap, p: 1}] ); assert.isOk(tokenization); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/transform-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/transform-tokenization.tests.ts index e048b322d8..273cfeb3aa 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/transform-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/transform-tokenization.tests.ts @@ -7,11 +7,35 @@ import { LexicalModelTypes } from '@keymanapp/common-types'; import { tokenizeTransform } from '@keymanapp/lm-worker/test-index'; import Context = LexicalModelTypes.Context; +import Transform = LexicalModelTypes.Transform; const defaultTokenize = (context: Context) => tokenize(defaultBreaker, context); describe('tokenizeTransform', () => { describe('with default wordbreaking', () => { + it('produces a single empty transform at index 0 when an empty transform is input', () => { + const context: Context = { + left: 'an apple a date', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const editTransform = { + insert: '', + deleteLeft: 0 + }; + + const result = tokenizeTransform( + defaultTokenize, + context, + editTransform + ); + + assert.equal(result.size, 1); + assert.deepEqual(result.get(0), editTransform); + }); + it('properly handles simple token-edit transform', () => { const context: Context = { left: 'an apple a date', @@ -31,8 +55,8 @@ describe('tokenizeTransform', () => { editTransform ); - assert.equal(result.length, 1); - assert.deepEqual(result[0], editTransform); + assert.equal(result.size, 1); + assert.deepEqual(result.get(0), editTransform); }); it('properly handles simple token-replacing transform', () => { @@ -54,8 +78,8 @@ describe('tokenizeTransform', () => { editTransform ); - assert.equal(result.length, 1); - assert.deepEqual(result[0], editTransform); + assert.equal(result.size, 1); + assert.deepEqual(result.get(0), editTransform); }); it('handles simple token-replacing transform with cross-token deleteLeft', () => { @@ -69,7 +93,7 @@ describe('tokenizeTransform', () => { // 'an apple any' const editTransform = { insert: 'ny', - deleteLeft: 6 + deleteLeft: 5 }; const result = tokenizeTransform( @@ -78,8 +102,22 @@ describe('tokenizeTransform', () => { editTransform ); - assert.equal(result.length, 1); - assert.deepEqual(result[0], editTransform); + const expectedMap = new Map(); + expectedMap.set(-2, { + insert: 'ny', + deleteLeft: 0 + }); + expectedMap.set(-1, { + insert: '', + deleteLeft: 1 + }); + expectedMap.set(0, { + insert: '', + deleteLeft: 4 + }); + + assert.equal(result.size, expectedMap.size); + assert.deepEqual(result, expectedMap); }); it('properly handles a simple appended whitespace', () => { @@ -101,14 +139,15 @@ describe('tokenizeTransform', () => { editTransform ); - assert.equal(result.length, 2); - assert.deepEqual(result, [ - // The whitespace belongs on the whitespace token that will be added. - editTransform, - // The default-breaker adds an empty token after whitespace, hence this - // empty transform. - { insert: '', deleteLeft: 0 } - ]); + const expectedMap = new Map(); + // The whitespace belongs on the whitespace token that will be added. + expectedMap.set(1, editTransform); + // The default-breaker adds an empty token after whitespace, hence this + // empty transform. + expectedMap.set(2, { insert: '', deleteLeft: 0 }); + + assert.equal(result.size, expectedMap.size); + assert.deepEqual(result, expectedMap); }); it('properly handles a simple appended period', () => { @@ -132,10 +171,40 @@ describe('tokenizeTransform', () => { // The default wordbreaker does not (currently) append a blank token // after standard English punctuation. - assert.equal(result.length, 1); - assert.deepEqual(result, [ + const expectedMap = new Map(); + expectedMap.set(1, editTransform); + assert.equal(result.size, expectedMap.size); + assert.deepEqual(result, expectedMap); + }); + + it('properly deletes a simple appended whitespace', () => { + const context = { + left: 'an apple a day ', + right: '', + startOfBuffer: true, + endOfBuffer: true + }; + + const editTransform = { + insert: '', + deleteLeft: 1 + }; + + const result = tokenizeTransform( + defaultTokenize, + context, editTransform - ]); + ); + + const expectedMap = new Map(); + // The whitespace belongs on the whitespace token that will be added. + expectedMap.set(-1, editTransform); + // The default-breaker adds an empty token after whitespace, hence this + // empty transform. + expectedMap.set(0, { insert: '', deleteLeft: 0 }); + + assert.equal(result.size, expectedMap.size); + assert.deepEqual(result, expectedMap); }); it('handles word-breakable transforms (case 1)', () => { @@ -157,12 +226,15 @@ describe('tokenizeTransform', () => { editTransform ); - assert.equal(result.length, 3); - assert.deepEqual(result, [ - { insert: 'y', deleteLeft: 1 }, - { insert: ' ', deleteLeft: 0 }, - { insert: 'k', deleteLeft: 0 } - ]); + const expectedMap = new Map(); + // dat => day + expectedMap.set(0, { insert: 'y', deleteLeft: 1 }); + // new whitespace + expectedMap.set(1, { insert: ' ', deleteLeft: 0 }); + // new 'k' token + expectedMap.set(2, { insert: 'k', deleteLeft: 0 }); + assert.equal(result.size, expectedMap.size); + assert.deepEqual(result, expectedMap); }); it('handles word-breakable transforms (case 2)', () => { @@ -184,18 +256,18 @@ describe('tokenizeTransform', () => { editTransform ); - assert.equal(result.length, 4); - assert.deepEqual(result, [ - { insert: 'y', deleteLeft: 1 }, - { insert: '.', deleteLeft: 0 }, - { insert: ' ', deleteLeft: 0 }, - { insert: '', deleteLeft: 0 } - ]); + const expectedMap = new Map(); + expectedMap.set(0, { insert: 'y', deleteLeft: 1 }); + expectedMap.set(1, { insert: '.', deleteLeft: 0 }); + expectedMap.set(2, { insert: ' ', deleteLeft: 0 }); + expectedMap.set(3, { insert: '', deleteLeft: 0 }); + assert.equal(result.size, 4); + assert.deepEqual(result, expectedMap); }); it('handles complex breakable cases', () => { const context = { - left: 'an apple a date', + left: 'an apple as date', right: '', startOfBuffer: true, endOfBuffer: true @@ -213,22 +285,31 @@ describe('tokenizeTransform', () => { editTransform ); - assert.equal(result.length, 3); - assert.deepEqual(result, [ - { insert: 'ny', deleteLeft: 6 }, - { insert: ' ', deleteLeft: 0 }, - { insert: 'day', deleteLeft: 0 } - ]); + const expectedMap = new Map(); + // as => any + expectedMap.set(-2, { insert: 'ny', deleteLeft: 1 }); // 2 back from the last token before the text insertion point. + // ' ' replaced with another ' ' (but still edited) + expectedMap.set(-1, { insert: ' ', deleteLeft: 1 }); + // date => day, but with full replacement due to the large deleteLeft. + expectedMap.set( 0, { insert: 'day', deleteLeft: 4 }); // The original token before the text insertion point. + assert.equal(result.size, expectedMap.size); + assert.deepEqual(result, expectedMap); }); }); describe('with mocked dictionary-based wordbreaking', () => { - function mockedTokenization(entries: string[]) { - return { - left: entries.map((text) => { - return {text: text} as Token - }) - } as Tokenization; + function mockedTokenization(map: Map) { + return (context: Context) => { + let tokens = map.get(context.left); + if(!tokens) { + assert.fail("Mocked tokenization was not properly constructed"); + } + return { + left: tokens.map((text) => { + return {text: text} as Token + }) + } as Tokenization; + } } it('properly handles simple token-edit transform', () => { @@ -244,14 +325,19 @@ describe('tokenizeTransform', () => { deleteLeft: 2 }; + const mockMap = new Map(); + mockMap.set('anappleadate', ['an', 'apple', 'a', 'date']); + mockMap.set('anappleaday', ['an', 'apple', 'a', 'day']); const result = tokenizeTransform( - () => mockedTokenization(['an', 'apple', 'a', 'day']), + mockedTokenization(mockMap), context, editTransform ); - assert.equal(result.length, 1); - assert.deepEqual(result[0], editTransform); + const expectedMap = new Map(); + expectedMap.set(0, editTransform); // The original token before the text insertion point. + assert.equal(result.size, expectedMap.size); + assert.deepEqual(result, expectedMap); }); it('properly handles simple token-replacing transform', () => { @@ -267,14 +353,19 @@ describe('tokenizeTransform', () => { deleteLeft: 4 }; + const mockMap = new Map(); + mockMap.set('anappleadate', ['an', 'apple', 'a', 'date']); + mockMap.set('anappleaweek', ['an', 'apple', 'a', 'week']); const result = tokenizeTransform( - () => mockedTokenization(['an', 'apple', 'a', 'week']), + mockedTokenization(mockMap), context, editTransform ); - assert.equal(result.length, 1); - assert.deepEqual(result[0], editTransform); + const expectedMap = new Map(); + expectedMap.set(0, editTransform); // The original token before the text insertion point. + assert.equal(result.size, expectedMap.size); + assert.deepEqual(result, expectedMap); }); it('handles simple token-replacing transform with cross-token deleteLeft', () => { @@ -288,17 +379,23 @@ describe('tokenizeTransform', () => { // 'an apple any' const editTransform = { insert: 'ny', - deleteLeft: 5 + deleteLeft: 4 }; + const mockMap = new Map(); + mockMap.set('anappleadate', ['an', 'apple', 'a', 'date']); + mockMap.set('anappleany', ['an', 'apple', 'any']); const result = tokenizeTransform( - () => mockedTokenization(['an', 'apple', 'any']), + mockedTokenization(mockMap), context, editTransform ); - assert.equal(result.length, 1); - assert.deepEqual(result[0], editTransform); + const expectedMap = new Map(); + expectedMap.set(-1, { insert: 'ny', deleteLeft: 0 }); + expectedMap.set( 0, { insert: '', deleteLeft: 4 }); + assert.equal(result.size, expectedMap.size); + assert.deepEqual(result, expectedMap); }); it('handles word-breakable transforms (case 1)', () => { @@ -314,17 +411,20 @@ describe('tokenizeTransform', () => { deleteLeft: 1 }; + const mockMap = new Map(); + mockMap.set('anappleadat', ['an', 'apple', 'a', 'dat']); + mockMap.set('anappleadayk', ['an', 'apple', 'a', 'day', 'k']); const result = tokenizeTransform( - () => mockedTokenization(['an', 'apple', 'any', 'k']), + mockedTokenization(mockMap), context, editTransform ); - assert.equal(result.length, 2); - assert.deepEqual(result, [ - { insert: 'y', deleteLeft: 1 }, - { insert: 'k', deleteLeft: 0 } - ]); + const expectedMap = new Map(); + expectedMap.set(0, { insert: 'y', deleteLeft: 1 }); + expectedMap.set(1, { insert: 'k', deleteLeft: 0 }); + assert.equal(result.size, expectedMap.size); + assert.deepEqual(result, expectedMap); }); it('handles word-breakable transforms (case 2)', () => { @@ -340,17 +440,20 @@ describe('tokenizeTransform', () => { deleteLeft: 1 }; + const mockMap = new Map(); + mockMap.set('anappleadat', ['an', 'apple', 'a', 'dat']); + mockMap.set('anappleaday.', ['an', 'apple', 'a', 'day', '.']); const result = tokenizeTransform( - () => mockedTokenization(['an', 'apple', 'any', 'day', '.']), + mockedTokenization(mockMap), context, editTransform ); - assert.equal(result.length, 2); - assert.deepEqual(result, [ - { insert: 'y', deleteLeft: 1 }, - { insert: '.', deleteLeft: 0 } - ]); + const expectedMap = new Map(); + expectedMap.set(0, { insert: 'y', deleteLeft: 1 }); + expectedMap.set(1, { insert: '.', deleteLeft: 0 }); + assert.equal(result.size, expectedMap.size); + assert.deepEqual(result, expectedMap); }); it('handles word-breakable transforms (case 2 alternate output)', () => { @@ -366,18 +469,21 @@ describe('tokenizeTransform', () => { deleteLeft: 1 }; + const mockMap = new Map(); + mockMap.set('anappleadat', ['an', 'apple', 'a', 'dat']); + mockMap.set('anappleaday.', ['an', 'apple', 'a', 'day', '.', '']); const result = tokenizeTransform( - () => mockedTokenization(['an', 'apple', 'any', 'day', '.', '']), + mockedTokenization(mockMap), context, editTransform ); - assert.equal(result.length, 3); - assert.deepEqual(result, [ - { insert: 'y', deleteLeft: 1 }, - { insert: '.', deleteLeft: 0 }, - { insert: '', deleteLeft: 0 } - ]); + const expectedMap = new Map(); + expectedMap.set(0, { insert: 'y', deleteLeft: 1 }); + expectedMap.set(1, { insert: '.', deleteLeft: 0 }); + expectedMap.set(2, { insert: '', deleteLeft: 0}); + assert.equal(result.size, expectedMap.size); + assert.deepEqual(result, expectedMap); }); it('handles complex breakable cases', () => { @@ -391,20 +497,23 @@ describe('tokenizeTransform', () => { // 'an apple any' const editTransform = { insert: 'nyday', - deleteLeft: 5 + deleteLeft: 4 }; + const mockMap = new Map(); + mockMap.set('anappleadate', ['an', 'apple', 'a', 'date']); + mockMap.set('anappleanyday', ['an', 'apple', 'any', 'day']); const result = tokenizeTransform( - () => mockedTokenization(['an', 'apple', 'any', 'day']), + mockedTokenization(mockMap), context, editTransform ); - assert.equal(result.length, 2); - assert.deepEqual(result, [ - { insert: 'ny', deleteLeft: 5 }, - { insert: 'day', deleteLeft: 0 } - ]); + const expectedMap = new Map(); + expectedMap.set(-1, { insert: 'ny', deleteLeft: 0 }); + expectedMap.set( 0, { insert: 'day', deleteLeft: 4 }); + assert.equal(result.size, expectedMap.size); + assert.deepEqual(result, expectedMap); }); }); }); \ No newline at end of file