diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 8adf19ab96..6cce69bf3e 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -44,42 +44,53 @@ interface TokenSplitMap { }; /** - * Represents the expected properties for the context and transform tokenizations - * that would result from applying a transform to the state underlying an existing - * context tokenization. + * Represents the portion of a tokenization that may be affected by incoming + * Transforms and how the existing components of the tokenization are affected + * by any word-boundary shifts in existing content that occur due incoming + * Transform side-effects. */ -interface PrecomputedTokenizationTransition { +export interface TokenizationEdgeAlignment { /** - * Metadata about how the existing tokenization and its contents correspond to - * their counterparts within the input `Transform`'s target tokenization. + * Denotes any token merge edits needed after applying the Transform. */ - tokenMapping: { - /** - * Denotes any token merge edits needed after applying the Transform. - */ - merges: TokenMergeMap[]; - /** - * Denotes any token split edits needed after applying the Transform. - */ - splits: TokenSplitMap[]; - /** - * Denotes any further token edits needed that cannot be attributed to - * 'merge's, 'split's, or edits from the input `Transform`. - */ - unmappedEdits: EditTuple[]; + merges: TokenMergeMap[]; + /** + * Denotes any token split edits needed after applying the Transform. + */ + splits: TokenSplitMap[]; + /** + * Denotes any further token edits needed that cannot be attributed to + * 'merge's, 'split's, or edits from the input `Transform`. + */ + unmappedEdits: EditTuple[]; - /** - * The edge window defining the potential range of tokenization adjustments - * needed after applying the Transform - */ - edgeWindow: RetokenizedEdgeWindow - } + /** + * The edge window defining the potential range of tokenization adjustments + * needed after applying the Transform + */ + edgeWindow: RetokenizedEdgeWindow +} + +/** + * An analysis on how an existing tokenization and its contents will be affected + * by applying an input `Transform`. + */ +export interface TokenizationTransitionEdits { + /** + * Represents the portion of the existing tokenization that may be affected by + * the incoming Transform. + * + * Also represents the changes needed to model any word-boundary shifts + * implied by the Transform but not generated within new content produced by + * its text edits. + */ + alignment: TokenizationEdgeAlignment; /** * The tokenized form of the input Transform, indexed by position relative to * the end of the original context's tail token. */ - transformMap: Map + tokenizedTransform: Map } /** @@ -307,7 +318,7 @@ export class ContextTokenization { lexicalModel: LexicalModel, transform: Transform, edgeOptions?: EdgeWindowOptions - ): PrecomputedTokenizationTransition { + ): TokenizationTransitionEdits { // Step 4: now that our window's been properly updated, determine what the // input's effects on the context is. // @@ -432,13 +443,13 @@ export class ContextTokenization { } return { - tokenMapping: { + alignment: { edgeWindow: {...edgeWindow, retokenization: retokenizedEdge}, merges, splits, unmappedEdits }, - transformMap, + tokenizedTransform: transformMap, }; } diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index c99a1af2da..0729257af9 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -976,11 +976,11 @@ describe('ContextTokenization', function() { ); // for tokenization. - assert.equal(results.transformMap.size, 1); - assert.deepEqual(results.transformMap.get(0), editTransform); - assert.deepEqual(results.tokenMapping.merges, []); - assert.deepEqual(results.tokenMapping.splits, []); - assert.deepEqual(results.tokenMapping.unmappedEdits, []); + assert.equal(results.tokenizedTransform.size, 1); + assert.deepEqual(results.tokenizedTransform.get(0), editTransform); + assert.deepEqual(results.alignment.merges, []); + assert.deepEqual(results.alignment.splits, []); + assert.deepEqual(results.alignment.unmappedEdits, []); }); it('properly handles basic token-edit transform', () => { @@ -997,11 +997,11 @@ describe('ContextTokenization', function() { editTransform ); - assert.equal(results.transformMap.size, 1); - assert.deepEqual(results.transformMap.get(0), editTransform); - assert.deepEqual(results.tokenMapping.merges, []); - assert.deepEqual(results.tokenMapping.splits, []); - assert.deepEqual(results.tokenMapping.unmappedEdits, []); + assert.equal(results.tokenizedTransform.size, 1); + assert.deepEqual(results.tokenizedTransform.get(0), editTransform); + assert.deepEqual(results.alignment.merges, []); + assert.deepEqual(results.alignment.splits, []); + assert.deepEqual(results.alignment.unmappedEdits, []); }); it('properly handles simple token-edit transform', () => { @@ -1019,13 +1019,13 @@ describe('ContextTokenization', function() { edgeWindowSpec ); - assert.equal(results.transformMap.size, 1); - assert.deepEqual(results.transformMap.get(0), editTransform); - assert.deepEqual(results.tokenMapping.merges, []); - assert.deepEqual(results.tokenMapping.splits, []); - assert.deepEqual(results.tokenMapping.unmappedEdits, []); + assert.equal(results.tokenizedTransform.size, 1); + assert.deepEqual(results.tokenizedTransform.get(0), editTransform); + assert.deepEqual(results.alignment.merges, []); + assert.deepEqual(results.alignment.splits, []); + assert.deepEqual(results.alignment.unmappedEdits, []); - assert.deepEqual(results.tokenMapping.edgeWindow, { + assert.deepEqual(results.alignment.edgeWindow, { retokenization: ['apple', ' ', 'a', ' ', 'da'], retokenizationText: 'apple a da', editBoundary: { @@ -1054,13 +1054,13 @@ describe('ContextTokenization', function() { edgeWindowSpec ); - assert.equal(results.transformMap.size, 1); - assert.deepEqual(results.transformMap.get(0), editTransform); - assert.deepEqual(results.tokenMapping.merges, []); - assert.deepEqual(results.tokenMapping.splits, []); - assert.deepEqual(results.tokenMapping.unmappedEdits, []); + assert.equal(results.tokenizedTransform.size, 1); + assert.deepEqual(results.tokenizedTransform.get(0), editTransform); + assert.deepEqual(results.alignment.merges, []); + assert.deepEqual(results.alignment.splits, []); + assert.deepEqual(results.alignment.unmappedEdits, []); - assert.deepEqual(results.tokenMapping.edgeWindow, { + assert.deepEqual(results.alignment.edgeWindow, { retokenization: ['apple', ' ', 'a', ' ', 'da'].map(t => toMathematicalSMP(t)), retokenizationText: toMathematicalSMP('apple a da'), editBoundary: { @@ -1087,11 +1087,11 @@ describe('ContextTokenization', function() { editTransform ); - assert.equal(results.transformMap.size, 1); - assert.deepEqual(results.transformMap.get(0), editTransform); - assert.deepEqual(results.tokenMapping.merges, []); - assert.deepEqual(results.tokenMapping.splits, []); - assert.deepEqual(results.tokenMapping.unmappedEdits, []); + assert.equal(results.tokenizedTransform.size, 1); + assert.deepEqual(results.tokenizedTransform.get(0), editTransform); + assert.deepEqual(results.alignment.merges, []); + assert.deepEqual(results.alignment.splits, []); + assert.deepEqual(results.alignment.unmappedEdits, []); }); it('handles simple token-replacing transform with cross-token deleteLeft', () => { @@ -1123,11 +1123,11 @@ describe('ContextTokenization', function() { deleteLeft: 4 }); - assert.equal(results.transformMap.size, expectedMap.size); - assert.deepEqual(results.transformMap, expectedMap); - assert.deepEqual(results.tokenMapping.merges, []); - assert.deepEqual(results.tokenMapping.splits, []); - assert.deepEqual(results.tokenMapping.unmappedEdits, []); + assert.equal(results.tokenizedTransform.size, expectedMap.size); + assert.deepEqual(results.tokenizedTransform, expectedMap); + assert.deepEqual(results.alignment.merges, []); + assert.deepEqual(results.alignment.splits, []); + assert.deepEqual(results.alignment.unmappedEdits, []); }); it('properly handles a simple appended whitespace', () => { @@ -1151,11 +1151,11 @@ describe('ContextTokenization', function() { // empty transform. expectedMap.set(2, { insert: '', deleteLeft: 0 }); - assert.equal(results.transformMap.size, expectedMap.size); - assert.deepEqual(results.transformMap, expectedMap); - assert.deepEqual(results.tokenMapping.merges, []); - assert.deepEqual(results.tokenMapping.splits, []); - assert.deepEqual(results.tokenMapping.unmappedEdits, []); + assert.equal(results.tokenizedTransform.size, expectedMap.size); + assert.deepEqual(results.tokenizedTransform, expectedMap); + assert.deepEqual(results.alignment.merges, []); + assert.deepEqual(results.alignment.splits, []); + assert.deepEqual(results.alignment.unmappedEdits, []); }); it('properly handles a simple appended period', () => { @@ -1176,11 +1176,11 @@ describe('ContextTokenization', function() { // after standard English punctuation. const expectedMap = new Map(); expectedMap.set(1, editTransform); - assert.equal(results.transformMap.size, expectedMap.size); - assert.deepEqual(results.transformMap, expectedMap); - assert.deepEqual(results.tokenMapping.merges, []); - assert.deepEqual(results.tokenMapping.splits, []); - assert.deepEqual(results.tokenMapping.unmappedEdits, []); + assert.equal(results.tokenizedTransform.size, expectedMap.size); + assert.deepEqual(results.tokenizedTransform, expectedMap); + assert.deepEqual(results.alignment.merges, []); + assert.deepEqual(results.alignment.splits, []); + assert.deepEqual(results.alignment.unmappedEdits, []); }); it('properly deletes a simple appended whitespace', () => { @@ -1201,11 +1201,11 @@ describe('ContextTokenization', function() { // The whitespace belongs on the whitespace token that will be added. expectedMap.set(-1, editTransform); - assert.equal(results.transformMap.size, expectedMap.size); - assert.deepEqual(results.transformMap, expectedMap); - assert.deepEqual(results.tokenMapping.merges, []); - assert.deepEqual(results.tokenMapping.splits, []); - assert.deepEqual(results.tokenMapping.unmappedEdits, []); + assert.equal(results.tokenizedTransform.size, expectedMap.size); + assert.deepEqual(results.tokenizedTransform, expectedMap); + assert.deepEqual(results.alignment.merges, []); + assert.deepEqual(results.alignment.splits, []); + assert.deepEqual(results.alignment.unmappedEdits, []); }); it('handles word-breakable transforms (case 1)', () => { @@ -1229,13 +1229,13 @@ describe('ContextTokenization', function() { expectedMap.set(1, { insert: ' ', deleteLeft: 0 }); // new 'k' token expectedMap.set(2, { insert: 'k', deleteLeft: 0 }); - assert.equal(results.transformMap.size, expectedMap.size); - assert.deepEqual(results.transformMap, expectedMap); - assert.deepEqual(results.tokenMapping.merges, []); - assert.deepEqual(results.tokenMapping.splits, []); - assert.deepEqual(results.tokenMapping.unmappedEdits, []); + assert.equal(results.tokenizedTransform.size, expectedMap.size); + assert.deepEqual(results.tokenizedTransform, expectedMap); + assert.deepEqual(results.alignment.merges, []); + assert.deepEqual(results.alignment.splits, []); + assert.deepEqual(results.alignment.unmappedEdits, []); - assert.deepEqual(results.tokenMapping.edgeWindow, { + assert.deepEqual(results.alignment.edgeWindow, { retokenization: ['apple', ' ', 'a', ' ', 'da'], retokenizationText: 'apple a da', editBoundary: { @@ -1270,13 +1270,13 @@ describe('ContextTokenization', function() { expectedMap.set(1, { insert: toMathematicalSMP(' '), deleteLeft: 0 }); // new 'k' token expectedMap.set(2, { insert: toMathematicalSMP('k'), deleteLeft: 0 }); - assert.equal(results.transformMap.size, expectedMap.size); - assert.deepEqual(results.transformMap, expectedMap); - assert.deepEqual(results.tokenMapping.merges, []); - assert.deepEqual(results.tokenMapping.splits, []); - assert.deepEqual(results.tokenMapping.unmappedEdits, []); + assert.equal(results.tokenizedTransform.size, expectedMap.size); + assert.deepEqual(results.tokenizedTransform, expectedMap); + assert.deepEqual(results.alignment.merges, []); + assert.deepEqual(results.alignment.splits, []); + assert.deepEqual(results.alignment.unmappedEdits, []); - assert.deepEqual(results.tokenMapping.edgeWindow, { + assert.deepEqual(results.alignment.edgeWindow, { retokenization: ['apple', ' ', 'a', ' ', 'da'].map(t => toMathematicalSMP(t)), retokenizationText: toMathematicalSMP('apple a da'), editBoundary: { @@ -1309,11 +1309,11 @@ describe('ContextTokenization', function() { expectedMap.set(1, { insert: '.', deleteLeft: 0 }); expectedMap.set(2, { insert: ' ', deleteLeft: 0 }); expectedMap.set(3, { insert: '', deleteLeft: 0 }); - assert.equal(results.transformMap.size, 4); - assert.deepEqual(results.transformMap, expectedMap); - assert.deepEqual(results.tokenMapping.merges, []); - assert.deepEqual(results.tokenMapping.splits, []); - assert.deepEqual(results.tokenMapping.unmappedEdits, []); + assert.equal(results.tokenizedTransform.size, 4); + assert.deepEqual(results.tokenizedTransform, expectedMap); + assert.deepEqual(results.alignment.merges, []); + assert.deepEqual(results.alignment.splits, []); + assert.deepEqual(results.alignment.unmappedEdits, []); }); it('handles complex breakable cases', () => { @@ -1338,11 +1338,11 @@ describe('ContextTokenization', function() { expectedMap.set(-1, { insert: ' ', deleteLeft: 1 }); // date => day, but with full replacement due to the large deleteLeft. expectedMap.set( 0, { insert: 'day', deleteLeft: 4 }); // The original token before the text insertion point. - assert.equal(results.transformMap.size, expectedMap.size); - assert.deepEqual(results.transformMap, expectedMap); - assert.deepEqual(results.tokenMapping.merges, []); - assert.deepEqual(results.tokenMapping.splits, []); - assert.deepEqual(results.tokenMapping.unmappedEdits, []); + assert.equal(results.tokenizedTransform.size, expectedMap.size); + assert.deepEqual(results.tokenizedTransform, expectedMap); + assert.deepEqual(results.alignment.merges, []); + assert.deepEqual(results.alignment.splits, []); + assert.deepEqual(results.alignment.unmappedEdits, []); }); it('properly aligns tokenization of transforms that match-replace existing tokens (1)', () => { @@ -1365,11 +1365,11 @@ describe('ContextTokenization', function() { expectedMap.set(0, { insert: 'properly', deleteLeft: 8 }); expectedMap.set(1, { insert: ' ', deleteLeft: 0 }); expectedMap.set(2, { insert: '', deleteLeft: 0 }); - assert.equal(results.transformMap.size, 3); - assert.deepEqual(results.transformMap, expectedMap); - assert.deepEqual(results.tokenMapping.merges, []); - assert.deepEqual(results.tokenMapping.splits, []); - assert.deepEqual(results.tokenMapping.unmappedEdits, []); + assert.equal(results.tokenizedTransform.size, 3); + assert.deepEqual(results.tokenizedTransform, expectedMap); + assert.deepEqual(results.alignment.merges, []); + assert.deepEqual(results.alignment.splits, []); + assert.deepEqual(results.alignment.unmappedEdits, []); }); it('properly aligns tokenization of transforms that match-replace existing tokens (2)', () => { @@ -1392,11 +1392,11 @@ describe('ContextTokenization', function() { expectedMap.set(0, { insert: 'properly', deleteLeft: 8 }); expectedMap.set(1, { insert: ' ', deleteLeft: 0 }); expectedMap.set(2, { insert: '', deleteLeft: 0 }); - assert.equal(results.transformMap.size, 3); - assert.deepEqual(results.transformMap, expectedMap); - assert.deepEqual(results.tokenMapping.merges, []); - assert.deepEqual(results.tokenMapping.splits, []); - assert.deepEqual(results.tokenMapping.unmappedEdits, []); + assert.equal(results.tokenizedTransform.size, 3); + assert.deepEqual(results.tokenizedTransform, expectedMap); + assert.deepEqual(results.alignment.merges, []); + assert.deepEqual(results.alignment.splits, []); + assert.deepEqual(results.alignment.unmappedEdits, []); }); it('properly places extra whitespaces on preceding whitespace token', () => { @@ -1417,11 +1417,11 @@ describe('ContextTokenization', function() { const expectedMap = new Map(); expectedMap.set(-1, { insert: ' ', deleteLeft: 0 }); expectedMap.set(0, { insert: '', deleteLeft: 0 }); - assert.equal(results.transformMap.size, 2); - assert.deepEqual(results.transformMap, expectedMap); - assert.deepEqual(results.tokenMapping.merges, []); - assert.deepEqual(results.tokenMapping.splits, []); - assert.deepEqual(results.tokenMapping.unmappedEdits, []); + assert.equal(results.tokenizedTransform.size, 2); + assert.deepEqual(results.tokenizedTransform, expectedMap); + assert.deepEqual(results.alignment.merges, []); + assert.deepEqual(results.alignment.splits, []); + assert.deepEqual(results.alignment.unmappedEdits, []); }); it('properly aligns degenerate input cases (1)', () => { @@ -1447,11 +1447,11 @@ describe('ContextTokenization', function() { expectedMap.set(2, { insert: 'brown', deleteLeft: 0 }); expectedMap.set(3, { insert: ' ', deleteLeft: 0 }); expectedMap.set(4, { insert: 'fox', deleteLeft: 0 }); - assert.equal(results.transformMap.size, 7); - assert.deepEqual(results.transformMap, expectedMap); - assert.deepEqual(results.tokenMapping.merges, []); - assert.deepEqual(results.tokenMapping.splits, []); - assert.deepEqual(results.tokenMapping.unmappedEdits, []); + assert.equal(results.tokenizedTransform.size, 7); + assert.deepEqual(results.tokenizedTransform, expectedMap); + assert.deepEqual(results.alignment.merges, []); + assert.deepEqual(results.alignment.splits, []); + assert.deepEqual(results.alignment.unmappedEdits, []); }); it('returns the standard edge window for empty transform inputs', () => { @@ -1471,8 +1471,8 @@ describe('ContextTokenization', function() { edgeWindowSpec ); - assert.deepEqual(results.tokenMapping.edgeWindow, {...windowResults, retokenization: results.tokenMapping.edgeWindow.retokenization}); - assert.deepEqual(results.tokenMapping.edgeWindow, { + assert.deepEqual(results.alignment.edgeWindow, {...windowResults, retokenization: results.alignment.edgeWindow.retokenization}); + assert.deepEqual(results.alignment.edgeWindow, { retokenizationText: 'brown fox', retokenization: ['brown', ' ', 'fox'], editBoundary: { @@ -1503,8 +1503,8 @@ describe('ContextTokenization', function() { edgeWindowSpec ); - assert.deepEqual(results.tokenMapping.edgeWindow, {...windowResults, retokenization: results.tokenMapping.edgeWindow.retokenization}); - assert.deepEqual(results.tokenMapping.edgeWindow, { + assert.deepEqual(results.alignment.edgeWindow, {...windowResults, retokenization: results.alignment.edgeWindow.retokenization}); + assert.deepEqual(results.alignment.edgeWindow, { retokenizationText: 'brown fox ', retokenization: ['brown', ' ', 'fox', ' '], // no final '' token editBoundary: { @@ -1536,8 +1536,8 @@ describe('ContextTokenization', function() { edgeWindowSpec ); - assert.deepEqual(results.tokenMapping.edgeWindow, {...windowResults, retokenization: results.tokenMapping.edgeWindow.retokenization}); - assert.deepEqual(results.tokenMapping.edgeWindow, { + assert.deepEqual(results.alignment.edgeWindow, {...windowResults, retokenization: results.alignment.edgeWindow.retokenization}); + assert.deepEqual(results.alignment.edgeWindow, { retokenizationText: 'brown fox', retokenization: ['brown', ' ', 'fox'], editBoundary: { @@ -1568,8 +1568,8 @@ describe('ContextTokenization', function() { edgeWindowSpec ); - assert.deepEqual(results.tokenMapping.edgeWindow, {...windowResults, retokenization: results.tokenMapping.edgeWindow.retokenization}); - assert.deepEqual(results.tokenMapping.edgeWindow, { + assert.deepEqual(results.alignment.edgeWindow, {...windowResults, retokenization: results.alignment.edgeWindow.retokenization}); + assert.deepEqual(results.alignment.edgeWindow, { retokenizationText: ' brown f', retokenization: [' ', 'brown', ' ', 'f'], editBoundary: { @@ -1600,8 +1600,8 @@ describe('ContextTokenization', function() { edgeWindowSpec ); - assert.deepEqual(results.tokenMapping.edgeWindow, {...windowResults, retokenization: results.tokenMapping.edgeWindow.retokenization}); - assert.deepEqual(results.tokenMapping.edgeWindow, { + assert.deepEqual(results.alignment.edgeWindow, {...windowResults, retokenization: results.alignment.edgeWindow.retokenization}); + assert.deepEqual(results.alignment.edgeWindow, { retokenizationText: 'quick ', retokenization: ['quick', ' '], editBoundary: { @@ -1633,10 +1633,10 @@ describe('ContextTokenization', function() { const expectedMap = new Map(); // index 0: the merged `can'` token expectedMap.set(0, { insert: 't', deleteLeft: 0 }); - assert.equal(results.transformMap.size, 1); - assert.deepEqual(results.transformMap, expectedMap); - assert.equal(results.tokenMapping.edgeWindow.sliceIndex, 4); - assert.deepEqual(results.tokenMapping.merges, [ + assert.equal(results.tokenizedTransform.size, 1); + assert.deepEqual(results.tokenizedTransform, expectedMap); + assert.equal(results.alignment.edgeWindow.sliceIndex, 4); + assert.deepEqual(results.alignment.merges, [ { inputs: [ { text: 'can', index: 2 /* + 4 */ }, @@ -1644,8 +1644,8 @@ describe('ContextTokenization', function() { ], match: { text: 'can\'t', index: 2 /* + 4 */} } ]); - assert.deepEqual(results.tokenMapping.splits, []); - assert.deepEqual(results.tokenMapping.unmappedEdits, []); + assert.deepEqual(results.alignment.splits, []); + assert.deepEqual(results.alignment.unmappedEdits, []); }); it('properly handles English contraction transitions (2)', () => { @@ -1667,11 +1667,11 @@ describe('ContextTokenization', function() { // index 0: the split-off `'` token. expectedMap.set(1, { insert: ' ', deleteLeft: 0 }); expectedMap.set(2, { insert: '', deleteLeft: 0 }); - assert.equal(results.transformMap.size, 2); - assert.deepEqual(results.transformMap, expectedMap); - assert.equal(results.tokenMapping.edgeWindow.sliceIndex, 4); - assert.deepEqual(results.tokenMapping.merges, []); - assert.deepEqual(results.tokenMapping.splits, [ + assert.equal(results.tokenizedTransform.size, 2); + assert.deepEqual(results.tokenizedTransform, expectedMap); + assert.equal(results.alignment.edgeWindow.sliceIndex, 4); + assert.deepEqual(results.alignment.merges, []); + assert.deepEqual(results.alignment.splits, [ { input: { text: 'can\'', index: 2 /* + 4 */}, matches: [ @@ -1680,7 +1680,7 @@ describe('ContextTokenization', function() { ] } ]); - assert.deepEqual(results.tokenMapping.unmappedEdits, []); + assert.deepEqual(results.alignment.unmappedEdits, []); }); it('properly handles English contraction transitions (3)', () => { @@ -1699,11 +1699,11 @@ describe('ContextTokenization', function() { const expectedMap = new Map(); expectedMap.set(1, { insert: '?', deleteLeft: 0 }); - assert.equal(results.transformMap.size, 1); - assert.deepEqual(results.transformMap, expectedMap); - assert.equal(results.tokenMapping.edgeWindow.sliceIndex, 4); - assert.deepEqual(results.tokenMapping.merges, []); - assert.deepEqual(results.tokenMapping.splits, [ + assert.equal(results.tokenizedTransform.size, 1); + assert.deepEqual(results.tokenizedTransform, expectedMap); + assert.equal(results.alignment.edgeWindow.sliceIndex, 4); + assert.deepEqual(results.alignment.merges, []); + assert.deepEqual(results.alignment.splits, [ { input: { text: 'can\'', index: 2 /* + 4 */}, matches: [ @@ -1712,7 +1712,7 @@ describe('ContextTokenization', function() { ] } ]); - assert.deepEqual(results.tokenMapping.unmappedEdits, []); + assert.deepEqual(results.alignment.unmappedEdits, []); }); });