diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 157ce10ae8..3c0b3edff4 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -68,7 +68,12 @@ export interface TokenizationEdgeAlignment { * The edge window defining the potential range of tokenization adjustments * needed after applying the Transform */ - edgeWindow: RetokenizedEdgeWindow + edgeWindow: RetokenizedEdgeWindow; + + /** + * The number of tokens deleted from the end, without being erased. + */ + removedTokenCount: number; } /** @@ -430,8 +435,17 @@ export class ContextTokenization { // accordingly. const tailIndex = 0 - (stackedDeletes.length - 1) + (editBoundary.omitsEmptyToken ? -1 : 0); // Mutates stackedInserts, stackedDeletes. + const baseRemovedTokenCount = Math.max(0, stackedDeletes.length - stackedInserts.length); const transformMap = assembleTransforms(stackedInserts, stackedDeletes, tailIndex); + // If there's an empty transform in the 0 position and we already know we're + // dropping tokens - and only deleting - we're dropping an + // otherwise-untracked empty token - make sure it's included! + const droppedFinalTransform = baseRemovedTokenCount > 0 && transform.insert == '' && TransformUtils.isEmpty(transformMap.get(0)); + // Past that, if we have more delete entries than insert entries for our transforms, we + // dropped some tokens outright. + const removedTokenCount = baseRemovedTokenCount + (droppedFinalTransform ? 1 : 0); + // Final step: check for any unexpected boundary shifts not mappable to 'merge' / 'split' // and not caused by transforms. All transforms always apply in sequence at the end. const unmappedEdits: EditTuple[] = []; @@ -457,7 +471,8 @@ export class ContextTokenization { edgeWindow: {...edgeWindow, retokenization: retokenizedEdge}, merges, splits, - unmappedEdits + unmappedEdits, + removedTokenCount }, tokenizedTransform: transformMap, }; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index 2c15625094..c851c8e379 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -1010,6 +1010,7 @@ describe('ContextTokenization', function() { assert.deepEqual(results.alignment.merges, []); assert.deepEqual(results.alignment.splits, []); assert.deepEqual(results.alignment.unmappedEdits, []); + assert.equal(results.alignment.removedTokenCount, 0); }); it('properly handles basic token-edit transform', () => { @@ -1031,6 +1032,7 @@ describe('ContextTokenization', function() { assert.deepEqual(results.alignment.merges, []); assert.deepEqual(results.alignment.splits, []); assert.deepEqual(results.alignment.unmappedEdits, []); + assert.equal(results.alignment.removedTokenCount, 0); }); it('properly handles basic char-delete transform', () => { @@ -1052,6 +1054,7 @@ describe('ContextTokenization', function() { assert.deepEqual(results.alignment.merges, []); assert.deepEqual(results.alignment.splits, []); assert.deepEqual(results.alignment.unmappedEdits, []); + assert.equal(results.alignment.removedTokenCount, 0); }); it('properly handles simple token-edit transform', () => { @@ -1074,6 +1077,7 @@ describe('ContextTokenization', function() { assert.deepEqual(results.alignment.merges, []); assert.deepEqual(results.alignment.splits, []); assert.deepEqual(results.alignment.unmappedEdits, []); + assert.equal(results.alignment.removedTokenCount, 0); assert.deepEqual(results.alignment.edgeWindow, { retokenization: ['apple', ' ', 'a', ' ', 'da'], @@ -1111,6 +1115,7 @@ describe('ContextTokenization', function() { assert.deepEqual(results.alignment.merges, []); assert.deepEqual(results.alignment.splits, []); assert.deepEqual(results.alignment.unmappedEdits, []); + assert.equal(results.alignment.removedTokenCount, 0); assert.deepEqual(results.alignment.edgeWindow, { retokenization: ['apple', ' ', 'a', ' ', 'da'].map(t => toMathematicalSMP(t)), @@ -1146,6 +1151,7 @@ describe('ContextTokenization', function() { assert.deepEqual(results.alignment.merges, []); assert.deepEqual(results.alignment.splits, []); assert.deepEqual(results.alignment.unmappedEdits, []); + assert.equal(results.alignment.removedTokenCount, 0); }); it('handles simple token-replacing transform with cross-token deleteLeft', () => { @@ -1182,6 +1188,7 @@ describe('ContextTokenization', function() { assert.deepEqual(results.alignment.merges, []); assert.deepEqual(results.alignment.splits, []); assert.deepEqual(results.alignment.unmappedEdits, []); + assert.equal(results.alignment.removedTokenCount, 2); }); it('handles token-replacing transform with cross-token deleteLeft after whitespace', () => { @@ -1226,6 +1233,7 @@ describe('ContextTokenization', function() { assert.deepEqual(results.alignment.merges, []); assert.deepEqual(results.alignment.splits, []); assert.deepEqual(results.alignment.unmappedEdits, []); + assert.equal(results.alignment.removedTokenCount, 4); }); it('properly handles a simple appended whitespace', () => { @@ -1254,6 +1262,7 @@ describe('ContextTokenization', function() { assert.deepEqual(results.alignment.merges, []); assert.deepEqual(results.alignment.splits, []); assert.deepEqual(results.alignment.unmappedEdits, []); + assert.equal(results.alignment.removedTokenCount, 0); }); it('properly handles a simple appended period', () => { @@ -1279,6 +1288,7 @@ describe('ContextTokenization', function() { assert.deepEqual(results.alignment.merges, []); assert.deepEqual(results.alignment.splits, []); assert.deepEqual(results.alignment.unmappedEdits, []); + assert.equal(results.alignment.removedTokenCount, 0); }); it('properly deletes a simple appended whitespace', () => { @@ -1305,6 +1315,7 @@ describe('ContextTokenization', function() { assert.deepEqual(results.alignment.merges, []); assert.deepEqual(results.alignment.splits, []); assert.deepEqual(results.alignment.unmappedEdits, []); + assert.equal(results.alignment.removedTokenCount, 2); }); it('handles word-breakable transforms (case 1)', () => { @@ -1333,6 +1344,7 @@ describe('ContextTokenization', function() { assert.deepEqual(results.alignment.merges, []); assert.deepEqual(results.alignment.splits, []); assert.deepEqual(results.alignment.unmappedEdits, []); + assert.equal(results.alignment.removedTokenCount, 0); assert.deepEqual(results.alignment.edgeWindow, { retokenization: ['apple', ' ', 'a', ' ', 'da'], @@ -1376,6 +1388,7 @@ describe('ContextTokenization', function() { assert.deepEqual(results.alignment.merges, []); assert.deepEqual(results.alignment.splits, []); assert.deepEqual(results.alignment.unmappedEdits, []); + assert.equal(results.alignment.removedTokenCount, 0); assert.deepEqual(results.alignment.edgeWindow, { retokenization: ['apple', ' ', 'a', ' ', 'da'].map(t => toMathematicalSMP(t)), @@ -1417,6 +1430,7 @@ describe('ContextTokenization', function() { assert.deepEqual(results.alignment.merges, []); assert.deepEqual(results.alignment.splits, []); assert.deepEqual(results.alignment.unmappedEdits, []); + assert.equal(results.alignment.removedTokenCount, 0); }); it('handles complex breakable cases', () => { @@ -1446,6 +1460,7 @@ describe('ContextTokenization', function() { assert.deepEqual(results.alignment.merges, []); assert.deepEqual(results.alignment.splits, []); assert.deepEqual(results.alignment.unmappedEdits, []); + assert.equal(results.alignment.removedTokenCount, 0); }); it('properly aligns tokenization of transforms that match-replace existing tokens (1)', () => { @@ -1473,6 +1488,7 @@ describe('ContextTokenization', function() { assert.deepEqual(results.alignment.merges, []); assert.deepEqual(results.alignment.splits, []); assert.deepEqual(results.alignment.unmappedEdits, []); + assert.equal(results.alignment.removedTokenCount, 0); }); it('properly aligns tokenization of transforms that match-replace existing tokens (2)', () => { @@ -1500,6 +1516,7 @@ describe('ContextTokenization', function() { assert.deepEqual(results.alignment.merges, []); assert.deepEqual(results.alignment.splits, []); assert.deepEqual(results.alignment.unmappedEdits, []); + assert.equal(results.alignment.removedTokenCount, 0); }); it('properly places extra whitespaces on preceding whitespace token', () => { @@ -1525,6 +1542,7 @@ describe('ContextTokenization', function() { assert.deepEqual(results.alignment.merges, []); assert.deepEqual(results.alignment.splits, []); assert.deepEqual(results.alignment.unmappedEdits, []); + assert.equal(results.alignment.removedTokenCount, 0); }); it('properly aligns degenerate input cases (1)', () => { @@ -1555,6 +1573,7 @@ describe('ContextTokenization', function() { assert.deepEqual(results.alignment.merges, []); assert.deepEqual(results.alignment.splits, []); assert.deepEqual(results.alignment.unmappedEdits, []); + assert.equal(results.alignment.removedTokenCount, 0); }); it('returns the standard edge window for empty transform inputs', () => { @@ -1760,6 +1779,7 @@ describe('ContextTokenization', function() { ]); assert.deepEqual(results.alignment.splits, []); assert.deepEqual(results.alignment.unmappedEdits, []); + assert.equal(results.alignment.removedTokenCount, 0); }); it('properly handles English contraction transitions (2)', () => { @@ -1795,6 +1815,7 @@ describe('ContextTokenization', function() { } ]); assert.deepEqual(results.alignment.unmappedEdits, []); + assert.equal(results.alignment.removedTokenCount, 0); }); it('properly handles English contraction transitions (3)', () => { @@ -1827,6 +1848,7 @@ describe('ContextTokenization', function() { } ]); assert.deepEqual(results.alignment.unmappedEdits, []); + assert.equal(results.alignment.removedTokenCount, 0); }); }); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts index cddda6242a..7ebccf7636 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts @@ -49,7 +49,8 @@ describe('precomputationSubsetKeyer', function() { false ), retokenization: [...rawTextTokens] - } + }, + removedTokenCount: 0 }, tokenizedTransform: (() => { const map = new Map(); @@ -78,7 +79,8 @@ describe('precomputationSubsetKeyer', function() { false ), retokenization: [...rawTextTokens] - } + }, + removedTokenCount: 0 }, tokenizedTransform: (() => { const map = new Map(); @@ -124,7 +126,8 @@ describe('precomputationSubsetKeyer', function() { false ), retokenization: [...rawTextTokens] - } + }, + removedTokenCount: 0 }, tokenizedTransform: (() => { const map = new Map(); @@ -179,7 +182,8 @@ describe('precomputationSubsetKeyer', function() { false ), retokenization: [...rawTextTokens] - } + }, + removedTokenCount: 0 }, tokenizedTransform: (() => { const map = new Map(); @@ -246,7 +250,8 @@ describe('precomputationSubsetKeyer', function() { false ), retokenization: [...rawTextTokens] - } + }, + removedTokenCount: 0 }, tokenizedTransform: (() => { const map = new Map(); @@ -321,7 +326,8 @@ describe('precomputationSubsetKeyer', function() { }], splits: [], unmappedEdits: [], - edgeWindow: edgeWindow1 + edgeWindow: edgeWindow1, + removedTokenCount: 0 }, tokenizedTransform: (() => { const map = new Map(); @@ -363,7 +369,8 @@ describe('precomputationSubsetKeyer', function() { }], splits: [], unmappedEdits: [], - edgeWindow: edgeWindow1 + edgeWindow: edgeWindow1, + removedTokenCount: 0 }, tokenizedTransform: (() => { const map = new Map(); @@ -427,7 +434,8 @@ describe('precomputationSubsetKeyer', function() { false ), retokenization: [...rawTextTokens] - } + }, + removedTokenCount: 0 }, tokenizedTransform: (() => { const map = new Map(); @@ -479,7 +487,8 @@ describe('precomputationSubsetKeyer', function() { false ), retokenization: [...rawTextTokens] - } + }, + removedTokenCount: 0 }, tokenizedTransform: (() => { const map = new Map(); @@ -529,7 +538,8 @@ describe('precomputationSubsetKeyer', function() { false ), retokenization: [...rawTextTokens] - } + }, + removedTokenCount: 0 }, tokenizedTransform: (() => { const map = new Map();