diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index a3859f2c7b..5501a713c7 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -314,7 +314,7 @@ export class ContextTokenization { * @param edgeOptions * @returns */ - precomputeTokenizationAfterInput( + mapWhitespacedTokenization( lexicalModel: LexicalModel, transform: Transform, edgeOptions?: EdgeWindowOptions @@ -324,7 +324,7 @@ export class ContextTokenization { // // Context does not slide within this function. // - // Assertion: this alignment cannot fail; we KNOW there's a solid + // Assumption: this alignment cannot fail; we KNOW there's a solid // before-and-after relationship here, and we can base it on the results of // a prior syncToSourceWindow call. // diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index 4be0f15730..74c104c655 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -983,7 +983,7 @@ describe('ContextTokenization', function() { }); }); - describe('precomputeTokenizationAfterInput', () => { + describe('mapWhitespacedTokenization', () => { const edgeWindowSpec = { minTokens: 3, minChars: 8 @@ -999,7 +999,7 @@ describe('ContextTokenization', function() { deleteLeft: 0 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -1021,7 +1021,7 @@ describe('ContextTokenization', function() { deleteLeft: 0 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -1042,7 +1042,7 @@ describe('ContextTokenization', function() { deleteLeft: 2 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform, edgeWindowSpec @@ -1079,7 +1079,7 @@ describe('ContextTokenization', function() { deleteLeft: 2 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform, edgeWindowSpec @@ -1115,7 +1115,7 @@ describe('ContextTokenization', function() { deleteLeft: 4 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -1137,7 +1137,7 @@ describe('ContextTokenization', function() { deleteLeft: 5 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -1172,7 +1172,7 @@ describe('ContextTokenization', function() { deleteLeft: 0 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -1200,7 +1200,7 @@ describe('ContextTokenization', function() { deleteLeft: 0 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -1225,7 +1225,7 @@ describe('ContextTokenization', function() { deleteLeft: 1 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -1250,7 +1250,7 @@ describe('ContextTokenization', function() { deleteLeft: 1 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -1293,7 +1293,7 @@ describe('ContextTokenization', function() { deleteLeft: 1 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -1336,7 +1336,7 @@ describe('ContextTokenization', function() { deleteLeft: 1 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -1363,7 +1363,7 @@ describe('ContextTokenization', function() { deleteLeft: 5 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -1393,7 +1393,7 @@ describe('ContextTokenization', function() { deleteLeft: 8 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -1420,7 +1420,7 @@ describe('ContextTokenization', function() { deleteLeft: 8 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -1446,7 +1446,7 @@ describe('ContextTokenization', function() { deleteLeft: 0 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -1470,7 +1470,7 @@ describe('ContextTokenization', function() { deleteLeft: 9 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform, edgeWindowSpec @@ -1503,7 +1503,7 @@ describe('ContextTokenization', function() { const windowResults = buildEdgeWindow(baseTokenization.tokens, {...editTransform, deleteRight: 0}, false, edgeWindowSpec); - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform, edgeWindowSpec @@ -1537,7 +1537,7 @@ describe('ContextTokenization', function() { const windowResults = buildEdgeWindow(baseTokenization.tokens, {...editTransform, deleteRight: 0}, false, edgeWindowSpec); - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform, edgeWindowSpec @@ -1572,7 +1572,7 @@ describe('ContextTokenization', function() { const windowResults = buildEdgeWindow(baseTokenization.tokens, {...editTransform, deleteRight: 0}, false, edgeWindowSpec); - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform, edgeWindowSpec @@ -1606,7 +1606,7 @@ describe('ContextTokenization', function() { const windowResults = buildEdgeWindow(baseTokenization.tokens, {...editTransform, deleteRight: 0}, false, edgeWindowSpec); - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform, edgeWindowSpec @@ -1641,7 +1641,7 @@ describe('ContextTokenization', function() { const windowResults = buildEdgeWindow(baseTokenization.tokens, {...editTransform, deleteRight: 0}, false, edgeWindowSpec); - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform, edgeWindowSpec @@ -1672,7 +1672,7 @@ describe('ContextTokenization', function() { deleteLeft: 0 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform, edgeWindowSpec @@ -1683,13 +1683,13 @@ describe('ContextTokenization', function() { expectedMap.set(0, { insert: 't', deleteLeft: 0 }); assert.equal(results.tokenizedTransform.size, 1); assert.deepEqual(results.tokenizedTransform, expectedMap); - assert.equal(results.alignment.edgeWindow.sliceIndex, 4); assert.deepEqual(results.alignment.merges, [ { inputs: [ - { text: 'can', index: 2 /* + 4 */ }, - { text: '\'', index: 3 /* + 4 */ } - ], match: { text: 'can\'t', index: 2 /* + 4 */} + // The `index` values here are pre-offset from the edge window's .sliceIndex. + { text: 'can', index: 6 - results.alignment.edgeWindow.sliceIndex }, + { text: '\'', index: 7 - results.alignment.edgeWindow.sliceIndex } + ], match: { text: 'can\'t', index: 6 - results.alignment.edgeWindow.sliceIndex } } ]); assert.deepEqual(results.alignment.splits, []); @@ -1705,7 +1705,7 @@ describe('ContextTokenization', function() { deleteLeft: 0 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform, edgeWindowSpec @@ -1717,14 +1717,13 @@ describe('ContextTokenization', function() { expectedMap.set(2, { insert: '', deleteLeft: 0 }); assert.equal(results.tokenizedTransform.size, 2); assert.deepEqual(results.tokenizedTransform, expectedMap); - assert.equal(results.alignment.edgeWindow.sliceIndex, 4); assert.deepEqual(results.alignment.merges, []); assert.deepEqual(results.alignment.splits, [ { - input: { text: 'can\'', index: 2 /* + 4 */}, + input: { text: 'can\'', index: 6 - results.alignment.edgeWindow.sliceIndex}, matches: [ - { text: 'can', index: 2 /* + 4 */, textOffset: 0 }, - { text: '\'', index: 3 /* + 4 */, textOffset: 3 } + { text: 'can', index: 6 - results.alignment.edgeWindow.sliceIndex, textOffset: 0 }, + { text: '\'', index: 7 - results.alignment.edgeWindow.sliceIndex, textOffset: 3 } ] } ]); @@ -1740,7 +1739,7 @@ describe('ContextTokenization', function() { deleteLeft: 0 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -1749,14 +1748,13 @@ describe('ContextTokenization', function() { expectedMap.set(1, { insert: '?', deleteLeft: 0 }); assert.equal(results.tokenizedTransform.size, 1); assert.deepEqual(results.tokenizedTransform, expectedMap); - assert.equal(results.alignment.edgeWindow.sliceIndex, 4); assert.deepEqual(results.alignment.merges, []); assert.deepEqual(results.alignment.splits, [ { - input: { text: 'can\'', index: 2 /* + 4 */}, + input: { text: 'can\'', index: 6 - results.alignment.edgeWindow.sliceIndex}, matches: [ - { text: 'can', index: 2 /* + 4 */, textOffset: 0 }, - { text: '\'', index: 3 /* + 4 */, textOffset: 3 } + { text: 'can', index: 6 - results.alignment.edgeWindow.sliceIndex, textOffset: 0 }, + { text: '\'', index: 7 - results.alignment.edgeWindow.sliceIndex, textOffset: 3 } ] } ]); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts index cddda6242a..9707fbb3d2 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts @@ -4,8 +4,8 @@ * Created by jahorton on 2025-09-23 * * This file contains low-level tests designed to validate the behavior of the - * of the ContextTokenization class and its integration with the lower-level - * classes that it utilizes. + * ContextTokenization class and its integration with the lower-level classes + * that it utilizes. */ import { assert } from 'chai'; @@ -567,7 +567,7 @@ describe('TokenizationSubsetBuilder', function() { inputChars.forEach((c) => { const {sample: transform, p} = inputDistribution.find(s => s.sample.insert == c); - const precomputation = baseTokenization.precomputeTokenizationAfterInput(plainModel, transform); + const precomputation = baseTokenization.mapWhitespacedTokenization(plainModel, transform); subsetBuilder.addPrecomputation(baseTokenization, precomputation, p); }); @@ -595,7 +595,7 @@ describe('TokenizationSubsetBuilder', function() { inputChars.forEach((c) => { const {sample: transform, p} = inputDistribution.find(s => s.sample.insert == c); - const precomputation = baseTokenization.precomputeTokenizationAfterInput(plainModel, transform); + const precomputation = baseTokenization.mapWhitespacedTokenization(plainModel, transform); subsetBuilder.addPrecomputation(baseTokenization, precomputation, p); }); @@ -645,7 +645,7 @@ describe('TokenizationSubsetBuilder', function() { const subsetBuilder = new TokenizationSubsetBuilder(); inputDistribution.forEach((entry) => { - const precomputation = baseTokenization.precomputeTokenizationAfterInput(plainModel, entry.sample); + const precomputation = baseTokenization.mapWhitespacedTokenization(plainModel, entry.sample); subsetBuilder.addPrecomputation(baseTokenization, precomputation, entry.p); }); @@ -727,10 +727,10 @@ describe('TokenizationSubsetBuilder', function() { const inputDistribution = [{sample: { insert: ' ', deleteLeft: 0 }, p: 1}]; inputDistribution.forEach((entry) => { - const precomputation1 = fourCharTokenization.precomputeTokenizationAfterInput(plainModel, entry.sample); + const precomputation1 = fourCharTokenization.mapWhitespacedTokenization(plainModel, entry.sample); subsetBuilder.addPrecomputation(fourCharTokenization, precomputation1, entry.p); - const precomputation2 = fiveCharTokenization.precomputeTokenizationAfterInput(plainModel, entry.sample); + const precomputation2 = fiveCharTokenization.mapWhitespacedTokenization(plainModel, entry.sample); subsetBuilder.addPrecomputation(fiveCharTokenization, precomputation2, entry.p); }); @@ -771,10 +771,10 @@ describe('TokenizationSubsetBuilder', function() { ]; inputDistribution.forEach((entry) => { - const precomputation1 = twoCharTokenization.precomputeTokenizationAfterInput(plainModel, entry.sample); + const precomputation1 = twoCharTokenization.mapWhitespacedTokenization(plainModel, entry.sample); subsetBuilder.addPrecomputation(twoCharTokenization, precomputation1, entry.p); - const precomputation2 = threeCharTokenization.precomputeTokenizationAfterInput(plainModel, entry.sample); + const precomputation2 = threeCharTokenization.mapWhitespacedTokenization(plainModel, entry.sample); subsetBuilder.addPrecomputation(threeCharTokenization, precomputation2, entry.p); });