From f1b8cb245769ce4584c722464faf09fd0c4bd97e Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 30 Sep 2025 09:18:56 -0500 Subject: [PATCH 1/5] docs(web): assertion -> assumption --- .../worker-thread/src/main/correction/context-tokenization.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 6cce69bf3e..99efc04b3c 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -324,7 +324,7 @@ export class ContextTokenization { // // Context does not slide within this function. // - // Assertion: this alignment cannot fail; we KNOW there's a solid + // Assumption: this alignment cannot fail; we KNOW there's a solid // before-and-after relationship here, and we can base it on the results of // a prior syncToSourceWindow call. // From 98a40a3f646ef7dbc9a67a01f1676b1fa6fc393f Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 6 Oct 2025 16:47:51 -0500 Subject: [PATCH 2/5] change(web): clarify role of sliceIndex in token-merge, token-split tests --- .../context/context-tokenization.tests.ts | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index 0729257af9..971f11e537 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -1635,13 +1635,13 @@ describe('ContextTokenization', function() { expectedMap.set(0, { insert: 't', deleteLeft: 0 }); assert.equal(results.tokenizedTransform.size, 1); assert.deepEqual(results.tokenizedTransform, expectedMap); - assert.equal(results.alignment.edgeWindow.sliceIndex, 4); assert.deepEqual(results.alignment.merges, [ { inputs: [ - { text: 'can', index: 2 /* + 4 */ }, - { text: '\'', index: 3 /* + 4 */ } - ], match: { text: 'can\'t', index: 2 /* + 4 */} + // The `index` values here are pre-offset from the edge window's .sliceIndex. + { text: 'can', index: 6 - results.alignment.edgeWindow.sliceIndex }, + { text: '\'', index: 7 - results.alignment.edgeWindow.sliceIndex } + ], match: { text: 'can\'t', index: 6 - results.alignment.edgeWindow.sliceIndex } } ]); assert.deepEqual(results.alignment.splits, []); @@ -1669,14 +1669,13 @@ describe('ContextTokenization', function() { expectedMap.set(2, { insert: '', deleteLeft: 0 }); assert.equal(results.tokenizedTransform.size, 2); assert.deepEqual(results.tokenizedTransform, expectedMap); - assert.equal(results.alignment.edgeWindow.sliceIndex, 4); assert.deepEqual(results.alignment.merges, []); assert.deepEqual(results.alignment.splits, [ { - input: { text: 'can\'', index: 2 /* + 4 */}, + input: { text: 'can\'', index: 6 - results.alignment.edgeWindow.sliceIndex}, matches: [ - { text: 'can', index: 2 /* + 4 */, textOffset: 0 }, - { text: '\'', index: 3 /* + 4 */, textOffset: 3 } + { text: 'can', index: 6 - results.alignment.edgeWindow.sliceIndex, textOffset: 0 }, + { text: '\'', index: 7 - results.alignment.edgeWindow.sliceIndex, textOffset: 3 } ] } ]); @@ -1701,14 +1700,13 @@ describe('ContextTokenization', function() { expectedMap.set(1, { insert: '?', deleteLeft: 0 }); assert.equal(results.tokenizedTransform.size, 1); assert.deepEqual(results.tokenizedTransform, expectedMap); - assert.equal(results.alignment.edgeWindow.sliceIndex, 4); assert.deepEqual(results.alignment.merges, []); assert.deepEqual(results.alignment.splits, [ { - input: { text: 'can\'', index: 2 /* + 4 */}, + input: { text: 'can\'', index: 6 - results.alignment.edgeWindow.sliceIndex}, matches: [ - { text: 'can', index: 2 /* + 4 */, textOffset: 0 }, - { text: '\'', index: 3 /* + 4 */, textOffset: 3 } + { text: 'can', index: 6 - results.alignment.edgeWindow.sliceIndex, textOffset: 0 }, + { text: '\'', index: 7 - results.alignment.edgeWindow.sliceIndex, textOffset: 3 } ] } ]); From 01a0eeec196d62d2dfe1d793bc08145c8d5bb373 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Wed, 8 Oct 2025 08:40:15 -0500 Subject: [PATCH 3/5] change(web): renames precomputeTokenizationAfterInput to mapWhitespacedTokenization --- .../main/correction/context-tokenization.ts | 2 +- .../context/context-tokenization.tests.ts | 52 +++++++++---------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 99efc04b3c..c00ecefbc9 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -314,7 +314,7 @@ export class ContextTokenization { * @param edgeOptions * @returns */ - precomputeTokenizationAfterInput( + mapWhitespacedTokenization( lexicalModel: LexicalModel, transform: Transform, edgeOptions?: EdgeWindowOptions diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index 971f11e537..1d3cdae455 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -954,7 +954,7 @@ describe('ContextTokenization', function() { }); }); - describe('precomputeTokenizationAfterInput', () => { + describe('mapWhitespacedTokenization', () => { const edgeWindowSpec = { minTokens: 3, minChars: 8 @@ -970,7 +970,7 @@ describe('ContextTokenization', function() { deleteLeft: 0 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -992,7 +992,7 @@ describe('ContextTokenization', function() { deleteLeft: 0 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -1013,7 +1013,7 @@ describe('ContextTokenization', function() { deleteLeft: 2 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform, edgeWindowSpec @@ -1048,7 +1048,7 @@ describe('ContextTokenization', function() { deleteLeft: 2 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform, edgeWindowSpec @@ -1082,7 +1082,7 @@ describe('ContextTokenization', function() { deleteLeft: 4 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -1104,7 +1104,7 @@ describe('ContextTokenization', function() { deleteLeft: 5 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -1139,7 +1139,7 @@ describe('ContextTokenization', function() { deleteLeft: 0 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -1167,7 +1167,7 @@ describe('ContextTokenization', function() { deleteLeft: 0 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -1192,7 +1192,7 @@ describe('ContextTokenization', function() { deleteLeft: 1 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -1217,7 +1217,7 @@ describe('ContextTokenization', function() { deleteLeft: 1 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -1258,7 +1258,7 @@ describe('ContextTokenization', function() { deleteLeft: 1 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -1299,7 +1299,7 @@ describe('ContextTokenization', function() { deleteLeft: 1 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -1326,7 +1326,7 @@ describe('ContextTokenization', function() { deleteLeft: 5 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -1356,7 +1356,7 @@ describe('ContextTokenization', function() { deleteLeft: 8 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -1383,7 +1383,7 @@ describe('ContextTokenization', function() { deleteLeft: 8 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -1409,7 +1409,7 @@ describe('ContextTokenization', function() { deleteLeft: 0 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); @@ -1433,7 +1433,7 @@ describe('ContextTokenization', function() { deleteLeft: 9 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform, edgeWindowSpec @@ -1465,7 +1465,7 @@ describe('ContextTokenization', function() { const windowResults = buildEdgeWindow(baseTokenization.tokens, {...editTransform, deleteRight: 0}, false, edgeWindowSpec); - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform, edgeWindowSpec @@ -1497,7 +1497,7 @@ describe('ContextTokenization', function() { const windowResults = buildEdgeWindow(baseTokenization.tokens, {...editTransform, deleteRight: 0}, false, edgeWindowSpec); - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform, edgeWindowSpec @@ -1530,7 +1530,7 @@ describe('ContextTokenization', function() { const windowResults = buildEdgeWindow(baseTokenization.tokens, {...editTransform, deleteRight: 0}, false, edgeWindowSpec); - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform, edgeWindowSpec @@ -1562,7 +1562,7 @@ describe('ContextTokenization', function() { const windowResults = buildEdgeWindow(baseTokenization.tokens, {...editTransform, deleteRight: 0}, false, edgeWindowSpec); - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform, edgeWindowSpec @@ -1594,7 +1594,7 @@ describe('ContextTokenization', function() { const windowResults = buildEdgeWindow(baseTokenization.tokens, {...editTransform, deleteRight: 0}, false, edgeWindowSpec); - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform, edgeWindowSpec @@ -1624,7 +1624,7 @@ describe('ContextTokenization', function() { deleteLeft: 0 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform, edgeWindowSpec @@ -1657,7 +1657,7 @@ describe('ContextTokenization', function() { deleteLeft: 0 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform, edgeWindowSpec @@ -1691,7 +1691,7 @@ describe('ContextTokenization', function() { deleteLeft: 0 }; - const results = baseTokenization.precomputeTokenizationAfterInput( + const results = baseTokenization.mapWhitespacedTokenization( plainModel, editTransform ); From 8ebfab10fcbb2127cea44b24f0bbac5bd35b95a3 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Wed, 8 Oct 2025 08:43:02 -0500 Subject: [PATCH 4/5] docs(web): remove duplicate words in file header comment --- .../worker-thread/context/tokenization-subsets.tests.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts index 132117812e..a416f434ba 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts @@ -4,8 +4,8 @@ * Created by jahorton on 2025-09-23 * * This file contains low-level tests designed to validate the behavior of the - * of the ContextTokenization class and its integration with the lower-level - * classes that it utilizes. + * ContextTokenization class and its integration with the lower-level classes + * that it utilizes. */ import { assert } from 'chai'; From 97789c008b523b1f281d0643a2b2216ac3a62e1a Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Wed, 8 Oct 2025 08:44:51 -0500 Subject: [PATCH 5/5] fix(web): update unit test use of method with name changed to mapWhitespacedTokenization --- .../context/tokenization-subsets.tests.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts index 319728de4e..9707fbb3d2 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts @@ -567,7 +567,7 @@ describe('TokenizationSubsetBuilder', function() { inputChars.forEach((c) => { const {sample: transform, p} = inputDistribution.find(s => s.sample.insert == c); - const precomputation = baseTokenization.precomputeTokenizationAfterInput(plainModel, transform); + const precomputation = baseTokenization.mapWhitespacedTokenization(plainModel, transform); subsetBuilder.addPrecomputation(baseTokenization, precomputation, p); }); @@ -595,7 +595,7 @@ describe('TokenizationSubsetBuilder', function() { inputChars.forEach((c) => { const {sample: transform, p} = inputDistribution.find(s => s.sample.insert == c); - const precomputation = baseTokenization.precomputeTokenizationAfterInput(plainModel, transform); + const precomputation = baseTokenization.mapWhitespacedTokenization(plainModel, transform); subsetBuilder.addPrecomputation(baseTokenization, precomputation, p); }); @@ -645,7 +645,7 @@ describe('TokenizationSubsetBuilder', function() { const subsetBuilder = new TokenizationSubsetBuilder(); inputDistribution.forEach((entry) => { - const precomputation = baseTokenization.precomputeTokenizationAfterInput(plainModel, entry.sample); + const precomputation = baseTokenization.mapWhitespacedTokenization(plainModel, entry.sample); subsetBuilder.addPrecomputation(baseTokenization, precomputation, entry.p); }); @@ -727,10 +727,10 @@ describe('TokenizationSubsetBuilder', function() { const inputDistribution = [{sample: { insert: ' ', deleteLeft: 0 }, p: 1}]; inputDistribution.forEach((entry) => { - const precomputation1 = fourCharTokenization.precomputeTokenizationAfterInput(plainModel, entry.sample); + const precomputation1 = fourCharTokenization.mapWhitespacedTokenization(plainModel, entry.sample); subsetBuilder.addPrecomputation(fourCharTokenization, precomputation1, entry.p); - const precomputation2 = fiveCharTokenization.precomputeTokenizationAfterInput(plainModel, entry.sample); + const precomputation2 = fiveCharTokenization.mapWhitespacedTokenization(plainModel, entry.sample); subsetBuilder.addPrecomputation(fiveCharTokenization, precomputation2, entry.p); }); @@ -771,10 +771,10 @@ describe('TokenizationSubsetBuilder', function() { ]; inputDistribution.forEach((entry) => { - const precomputation1 = twoCharTokenization.precomputeTokenizationAfterInput(plainModel, entry.sample); + const precomputation1 = twoCharTokenization.mapWhitespacedTokenization(plainModel, entry.sample); subsetBuilder.addPrecomputation(twoCharTokenization, precomputation1, entry.p); - const precomputation2 = threeCharTokenization.precomputeTokenizationAfterInput(plainModel, entry.sample); + const precomputation2 = threeCharTokenization.mapWhitespacedTokenization(plainModel, entry.sample); subsetBuilder.addPrecomputation(threeCharTokenization, precomputation2, entry.p); });