From 98a40a3f646ef7dbc9a67a01f1676b1fa6fc393f Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 6 Oct 2025 16:47:51 -0500 Subject: [PATCH] change(web): clarify role of sliceIndex in token-merge, token-split tests --- .../context/context-tokenization.tests.ts | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index 0729257af9..971f11e537 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -1635,13 +1635,13 @@ describe('ContextTokenization', function() { expectedMap.set(0, { insert: 't', deleteLeft: 0 }); assert.equal(results.tokenizedTransform.size, 1); assert.deepEqual(results.tokenizedTransform, expectedMap); - assert.equal(results.alignment.edgeWindow.sliceIndex, 4); assert.deepEqual(results.alignment.merges, [ { inputs: [ - { text: 'can', index: 2 /* + 4 */ }, - { text: '\'', index: 3 /* + 4 */ } - ], match: { text: 'can\'t', index: 2 /* + 4 */} + // The `index` values here are pre-offset from the edge window's .sliceIndex. + { text: 'can', index: 6 - results.alignment.edgeWindow.sliceIndex }, + { text: '\'', index: 7 - results.alignment.edgeWindow.sliceIndex } + ], match: { text: 'can\'t', index: 6 - results.alignment.edgeWindow.sliceIndex } } ]); assert.deepEqual(results.alignment.splits, []); @@ -1669,14 +1669,13 @@ describe('ContextTokenization', function() { expectedMap.set(2, { insert: '', deleteLeft: 0 }); assert.equal(results.tokenizedTransform.size, 2); assert.deepEqual(results.tokenizedTransform, expectedMap); - assert.equal(results.alignment.edgeWindow.sliceIndex, 4); assert.deepEqual(results.alignment.merges, []); assert.deepEqual(results.alignment.splits, [ { - input: { text: 'can\'', index: 2 /* + 4 */}, + input: { text: 'can\'', index: 6 - results.alignment.edgeWindow.sliceIndex}, matches: [ - { text: 'can', index: 2 /* + 4 */, textOffset: 0 }, - { text: '\'', index: 3 /* + 4 */, textOffset: 3 } + { text: 'can', index: 6 - results.alignment.edgeWindow.sliceIndex, textOffset: 0 }, + { text: '\'', index: 7 - results.alignment.edgeWindow.sliceIndex, textOffset: 3 } ] } ]); @@ -1701,14 +1700,13 @@ describe('ContextTokenization', function() { expectedMap.set(1, { insert: '?', deleteLeft: 0 }); assert.equal(results.tokenizedTransform.size, 1); assert.deepEqual(results.tokenizedTransform, expectedMap); - assert.equal(results.alignment.edgeWindow.sliceIndex, 4); assert.deepEqual(results.alignment.merges, []); assert.deepEqual(results.alignment.splits, [ { - input: { text: 'can\'', index: 2 /* + 4 */}, + input: { text: 'can\'', index: 6 - results.alignment.edgeWindow.sliceIndex}, matches: [ - { text: 'can', index: 2 /* + 4 */, textOffset: 0 }, - { text: '\'', index: 3 /* + 4 */, textOffset: 3 } + { text: 'can', index: 6 - results.alignment.edgeWindow.sliceIndex, textOffset: 0 }, + { text: '\'', index: 7 - results.alignment.edgeWindow.sliceIndex, textOffset: 3 } ] } ]);