erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-08-31 20:57:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 15

change(web): clarify role of sliceIndex in token-merge, token-split tests

Browse source
This commit is contained in:
Joshua Horton 2025-10-06 16:47:51 -05:00
parent f1b8cb2457
commit 98a40a3f64
1 changed files with 10 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts
View file

@ -1635,13 +1635,13 @@ describe('ContextTokenization', function() {
expectedMap.set(0, { insert: 't', deleteLeft: 0 });
assert.equal(results.tokenizedTransform.size, 1);
assert.deepEqual(results.tokenizedTransform, expectedMap);
assert.equal(results.alignment.edgeWindow.sliceIndex, 4);
assert.deepEqual(results.alignment.merges, [
{
inputs: [
{ text: 'can', index: 2 /* + 4 */ },
{ text: '\'', index: 3 /* + 4 */ }
], match: { text: 'can\'t', index: 2 /* + 4 */}
// The `index` values here are pre-offset from the edge window's .sliceIndex.
{ text: 'can', index: 6 - results.alignment.edgeWindow.sliceIndex },
{ text: '\'', index: 7 - results.alignment.edgeWindow.sliceIndex }
], match: { text: 'can\'t', index: 6 - results.alignment.edgeWindow.sliceIndex }
}
]);
assert.deepEqual(results.alignment.splits, []);
@ -1669,14 +1669,13 @@ describe('ContextTokenization', function() {
expectedMap.set(2, { insert: '', deleteLeft: 0 });
assert.equal(results.tokenizedTransform.size, 2);
assert.deepEqual(results.tokenizedTransform, expectedMap);
assert.equal(results.alignment.edgeWindow.sliceIndex, 4);
assert.deepEqual(results.alignment.merges, []);
assert.deepEqual(results.alignment.splits, [
{
input: { text: 'can\'', index: 2 /* + 4 */},
input: { text: 'can\'', index: 6 - results.alignment.edgeWindow.sliceIndex},
matches: [
{ text: 'can', index: 2 /* + 4 */, textOffset: 0 },
{ text: '\'', index: 3 /* + 4 */, textOffset: 3 }
{ text: 'can', index: 6 - results.alignment.edgeWindow.sliceIndex, textOffset: 0 },
{ text: '\'', index: 7 - results.alignment.edgeWindow.sliceIndex, textOffset: 3 }
]
}
]);
@ -1701,14 +1700,13 @@ describe('ContextTokenization', function() {
expectedMap.set(1, { insert: '?', deleteLeft: 0 });
assert.equal(results.tokenizedTransform.size, 1);
assert.deepEqual(results.tokenizedTransform, expectedMap);
assert.equal(results.alignment.edgeWindow.sliceIndex, 4);
assert.deepEqual(results.alignment.merges, []);
assert.deepEqual(results.alignment.splits, [
{
input: { text: 'can\'', index: 2 /* + 4 */},
input: { text: 'can\'', index: 6 - results.alignment.edgeWindow.sliceIndex},
matches: [
{ text: 'can', index: 2 /* + 4 */, textOffset: 0 },
{ text: '\'', index: 3 /* + 4 */, textOffset: 3 }
{ text: 'can', index: 6 - results.alignment.edgeWindow.sliceIndex, textOffset: 0 },
{ text: '\'', index: 7 - results.alignment.edgeWindow.sliceIndex, textOffset: 3 }
]
}
]);