erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-08-31 04:37:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 10

Merge branch 'feat/web/tokenization-subset-builder' into feat/web/split-context-tokens

Browse source
This commit is contained in:
Joshua Horton 2025-10-08 08:45:19 -05:00
commit 2b290955a4
3 changed files with 47 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
View file

@ -314,7 +314,7 @@ export class ContextTokenization {
* @param edgeOptions
* @returns
*/
precomputeTokenizationAfterInput(
mapWhitespacedTokenization(
lexicalModel: LexicalModel,
transform: Transform,
edgeOptions?: EdgeWindowOptions
@ -324,7 +324,7 @@ export class ContextTokenization {
//
// Context does not slide within this function.
//
// Assertion: this alignment cannot fail; we KNOW there's a solid
// Assumption: this alignment cannot fail; we KNOW there's a solid
// before-and-after relationship here, and we can base it on the results of
// a prior syncToSourceWindow call.
//

74
web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts
View file

@ -983,7 +983,7 @@ describe('ContextTokenization', function() {
});
});
describe('precomputeTokenizationAfterInput', () => {
describe('mapWhitespacedTokenization', () => {
const edgeWindowSpec = {
minTokens: 3,
minChars: 8
@ -999,7 +999,7 @@ describe('ContextTokenization', function() {
deleteLeft: 0
};
const results = baseTokenization.precomputeTokenizationAfterInput(
const results = baseTokenization.mapWhitespacedTokenization(
plainModel,
editTransform
);
@ -1021,7 +1021,7 @@ describe('ContextTokenization', function() {
deleteLeft: 0
};
const results = baseTokenization.precomputeTokenizationAfterInput(
const results = baseTokenization.mapWhitespacedTokenization(
plainModel,
editTransform
);
@ -1042,7 +1042,7 @@ describe('ContextTokenization', function() {
deleteLeft: 2
};
const results = baseTokenization.precomputeTokenizationAfterInput(
const results = baseTokenization.mapWhitespacedTokenization(
plainModel,
editTransform,
edgeWindowSpec
@ -1079,7 +1079,7 @@ describe('ContextTokenization', function() {
deleteLeft: 2
};
const results = baseTokenization.precomputeTokenizationAfterInput(
const results = baseTokenization.mapWhitespacedTokenization(
plainModel,
editTransform,
edgeWindowSpec
@ -1115,7 +1115,7 @@ describe('ContextTokenization', function() {
deleteLeft: 4
};
const results = baseTokenization.precomputeTokenizationAfterInput(
const results = baseTokenization.mapWhitespacedTokenization(
plainModel,
editTransform
);
@ -1137,7 +1137,7 @@ describe('ContextTokenization', function() {
deleteLeft: 5
};
const results = baseTokenization.precomputeTokenizationAfterInput(
const results = baseTokenization.mapWhitespacedTokenization(
plainModel,
editTransform
);
@ -1172,7 +1172,7 @@ describe('ContextTokenization', function() {
deleteLeft: 0
};
const results = baseTokenization.precomputeTokenizationAfterInput(
const results = baseTokenization.mapWhitespacedTokenization(
plainModel,
editTransform
);
@ -1200,7 +1200,7 @@ describe('ContextTokenization', function() {
deleteLeft: 0
};
const results = baseTokenization.precomputeTokenizationAfterInput(
const results = baseTokenization.mapWhitespacedTokenization(
plainModel,
editTransform
);
@ -1225,7 +1225,7 @@ describe('ContextTokenization', function() {
deleteLeft: 1
};
const results = baseTokenization.precomputeTokenizationAfterInput(
const results = baseTokenization.mapWhitespacedTokenization(
plainModel,
editTransform
);
@ -1250,7 +1250,7 @@ describe('ContextTokenization', function() {
deleteLeft: 1
};
const results = baseTokenization.precomputeTokenizationAfterInput(
const results = baseTokenization.mapWhitespacedTokenization(
plainModel,
editTransform
);
@ -1293,7 +1293,7 @@ describe('ContextTokenization', function() {
deleteLeft: 1
};
const results = baseTokenization.precomputeTokenizationAfterInput(
const results = baseTokenization.mapWhitespacedTokenization(
plainModel,
editTransform
);
@ -1336,7 +1336,7 @@ describe('ContextTokenization', function() {
deleteLeft: 1
};
const results = baseTokenization.precomputeTokenizationAfterInput(
const results = baseTokenization.mapWhitespacedTokenization(
plainModel,
editTransform
);
@ -1363,7 +1363,7 @@ describe('ContextTokenization', function() {
deleteLeft: 5
};
const results = baseTokenization.precomputeTokenizationAfterInput(
const results = baseTokenization.mapWhitespacedTokenization(
plainModel,
editTransform
);
@ -1393,7 +1393,7 @@ describe('ContextTokenization', function() {
deleteLeft: 8
};
const results = baseTokenization.precomputeTokenizationAfterInput(
const results = baseTokenization.mapWhitespacedTokenization(
plainModel,
editTransform
);
@ -1420,7 +1420,7 @@ describe('ContextTokenization', function() {
deleteLeft: 8
};
const results = baseTokenization.precomputeTokenizationAfterInput(
const results = baseTokenization.mapWhitespacedTokenization(
plainModel,
editTransform
);
@ -1446,7 +1446,7 @@ describe('ContextTokenization', function() {
deleteLeft: 0
};
const results = baseTokenization.precomputeTokenizationAfterInput(
const results = baseTokenization.mapWhitespacedTokenization(
plainModel,
editTransform
);
@ -1470,7 +1470,7 @@ describe('ContextTokenization', function() {
deleteLeft: 9
};
const results = baseTokenization.precomputeTokenizationAfterInput(
const results = baseTokenization.mapWhitespacedTokenization(
plainModel,
editTransform,
edgeWindowSpec
@ -1503,7 +1503,7 @@ describe('ContextTokenization', function() {
const windowResults = buildEdgeWindow(baseTokenization.tokens, {...editTransform, deleteRight: 0}, false, edgeWindowSpec);
const results = baseTokenization.precomputeTokenizationAfterInput(
const results = baseTokenization.mapWhitespacedTokenization(
plainModel,
editTransform,
edgeWindowSpec
@ -1537,7 +1537,7 @@ describe('ContextTokenization', function() {
const windowResults = buildEdgeWindow(baseTokenization.tokens, {...editTransform, deleteRight: 0}, false, edgeWindowSpec);
const results = baseTokenization.precomputeTokenizationAfterInput(
const results = baseTokenization.mapWhitespacedTokenization(
plainModel,
editTransform,
edgeWindowSpec
@ -1572,7 +1572,7 @@ describe('ContextTokenization', function() {
const windowResults = buildEdgeWindow(baseTokenization.tokens, {...editTransform, deleteRight: 0}, false, edgeWindowSpec);
const results = baseTokenization.precomputeTokenizationAfterInput(
const results = baseTokenization.mapWhitespacedTokenization(
plainModel,
editTransform,
edgeWindowSpec
@ -1606,7 +1606,7 @@ describe('ContextTokenization', function() {
const windowResults = buildEdgeWindow(baseTokenization.tokens, {...editTransform, deleteRight: 0}, false, edgeWindowSpec);
const results = baseTokenization.precomputeTokenizationAfterInput(
const results = baseTokenization.mapWhitespacedTokenization(
plainModel,
editTransform,
edgeWindowSpec
@ -1641,7 +1641,7 @@ describe('ContextTokenization', function() {
const windowResults = buildEdgeWindow(baseTokenization.tokens, {...editTransform, deleteRight: 0}, false, edgeWindowSpec);
const results = baseTokenization.precomputeTokenizationAfterInput(
const results = baseTokenization.mapWhitespacedTokenization(
plainModel,
editTransform,
edgeWindowSpec
@ -1672,7 +1672,7 @@ describe('ContextTokenization', function() {
deleteLeft: 0
};
const results = baseTokenization.precomputeTokenizationAfterInput(
const results = baseTokenization.mapWhitespacedTokenization(
plainModel,
editTransform,
edgeWindowSpec
@ -1683,13 +1683,13 @@ describe('ContextTokenization', function() {
expectedMap.set(0, { insert: 't', deleteLeft: 0 });
assert.equal(results.tokenizedTransform.size, 1);
assert.deepEqual(results.tokenizedTransform, expectedMap);
assert.equal(results.alignment.edgeWindow.sliceIndex, 4);
assert.deepEqual(results.alignment.merges, [
{
inputs: [
{ text: 'can', index: 2 /* + 4 */ },
{ text: '\'', index: 3 /* + 4 */ }
], match: { text: 'can\'t', index: 2 /* + 4 */}
// The `index` values here are pre-offset from the edge window's .sliceIndex.
{ text: 'can', index: 6 - results.alignment.edgeWindow.sliceIndex },
{ text: '\'', index: 7 - results.alignment.edgeWindow.sliceIndex }
], match: { text: 'can\'t', index: 6 - results.alignment.edgeWindow.sliceIndex }
}
]);
assert.deepEqual(results.alignment.splits, []);
@ -1705,7 +1705,7 @@ describe('ContextTokenization', function() {
deleteLeft: 0
};
const results = baseTokenization.precomputeTokenizationAfterInput(
const results = baseTokenization.mapWhitespacedTokenization(
plainModel,
editTransform,
edgeWindowSpec
@ -1717,14 +1717,13 @@ describe('ContextTokenization', function() {
expectedMap.set(2, { insert: '', deleteLeft: 0 });
assert.equal(results.tokenizedTransform.size, 2);
assert.deepEqual(results.tokenizedTransform, expectedMap);
assert.equal(results.alignment.edgeWindow.sliceIndex, 4);
assert.deepEqual(results.alignment.merges, []);
assert.deepEqual(results.alignment.splits, [
{
input: { text: 'can\'', index: 2 /* + 4 */},
input: { text: 'can\'', index: 6 - results.alignment.edgeWindow.sliceIndex},
matches: [
{ text: 'can', index: 2 /* + 4 */, textOffset: 0 },
{ text: '\'', index: 3 /* + 4 */, textOffset: 3 }
{ text: 'can', index: 6 - results.alignment.edgeWindow.sliceIndex, textOffset: 0 },
{ text: '\'', index: 7 - results.alignment.edgeWindow.sliceIndex, textOffset: 3 }
]
}
]);
@ -1740,7 +1739,7 @@ describe('ContextTokenization', function() {
deleteLeft: 0
};
const results = baseTokenization.precomputeTokenizationAfterInput(
const results = baseTokenization.mapWhitespacedTokenization(
plainModel,
editTransform
);
@ -1749,14 +1748,13 @@ describe('ContextTokenization', function() {
expectedMap.set(1, { insert: '?', deleteLeft: 0 });
assert.equal(results.tokenizedTransform.size, 1);
assert.deepEqual(results.tokenizedTransform, expectedMap);
assert.equal(results.alignment.edgeWindow.sliceIndex, 4);
assert.deepEqual(results.alignment.merges, []);
assert.deepEqual(results.alignment.splits, [
{
input: { text: 'can\'', index: 2 /* + 4 */},
input: { text: 'can\'', index: 6 - results.alignment.edgeWindow.sliceIndex},
matches: [
{ text: 'can', index: 2 /* + 4 */, textOffset: 0 },
{ text: '\'', index: 3 /* + 4 */, textOffset: 3 }
{ text: 'can', index: 6 - results.alignment.edgeWindow.sliceIndex, textOffset: 0 },
{ text: '\'', index: 7 - results.alignment.edgeWindow.sliceIndex, textOffset: 3 }
]
}
]);

18
web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts
View file

@ -4,8 +4,8 @@
* Created by jahorton on 2025-09-23
*
* This file contains low-level tests designed to validate the behavior of the
* of the ContextTokenization class and its integration with the lower-level
* classes that it utilizes.
* ContextTokenization class and its integration with the lower-level classes
* that it utilizes.
*/
import { assert } from 'chai';
@ -567,7 +567,7 @@ describe('TokenizationSubsetBuilder', function() {
inputChars.forEach((c) => {
const {sample: transform, p} = inputDistribution.find(s => s.sample.insert == c);
const precomputation = baseTokenization.precomputeTokenizationAfterInput(plainModel, transform);
const precomputation = baseTokenization.mapWhitespacedTokenization(plainModel, transform);
subsetBuilder.addPrecomputation(baseTokenization, precomputation, p);
});
@ -595,7 +595,7 @@ describe('TokenizationSubsetBuilder', function() {
inputChars.forEach((c) => {
const {sample: transform, p} = inputDistribution.find(s => s.sample.insert == c);
const precomputation = baseTokenization.precomputeTokenizationAfterInput(plainModel, transform);
const precomputation = baseTokenization.mapWhitespacedTokenization(plainModel, transform);
subsetBuilder.addPrecomputation(baseTokenization, precomputation, p);
});
@ -645,7 +645,7 @@ describe('TokenizationSubsetBuilder', function() {
const subsetBuilder = new TokenizationSubsetBuilder();
inputDistribution.forEach((entry) => {
const precomputation = baseTokenization.precomputeTokenizationAfterInput(plainModel, entry.sample);
const precomputation = baseTokenization.mapWhitespacedTokenization(plainModel, entry.sample);
subsetBuilder.addPrecomputation(baseTokenization, precomputation, entry.p);
});
@ -727,10 +727,10 @@ describe('TokenizationSubsetBuilder', function() {
const inputDistribution = [{sample: { insert: ' ', deleteLeft: 0 }, p: 1}];
inputDistribution.forEach((entry) => {
const precomputation1 = fourCharTokenization.precomputeTokenizationAfterInput(plainModel, entry.sample);
const precomputation1 = fourCharTokenization.mapWhitespacedTokenization(plainModel, entry.sample);
subsetBuilder.addPrecomputation(fourCharTokenization, precomputation1, entry.p);
const precomputation2 = fiveCharTokenization.precomputeTokenizationAfterInput(plainModel, entry.sample);
const precomputation2 = fiveCharTokenization.mapWhitespacedTokenization(plainModel, entry.sample);
subsetBuilder.addPrecomputation(fiveCharTokenization, precomputation2, entry.p);
});
@ -771,10 +771,10 @@ describe('TokenizationSubsetBuilder', function() {
];
inputDistribution.forEach((entry) => {
const precomputation1 = twoCharTokenization.precomputeTokenizationAfterInput(plainModel, entry.sample);
const precomputation1 = twoCharTokenization.mapWhitespacedTokenization(plainModel, entry.sample);
subsetBuilder.addPrecomputation(twoCharTokenization, precomputation1, entry.p);
const precomputation2 = threeCharTokenization.precomputeTokenizationAfterInput(plainModel, entry.sample);
const precomputation2 = threeCharTokenization.mapWhitespacedTokenization(plainModel, entry.sample);
subsetBuilder.addPrecomputation(threeCharTokenization, precomputation2, entry.p);
});