erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-10-27 00:07:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 15

feat(web): compute number of tokens deleted in tokenization transition

Browse source 
Build-bot: skip build:web
Test-bot: skip
This commit is contained in:
Joshua Horton 2025-10-01 09:57:57 -05:00
parent f9b80db5e3
commit b7a1f8d73b
3 changed files with 59 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
View file

@ -68,7 +68,12 @@ export interface TokenizationEdgeAlignment {
* The edge window defining the potential range of tokenization adjustments
* needed after applying the Transform
*/
edgeWindow: RetokenizedEdgeWindow
edgeWindow: RetokenizedEdgeWindow;
/**
* The number of tokens deleted from the end, without being erased.
*/
removedTokenCount: number;
}
/**
@ -430,8 +435,17 @@ export class ContextTokenization {
// accordingly.
const tailIndex = 0 - (stackedDeletes.length - 1) + (editBoundary.omitsEmptyToken ? -1 : 0);
// Mutates stackedInserts, stackedDeletes.
const baseRemovedTokenCount = Math.max(0, stackedDeletes.length - stackedInserts.length);
const transformMap = assembleTransforms(stackedInserts, stackedDeletes, tailIndex);
// If there's an empty transform in the 0 position and we already know we're
// dropping tokens - and only deleting - we're dropping an
// otherwise-untracked empty token - make sure it's included!
const droppedFinalTransform = baseRemovedTokenCount > 0 && transform.insert == '' && TransformUtils.isEmpty(transformMap.get(0));
// Past that, if we have more delete entries than insert entries for our transforms, we
// dropped some tokens outright.
const removedTokenCount = baseRemovedTokenCount + (droppedFinalTransform ? 1 : 0);
// Final step: check for any unexpected boundary shifts not mappable to 'merge' / 'split'
// and not caused by transforms. All transforms always apply in sequence at the end.
const unmappedEdits: EditTuple<EditOperation>[] = [];
@ -457,7 +471,8 @@ export class ContextTokenization {
edgeWindow: {...edgeWindow, retokenization: retokenizedEdge},
merges,
splits,
unmappedEdits
unmappedEdits,
removedTokenCount
},
tokenizedTransform: transformMap,
};

22
web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts
View file

@ -1010,6 +1010,7 @@ describe('ContextTokenization', function() {
assert.deepEqual(results.alignment.merges, []);
assert.deepEqual(results.alignment.splits, []);
assert.deepEqual(results.alignment.unmappedEdits, []);
assert.equal(results.alignment.removedTokenCount, 0);
});
it('properly handles basic token-edit transform', () => {
@ -1031,6 +1032,7 @@ describe('ContextTokenization', function() {
assert.deepEqual(results.alignment.merges, []);
assert.deepEqual(results.alignment.splits, []);
assert.deepEqual(results.alignment.unmappedEdits, []);
assert.equal(results.alignment.removedTokenCount, 0);
});
it('properly handles basic char-delete transform', () => {
@ -1052,6 +1054,7 @@ describe('ContextTokenization', function() {
assert.deepEqual(results.alignment.merges, []);
assert.deepEqual(results.alignment.splits, []);
assert.deepEqual(results.alignment.unmappedEdits, []);
assert.equal(results.alignment.removedTokenCount, 0);
});
it('properly handles simple token-edit transform', () => {
@ -1074,6 +1077,7 @@ describe('ContextTokenization', function() {
assert.deepEqual(results.alignment.merges, []);
assert.deepEqual(results.alignment.splits, []);
assert.deepEqual(results.alignment.unmappedEdits, []);
assert.equal(results.alignment.removedTokenCount, 0);
assert.deepEqual(results.alignment.edgeWindow, {
retokenization: ['apple', ' ', 'a', ' ', 'da'],
@ -1111,6 +1115,7 @@ describe('ContextTokenization', function() {
assert.deepEqual(results.alignment.merges, []);
assert.deepEqual(results.alignment.splits, []);
assert.deepEqual(results.alignment.unmappedEdits, []);
assert.equal(results.alignment.removedTokenCount, 0);
assert.deepEqual(results.alignment.edgeWindow, {
retokenization: ['apple', ' ', 'a', ' ', 'da'].map(t => toMathematicalSMP(t)),
@ -1146,6 +1151,7 @@ describe('ContextTokenization', function() {
assert.deepEqual(results.alignment.merges, []);
assert.deepEqual(results.alignment.splits, []);
assert.deepEqual(results.alignment.unmappedEdits, []);
assert.equal(results.alignment.removedTokenCount, 0);
});
it('handles simple token-replacing transform with cross-token deleteLeft', () => {
@ -1182,6 +1188,7 @@ describe('ContextTokenization', function() {
assert.deepEqual(results.alignment.merges, []);
assert.deepEqual(results.alignment.splits, []);
assert.deepEqual(results.alignment.unmappedEdits, []);
assert.equal(results.alignment.removedTokenCount, 2);
});
it('handles token-replacing transform with cross-token deleteLeft after whitespace', () => {
@ -1226,6 +1233,7 @@ describe('ContextTokenization', function() {
assert.deepEqual(results.alignment.merges, []);
assert.deepEqual(results.alignment.splits, []);
assert.deepEqual(results.alignment.unmappedEdits, []);
assert.equal(results.alignment.removedTokenCount, 4);
});
it('properly handles a simple appended whitespace', () => {
@ -1254,6 +1262,7 @@ describe('ContextTokenization', function() {
assert.deepEqual(results.alignment.merges, []);
assert.deepEqual(results.alignment.splits, []);
assert.deepEqual(results.alignment.unmappedEdits, []);
assert.equal(results.alignment.removedTokenCount, 0);
});
it('properly handles a simple appended period', () => {
@ -1279,6 +1288,7 @@ describe('ContextTokenization', function() {
assert.deepEqual(results.alignment.merges, []);
assert.deepEqual(results.alignment.splits, []);
assert.deepEqual(results.alignment.unmappedEdits, []);
assert.equal(results.alignment.removedTokenCount, 0);
});
it('properly deletes a simple appended whitespace', () => {
@ -1305,6 +1315,7 @@ describe('ContextTokenization', function() {
assert.deepEqual(results.alignment.merges, []);
assert.deepEqual(results.alignment.splits, []);
assert.deepEqual(results.alignment.unmappedEdits, []);
assert.equal(results.alignment.removedTokenCount, 2);
});
it('handles word-breakable transforms (case 1)', () => {
@ -1333,6 +1344,7 @@ describe('ContextTokenization', function() {
assert.deepEqual(results.alignment.merges, []);
assert.deepEqual(results.alignment.splits, []);
assert.deepEqual(results.alignment.unmappedEdits, []);
assert.equal(results.alignment.removedTokenCount, 0);
assert.deepEqual(results.alignment.edgeWindow, {
retokenization: ['apple', ' ', 'a', ' ', 'da'],
@ -1376,6 +1388,7 @@ describe('ContextTokenization', function() {
assert.deepEqual(results.alignment.merges, []);
assert.deepEqual(results.alignment.splits, []);
assert.deepEqual(results.alignment.unmappedEdits, []);
assert.equal(results.alignment.removedTokenCount, 0);
assert.deepEqual(results.alignment.edgeWindow, {
retokenization: ['apple', ' ', 'a', ' ', 'da'].map(t => toMathematicalSMP(t)),
@ -1417,6 +1430,7 @@ describe('ContextTokenization', function() {
assert.deepEqual(results.alignment.merges, []);
assert.deepEqual(results.alignment.splits, []);
assert.deepEqual(results.alignment.unmappedEdits, []);
assert.equal(results.alignment.removedTokenCount, 0);
});
it('handles complex breakable cases', () => {
@ -1446,6 +1460,7 @@ describe('ContextTokenization', function() {
assert.deepEqual(results.alignment.merges, []);
assert.deepEqual(results.alignment.splits, []);
assert.deepEqual(results.alignment.unmappedEdits, []);
assert.equal(results.alignment.removedTokenCount, 0);
});
it('properly aligns tokenization of transforms that match-replace existing tokens (1)', () => {
@ -1473,6 +1488,7 @@ describe('ContextTokenization', function() {
assert.deepEqual(results.alignment.merges, []);
assert.deepEqual(results.alignment.splits, []);
assert.deepEqual(results.alignment.unmappedEdits, []);
assert.equal(results.alignment.removedTokenCount, 0);
});
it('properly aligns tokenization of transforms that match-replace existing tokens (2)', () => {
@ -1500,6 +1516,7 @@ describe('ContextTokenization', function() {
assert.deepEqual(results.alignment.merges, []);
assert.deepEqual(results.alignment.splits, []);
assert.deepEqual(results.alignment.unmappedEdits, []);
assert.equal(results.alignment.removedTokenCount, 0);
});
it('properly places extra whitespaces on preceding whitespace token', () => {
@ -1525,6 +1542,7 @@ describe('ContextTokenization', function() {
assert.deepEqual(results.alignment.merges, []);
assert.deepEqual(results.alignment.splits, []);
assert.deepEqual(results.alignment.unmappedEdits, []);
assert.equal(results.alignment.removedTokenCount, 0);
});
it('properly aligns degenerate input cases (1)', () => {
@ -1555,6 +1573,7 @@ describe('ContextTokenization', function() {
assert.deepEqual(results.alignment.merges, []);
assert.deepEqual(results.alignment.splits, []);
assert.deepEqual(results.alignment.unmappedEdits, []);
assert.equal(results.alignment.removedTokenCount, 0);
});
it('returns the standard edge window for empty transform inputs', () => {
@ -1760,6 +1779,7 @@ describe('ContextTokenization', function() {
]);
assert.deepEqual(results.alignment.splits, []);
assert.deepEqual(results.alignment.unmappedEdits, []);
assert.equal(results.alignment.removedTokenCount, 0);
});
it('properly handles English contraction transitions (2)', () => {
@ -1795,6 +1815,7 @@ describe('ContextTokenization', function() {
}
]);
assert.deepEqual(results.alignment.unmappedEdits, []);
assert.equal(results.alignment.removedTokenCount, 0);
});
it('properly handles English contraction transitions (3)', () => {
@ -1827,6 +1848,7 @@ describe('ContextTokenization', function() {
}
]);
assert.deepEqual(results.alignment.unmappedEdits, []);
assert.equal(results.alignment.removedTokenCount, 0);
});
});

30
web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts
View file

@ -49,7 +49,8 @@ describe('precomputationSubsetKeyer', function() {
false
),
retokenization: [...rawTextTokens]
}
},
removedTokenCount: 0
},
tokenizedTransform: (() => {
const map = new Map<number, Transform>();
@ -78,7 +79,8 @@ describe('precomputationSubsetKeyer', function() {
false
),
retokenization: [...rawTextTokens]
}
},
removedTokenCount: 0
},
tokenizedTransform: (() => {
const map = new Map<number, Transform>();
@ -124,7 +126,8 @@ describe('precomputationSubsetKeyer', function() {
false
),
retokenization: [...rawTextTokens]
}
},
removedTokenCount: 0
},
tokenizedTransform: (() => {
const map = new Map<number, Transform>();
@ -179,7 +182,8 @@ describe('precomputationSubsetKeyer', function() {
false
),
retokenization: [...rawTextTokens]
}
},
removedTokenCount: 0
},
tokenizedTransform: (() => {
const map = new Map<number, Transform>();
@ -246,7 +250,8 @@ describe('precomputationSubsetKeyer', function() {
false
),
retokenization: [...rawTextTokens]
}
},
removedTokenCount: 0
},
tokenizedTransform: (() => {
const map = new Map<number, Transform>();
@ -321,7 +326,8 @@ describe('precomputationSubsetKeyer', function() {
}],
splits: [],
unmappedEdits: [],
edgeWindow: edgeWindow1
edgeWindow: edgeWindow1,
removedTokenCount: 0
},
tokenizedTransform: (() => {
const map = new Map<number, Transform>();
@ -363,7 +369,8 @@ describe('precomputationSubsetKeyer', function() {
}],
splits: [],
unmappedEdits: [],
edgeWindow: edgeWindow1
edgeWindow: edgeWindow1,
removedTokenCount: 0
},
tokenizedTransform: (() => {
const map = new Map<number, Transform>();
@ -427,7 +434,8 @@ describe('precomputationSubsetKeyer', function() {
false
),
retokenization: [...rawTextTokens]
}
},
removedTokenCount: 0
},
tokenizedTransform: (() => {
const map = new Map<number, Transform>();
@ -479,7 +487,8 @@ describe('precomputationSubsetKeyer', function() {
false
),
retokenization: [...rawTextTokens]
}
},
removedTokenCount: 0
},
tokenizedTransform: (() => {
const map = new Map<number, Transform>();
@ -529,7 +538,8 @@ describe('precomputationSubsetKeyer', function() {
false
),
retokenization: [...rawTextTokens]
}
},
removedTokenCount: 0
},
tokenizedTransform: (() => {
const map = new Map<number, Transform>();