erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-10-17 03:17:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 12

Merge branch 'fix/web/sliding-tokenization-alignment' into refactor/web/independent-compute-alignment

Browse source
This commit is contained in:
Joshua Horton 2025-08-27 15:26:02 -05:00
commit 8b3b0affd4
1 changed files with 8 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
View file

@ -124,11 +124,15 @@ export class ContextTokenization {
if(leadTokenShift < 0) {
tokenization.splice(0, -leadTokenShift);
} else if(leadTokenShift > 0) {
// TODO: insert token(s) at the start to match the text that's back within the
// insert token(s) at the start to match the text that's back within the
// sliding context window.
//
// (was not part of original source method [`attemptContextMatch`])
return null;
const reinsertedTokens = tokenizedContext.slice(0, leadTokenShift);
while(reinsertedTokens.length > 0) {
const reinserted = reinsertedTokens.pop();
const token = new ContextToken(lexicalModel, reinserted.text);
tokenization.unshift(token);
}
}
// If no TAIL mutations have happened, we're safe to return now.