erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-10-27 00:07:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 15

feat(web): provide startOffset for detected splits based on start index within original token

Browse source
This commit is contained in:
Joshua Horton 2025-09-25 15:18:30 -05:00
parent 686ce86082
commit b1a9e07ef3
2 changed files with 7 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
View file

@ -40,7 +40,7 @@ interface TokenMergeMap {
interface TokenSplitMap {
input: EditTokenMap,
matches: EditTokenMap[]
matches: (EditTokenMap & { textOffset: number })[]
};
/**
@ -875,7 +875,8 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo
},
matches: [ {
index: match,
text: resultTokenization[match]
text: resultTokenization[match],
textOffset: 0
}],
};
let currentMerge = resultTokenization[match];
@ -883,10 +884,12 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo
// Look-ahead 1
let nextMerge = currentMerge + resultTokenization[match + matchOffset++];
for(/* next line */; splitTarget.indexOf(nextMerge) == 0; nextMerge = currentMerge + preTokenization[match + matchOffset++]) {
const textOffset = KMWString.length(currentMerge);
currentMerge = nextMerge;
split.matches.push({
index: match + matchOffset - 1,
text: resultTokenization[match + matchOffset-1]
text: resultTokenization[match + matchOffset-1],
textOffset
});
// Each time we 'pass' the condition, we've successfully processed an associated edit.

2
web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts
View file

@ -1062,7 +1062,7 @@ describe('ContextTokenization', function() {
merges: [],
splits: [ {
input: { text: 'can\'', index: 7 },
matches: [ { text: 'can', index: 7 }, { text: '\'', index: 8 }]
matches: [ { text: 'can', index: 7, textOffset: 0 }, { text: '\'', index: 8, textOffset: 3 }]
} ],
mergeOffset: 0,
splitOffset: -1,