erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-10-18 20:07:39 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge branch 'change/web/cache-states-selectively' into fix/web/sliding-tokenization-alignment

Browse source
This commit is contained in:
Joshua Horton 2025-08-15 14:06:09 -05:00
commit 470bf6a54f
2 changed files with 5 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts
View file

@ -179,13 +179,11 @@ export class ContextState {
// Apply all transforms to the base context state
const transformSequenceDistribution = tokenizeAndFilterDistribution(context, lexicalModel, transformDistribution);
const postContext = transformDistribution?.[0] ? applyTransform(transformDistribution[0].sample, context) : context;
if(transformDistribution?.[0]) {
context = applyTransform(transformDistribution[0].sample, context);
}
// Note for future: the next line's pattern asserts that there is only one true tokenization.
// We may eventually allow for multiple potential tokenizations (per epic-dict-breaker)
const tokenizedContext = determineModelTokenizer(lexicalModel)(context).left;
const tokenizedContext = determineModelTokenizer(lexicalModel)(postContext).left;
if(tokenizedContext.length == 0) {
tokenizedContext.push({text: ''});
}
@ -204,7 +202,7 @@ export class ContextState {
}
const resultTokenization = this.tokenization.transitionTo(
determineModelTokenizer(lexicalModel)(context).left,
tokenizedContext,
alignmentResults,
lexicalModel,
transformSequenceDistribution
@ -243,7 +241,7 @@ export class ContextState {
preservationTransform = preservationTransform ? buildMergedTransform(preservationTransform, primaryInput): primaryInput;
}
const state = new ContextState(context, lexicalModel);
const state = new ContextState(postContext, lexicalModel);
state.tokenization = resultTokenization;
state.appliedInput = transformDistribution?.[0].sample;
transition.finalize(state, transformDistribution, preservationTransform);

2
web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
View file

@ -325,7 +325,7 @@ export class ContextTokenization {
* with the tokenization represented by the current instance.
* @param alignment The alignment, as determined by a prior call to `computeAlignment`.
* @param lexicalModel The active lexical model
* @param alignedTransformDistribution The tokenized version of the input Transform instance accounting
* @param alignedTransformDistribution The tokenized version of the input distribution accounting
* for the difference between the context represented by this instance and that of `tokenizedContext`.
* @returns
*/