erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-10-29 09:17:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Merge branch 'refactor/web/init-state-from-reset' into refactor/web/distribution-tokenization

Browse source
This commit is contained in:
Joshua Horton 2025-08-05 12:04:53 -05:00
commit c72e655895
4 changed files with 24 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts
View file

@ -69,13 +69,32 @@ export class ContextState {
*/
isManuallyApplied?: boolean;
/**
* Deep-copies a prior instance.
* @param stateToClone
*/
constructor(stateToClone: ContextState);
constructor(context: Context, model: LexicalModel);
constructor(param1: Context | ContextState, model?: LexicalModel) {
/**
* Initializes a new ContextState instance based on the active model and context.
*
* If a precomputed tokenization of the context (with prior correction-search
* calculation data) is not available, it will be spun up from scratch.
*
* @param context
* @param model
* @param tokenization
*/
constructor(context: Context, model: LexicalModel, tokenization?: ContextTokenization);
constructor(param1: Context | ContextState, model?: LexicalModel, tokenization?: ContextTokenization) {
if(!(param1 instanceof ContextState)) {
const context = param1;
this._context = context;
this.model = model;
if(tokenization) {
this.tokenization = tokenization;
} else {
this.initFromReset();
}
} else {
const stateToClone = param1;
@ -102,7 +121,7 @@ export class ContextState {
* information is available - typically, immediately after engine
* initialization or a context reset.
*/
initFromReset() {
private initFromReset() {
const context = this.context;
const lexicalModel = this.model;

1
web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts
View file

@ -443,7 +443,6 @@ export class ContextTracker extends CircularArray<ContextState> {
// Assumption: as a caret needs to move to context before any actual transform distributions occur,
// this state is only reached on caret moves; thus, transformDistribution is actually just a single null transform.
let state = new ContextState(context, model);
state.initFromReset();
this.enqueue(state);
return { state, baseState: null, headTokensRemoved: 0, tailTokensAdded: 0 };
}

19
web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-state.js
View file

@ -19,21 +19,16 @@ describe('ContextState', () => {
assert.equal(state.context, context);
assert.equal(state.model, plainModel);
assert.isNotOk(state.tokenization);
assert.isOk(state.tokenization);
assert.isUndefined(state.isManuallyApplied);
assert.isNotOk(state.suggestions);
assert.isNotOk(state.appliedSuggestionId);
});
describe('initFromReset', () => {
describe('initializing without prior tokenization', () => {
it('<empty context>', () => {
let context = { left: '', right: '' };
let state = new ContextState(context, plainModel);
assert.isNotOk(state.tokenization);
assert.equal(state.context, context);
state.initFromReset();
assert.isOk(state.tokenization);
assert.equal(state.tokenization.tokens.length, 1);
assert.equal(state.tokenization.tail.exampleInput, '');
@ -42,11 +37,6 @@ describe('ContextState', () => {
it('with initial text (without ending whitespace', () => {
let context = { left: 'the quick brown fox', right: '' };
let state = new ContextState(context, plainModel);
assert.isNotOk(state.tokenization);
assert.equal(state.context, context);
state.initFromReset();
assert.isOk(state.tokenization);
assert.equal(state.tokenization.tokens.length, 7);
assert.deepEqual(state.tokenization.exampleInput, ['the', ' ', 'quick', ' ', 'brown', ' ', 'fox']);
@ -55,11 +45,6 @@ describe('ContextState', () => {
it('with initial text (with ending whitespace', () => {
let context = { left: 'the quick brown fox ', right: '' };
let state = new ContextState(context, plainModel);
assert.isNotOk(state.tokenization);
assert.equal(state.context, context);
state.initFromReset();
assert.isOk(state.tokenization);
assert.equal(state.tokenization.tokens.length, 9);
assert.deepEqual(state.tokenization.exampleInput, ['the', ' ', 'quick', ' ', 'brown', ' ', 'fox', ' ', '']);

12
web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js
View file

@ -38,7 +38,6 @@ describe('ContextTracker', function() {
let rawTokens = [" ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
let baseState = new ContextState(existingContext, plainModel);
baseState.initFromReset();
let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
assert.isNotNull(newContextMatch?.state);
assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
@ -60,7 +59,6 @@ describe('ContextTracker', function() {
let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
let baseState = new ContextState(existingContext, plainModel);
baseState.initFromReset();
let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
assert.isNotNull(newContextMatch?.state);
assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
@ -79,7 +77,6 @@ describe('ContextTracker', function() {
let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
let baseState = new ContextState(existingContext, plainModel);
baseState.initFromReset();
let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform));
assert.isNotNull(newContextMatch?.state);
assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
@ -99,7 +96,6 @@ describe('ContextTracker', function() {
let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
let baseState = new ContextState(existingContext, plainModel);
baseState.initFromReset();
let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform));
assert.isNotNull(newContextMatch?.state);
assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
@ -125,7 +121,6 @@ describe('ContextTracker', function() {
let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
let baseState = new ContextState(existingContext, plainModel);
baseState.initFromReset();
let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform));
assert.isOk(newContextMatch?.state);
assert.deepEqual(newContextMatch?.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
@ -146,7 +141,6 @@ describe('ContextTracker', function() {
let rawTokens = ["'", "a"];
let baseState = new ContextState(existingContext, plainModel);
baseState.initFromReset();
let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform));
assert.isNotNull(newContextMatch?.state);
assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
@ -176,7 +170,6 @@ describe('ContextTracker', function() {
let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
let baseState = new ContextState(existingContext, plainModel);
baseState.initFromReset();
let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
assert.isNotNull(newContextMatch?.state);
assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
@ -203,7 +196,6 @@ describe('ContextTracker', function() {
let rawTokens = ["and", " ", ""];
let baseState = new ContextState(existingContext, plainModel);
baseState.initFromReset();
let newContextMatch = ContextTracker.attemptMatchContext(
existingContext,
plainModel,
@ -235,7 +227,6 @@ describe('ContextTracker', function() {
let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
let baseState = new ContextState(existingContext, plainModel);
baseState.initFromReset();
let newContextMatch = ContextTracker.attemptMatchContext(
existingContext,
plainModel,
@ -263,7 +254,6 @@ describe('ContextTracker', function() {
assert.equal(baseContext.left.length, 1);
let baseState = new ContextState({ left: "text'" }, plainModel);
baseState.initFromReset();
// Now the actual check.
let newContext = models.tokenize(defaultBreaker, {
@ -293,7 +283,6 @@ describe('ContextTracker', function() {
let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
let state = new ContextState(context, plainModel);
state.initFromReset();
assert.deepEqual(state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
});
@ -302,7 +291,6 @@ describe('ContextTracker', function() {
let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
let state = new ContextState(context, plainModel);
state.initFromReset();
assert.deepEqual(state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
});
});