diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts index a0ca1bf8cb..8d42dfc1cc 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts @@ -151,7 +151,7 @@ export class ContextState { private initFromReset() { const tokenizedContext = determineModelTokenizer(this.model)(this.context).left; const baseTokens = tokenizedContext.map((entry) => { - const token = new ContextToken(this.model, entry.text); + const token = ContextToken.fromRawText(this.model, entry.text); if(entry.isWhitespace) { token.isWhitespace = true; @@ -162,7 +162,7 @@ export class ContextState { // And now build the final context state object, which includes whitespace 'tokens'.); if(baseTokens.length == 0) { - baseTokens.push(new ContextToken(this.model)); + baseTokens.push(ContextToken.fromRawText(this.model, '')); } this.tokenization = new ContextTokenization(baseTokens); this.inputTransforms = new Map(); diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts index 2f250f0e7c..c7da1d2691 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts @@ -44,7 +44,7 @@ export class ContextToken { /** * Indicates whether or not the token is considered whitespace. */ - isWhitespace: boolean; + isWhitespace: boolean = false; /** * Contains all relevant correction-search data for use in generating @@ -67,22 +67,16 @@ export class ContextToken { appliedTransitionId?: number; /** - * Constructs a new, empty instance for use with the specified LexicalModel. + * Constructs a new instance based directly on a pre-constructed SearchQuotientNode. * @param model */ - constructor(model: LexicalModel); - /** - * Constructs a new instance with pre-existing text for use with the specified LexicalModel. - * @param model - * @param rawText - */ - constructor(model: LexicalModel, rawText: string, isPartial?: boolean); + constructor(quotientNode: SearchQuotientNode, isPartial?: boolean); /** * This constructor deep-copies the specified instance. * @param baseToken */ constructor(baseToken: ContextToken); - constructor(param: ContextToken | LexicalModel, rawText?: string, isPartial?: boolean) { + constructor(param: ContextToken | SearchQuotientNode, isPartial?: boolean) { if(param instanceof ContextToken) { const priorToken = param; Object.assign(this, priorToken); @@ -93,32 +87,37 @@ export class ContextToken { // we need to ensure that only fully-utilized keystrokes are considered. this._searchModule = priorToken.searchModule; } else { - const model = param; - - // May be altered outside of the constructor. - this.isWhitespace = false; - this.isPartial = !!isPartial; - - rawText ||= ''; - - // Supports the old pathway for: updateWithBackspace(tokenText: string, transformId: number) - // Build a token that represents the current text with no ambiguity - probability at max (1.0) - let searchModule: SearchQuotientNode = new LegacyQuotientRoot(model); - const BASE_PROBABILITY = 1; - textToCharTransforms(rawText).forEach((transform) => { - let inputMetadata: PathInputProperties = { - segment: { - start: 0, - transitionId: undefined - }, - bestProbFromSet: BASE_PROBABILITY, - subsetId: generateSubsetId() - }; - searchModule = new LegacyQuotientSpur(searchModule, [{sample: transform, p: BASE_PROBABILITY}], inputMetadata); - }); - - this._searchModule = searchModule; + this._searchModule = param; } + + this.isPartial = !!isPartial; + } + + /** + * Constructs a new instance with pre-existing text for use with the specified LexicalModel. + * @param model + * @param rawText + */ + static fromRawText(model: LexicalModel, rawText: string, isPartial?: boolean) { + rawText ||= ''; + + // Supports the old pathway for: updateWithBackspace(tokenText: string, transformId: number) + // Build a token that represents the current text with no ambiguity - probability at max (1.0) + let searchModule: SearchQuotientNode = new LegacyQuotientRoot(model); + const BASE_PROBABILITY = 1; + textToCharTransforms(rawText).forEach((transform) => { + let inputMetadata: PathInputProperties = { + segment: { + start: 0, + transitionId: undefined + }, + bestProbFromSet: BASE_PROBABILITY, + subsetId: generateSubsetId() + }; + searchModule = new LegacyQuotientSpur(searchModule, [{sample: transform, p: BASE_PROBABILITY}], inputMetadata); + }); + + return new ContextToken(searchModule, isPartial); } /** diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index be50382a91..5e1ff1ead1 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -20,6 +20,7 @@ import { TransitionEdge } from './tokenization-subsets.js'; import LexicalModel = LexicalModelTypes.LexicalModel; import Transform = LexicalModelTypes.Transform; +import { LegacyQuotientRoot } from './legacy-quotient-root.js'; // May be able to "get away" with 2 & 5 or so, but having extra will likely help // with edit path stability. @@ -227,7 +228,7 @@ export class ContextTokenization { // token for it, but don't try to preserve its fat-finger data any more. // (To do so may be a complex problem for little return.) if(editBoundary.text) { - preservedTokens.unshift(new ContextToken(lexicalModel, editBoundary.text, editBoundary.isPartial)); + preservedTokens.unshift(ContextToken.fromRawText(lexicalModel, editBoundary.text, editBoundary.isPartial)); } // And done. Why retokenize? We already had a proper tokenization; @@ -314,7 +315,7 @@ export class ContextTokenization { // fallthrough; case 'insert': case 'substitute': - tokensToPrefix.push(new ContextToken(lexicalModel, slidAndRetokenized[match], i - mergeOffset == 0)); + tokensToPrefix.push(ContextToken.fromRawText(lexicalModel, slidAndRetokenized[match], i - mergeOffset == 0)); break; default: // do nothing. @@ -602,11 +603,11 @@ export class ContextTokenization { affectedToken = tailTokenization[tokenIndex]; if(!affectedToken) { - affectedToken = new ContextToken(lexicalModel); + affectedToken = new ContextToken(new LegacyQuotientRoot(lexicalModel)); tailTokenization.push(affectedToken); } else if(KMWString.length(affectedToken.exampleInput) == distribution[0].sample.deleteLeft) { // If the entire token will be replaced, throw out the old one and start anew. - affectedToken = new ContextToken(lexicalModel); + affectedToken = new ContextToken(new LegacyQuotientRoot(lexicalModel)); // Replace the token at the affected index with a brand-new token. tailTokenization.splice(tokenIndex, 1, affectedToken); } diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts index 791c594957..6a30826446 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts @@ -15,7 +15,7 @@ import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs' import { LexicalModelTypes } from '@keymanapp/common-types'; import { KMWString } from '@keymanapp/web-utils'; -import { ContextToken, correction, generateSubsetId, getBestMatches, InputSegment, models, SearchQuotientSpur } from '@keymanapp/lm-worker/test-index'; +import { ContextToken, correction, generateSubsetId, getBestMatches, InputSegment, LegacyQuotientRoot, models, SearchQuotientSpur } from '@keymanapp/lm-worker/test-index'; import { quotientPathHasInputs } from "../../helpers/quotientPathHasInputs.js"; @@ -54,7 +54,7 @@ describe('ContextToken', function() { describe("", () => { it("(model: LexicalModel)", async () => { - let token = new ContextToken(plainModel); + let token = new ContextToken(new LegacyQuotientRoot(plainModel)); assert.equal(token.searchModule.inputCount, 0); assert.isEmpty(token.exampleInput); @@ -67,7 +67,7 @@ describe('ContextToken', function() { }); it("(model: LexicalModel, text: string)", () => { - let token = new ContextToken(plainModel, "and"); + let token = ContextToken.fromRawText(plainModel, "and"); assert.equal(token.searchModule.bestExample.text, 'and'); assert.equal(token.exampleInput, 'and'); @@ -85,7 +85,7 @@ describe('ContextToken', function() { it("(token: ContextToken", () => { // Same as in a test above, since we verified that it works correctly. - let baseToken = new ContextToken(plainModel, "and"); + let baseToken = ContextToken.fromRawText(plainModel, "and"); let clonedToken = new ContextToken(baseToken); assert.equal(clonedToken.searchModule, baseToken.searchModule); @@ -103,9 +103,9 @@ describe('ContextToken', function() { describe("merge()", () => { it("merges three tokens without previously-split transforms", () => { - const token1 = new ContextToken(plainModel, "can"); - const token2 = new ContextToken(plainModel, "'"); - const token3 = new ContextToken(plainModel, "t"); + const token1 = ContextToken.fromRawText(plainModel, "can"); + const token2 = ContextToken.fromRawText(plainModel, "'"); + const token3 = ContextToken.fromRawText(plainModel, "t"); const merged = ContextToken.merge([token1, token2, token3]); assert.equal(merged.exampleInput, "can't"); @@ -127,9 +127,9 @@ describe('ContextToken', function() { const srcTransform = { insert: "can't", deleteLeft: 0, deleteRight: 0, id: 1 }; const srcSubsetId = generateSubsetId(); - const token1 = new ContextToken(plainModel); - const token2 = new ContextToken(plainModel); - const token3 = new ContextToken(plainModel); + const token1 = new ContextToken(new LegacyQuotientRoot(plainModel)); + const token2 = new ContextToken(new LegacyQuotientRoot(plainModel)); + const token3 = new ContextToken(new LegacyQuotientRoot(plainModel)); token1.addInput({ segment: { @@ -185,13 +185,13 @@ describe('ContextToken', function() { ]; // apples - const token1 = new ContextToken(plainModel); + const token1 = new ContextToken(new LegacyQuotientRoot(plainModel)); // and - const token2 = new ContextToken(plainModel); + const token2 = new ContextToken(new LegacyQuotientRoot(plainModel)); // sour - const token3 = new ContextToken(plainModel); + const token3 = new ContextToken(new LegacyQuotientRoot(plainModel)); // grapes - const token4 = new ContextToken(plainModel); + const token4 = new ContextToken(new LegacyQuotientRoot(plainModel)); const tokensToMerge = [token1, token2, token3, token4] token1.addInput({ @@ -275,13 +275,13 @@ describe('ContextToken', function() { ]; // apples - const token1 = new ContextToken(plainModel); + const token1 = new ContextToken(new LegacyQuotientRoot(plainModel)); // and - const token2 = new ContextToken(plainModel); + const token2 = new ContextToken(new LegacyQuotientRoot(plainModel)); // sour - const token3 = new ContextToken(plainModel); + const token3 = new ContextToken(new LegacyQuotientRoot(plainModel)); // grapes - const token4 = new ContextToken(plainModel); + const token4 = new ContextToken(new LegacyQuotientRoot(plainModel)); const tokensToMerge = [token1, token2, token3, token4] token1.addInput({ @@ -371,7 +371,7 @@ describe('ContextToken', function() { ] ] - const tokenToSplit = new ContextToken(plainModel); + const tokenToSplit = new ContextToken(new LegacyQuotientRoot(plainModel)); for(let i = 0; i < keystrokeDistributions.length; i++) { tokenToSplit.addInput({ segment: { @@ -414,7 +414,7 @@ describe('ContextToken', function() { const splitTextArray = ['big', 'large', 'transform']; const subsetId = generateSubsetId(); - const tokenToSplit = new ContextToken(plainModel); + const tokenToSplit = new ContextToken(new LegacyQuotientRoot(plainModel)); for(let i = 0; i < keystrokeDistributions.length; i++) { tokenToSplit.addInput({ segment: { @@ -485,7 +485,7 @@ describe('ContextToken', function() { generateSubsetId() ]; - const tokenToSplit = new ContextToken(plainModel); + const tokenToSplit = new ContextToken(new LegacyQuotientRoot(plainModel)); for(let i = 0; i < keystrokeDistributions.length; i++) { tokenToSplit.addInput({ segment: { @@ -612,7 +612,7 @@ describe('ContextToken', function() { generateSubsetId() ]; - const tokenToSplit = new ContextToken(plainModel); + const tokenToSplit = new ContextToken(new LegacyQuotientRoot(plainModel)); for(let i = 0; i < keystrokeDistributions.length; i++) { tokenToSplit.addInput({ segment: { diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index df2f027da9..60e8731755 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -39,7 +39,7 @@ var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'), function toToken(text: string) { let isWhitespace = text == ' '; - let token = new ContextToken(plainModel, text); + let token = ContextToken.fromRawText(plainModel, text); token.isWhitespace = isWhitespace; return token; } @@ -48,7 +48,7 @@ let TOKEN_TRANSFORM_SEED = 0; function toTransformToken(text: string, transformId?: number) { let idSeed = transformId === undefined ? TOKEN_TRANSFORM_SEED++ : transformId; let isWhitespace = text == ' '; - let token = new ContextToken(plainModel); + let token = ContextToken.fromRawText(plainModel, ''); const textAsTransform = { insert: text, deleteLeft: 0, id: idSeed }; token.addInput({ segment: { diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts index 986fa61ea5..48304cc667 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts @@ -36,7 +36,7 @@ var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'), function toToken(text: string) { let isWhitespace = text == ' '; - let token = new ContextToken(plainModel, text); + let token = ContextToken.fromRawText(plainModel, text); token.isWhitespace = isWhitespace; return token; } @@ -179,7 +179,7 @@ describe('precomputationSubsetKeyer', function() { edgeWindow: { ...buildEdgeWindow( [...tokenization.tokens, (() => { - const token = new ContextToken(plainModel, 'da'); + const token = ContextToken.fromRawText(plainModel, 'da'); // source text: 'date' token.addInput({ segment: { @@ -211,7 +211,7 @@ describe('precomputationSubsetKeyer', function() { precomputation2.alignment.edgeWindow = { ...buildEdgeWindow( [...tokenization.tokens, (() => { - const token = new ContextToken(plainModel, 'da'); + const token = ContextToken.fromRawText(plainModel, 'da'); // source text: 'date' token.addInput({ segment: { @@ -256,7 +256,7 @@ describe('precomputationSubsetKeyer', function() { edgeWindow: { ...buildEdgeWindow( [...tokenization.tokens, (() => { - const token = new ContextToken(plainModel, 'da'); + const token = ContextToken.fromRawText(plainModel, 'da'); token.isPartial = true; // source text: 'dat' token.addInput({ @@ -288,7 +288,7 @@ describe('precomputationSubsetKeyer', function() { precomputation2.alignment.edgeWindow = { ...buildEdgeWindow( [...tokenization.tokens, (() => { - const token = new ContextToken(plainModel, 'da'); + const token = ContextToken.fromRawText(plainModel, 'da'); token.isPartial = true; // source text: 'dat' token.addInput({