erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-08-29 03:37:42 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 14

change(web): reworks ContextToken construction patterns

Browse source 
Build-bot: skip build:web
Test-bot: skip
This commit is contained in:
Joshua Horton 2026-04-06 11:52:34 -05:00
parent d285478319
commit df097f3f1d
6 changed files with 70 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts
View file

@ -151,7 +151,7 @@ export class ContextState {
private initFromReset() {
const tokenizedContext = determineModelTokenizer(this.model)(this.context).left;
const baseTokens = tokenizedContext.map((entry) => {
const token = new ContextToken(this.model, entry.text);
const token = ContextToken.fromRawText(this.model, entry.text);
if(entry.isWhitespace) {
token.isWhitespace = true;
@ -162,7 +162,7 @@ export class ContextState {
// And now build the final context state object, which includes whitespace 'tokens'.);
if(baseTokens.length == 0) {
baseTokens.push(new ContextToken(this.model));
baseTokens.push(ContextToken.fromRawText(this.model, ''));
}
this.tokenization = new ContextTokenization(baseTokens);
this.inputTransforms = new Map();

69
web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts
View file

@ -44,7 +44,7 @@ export class ContextToken {
/**
* Indicates whether or not the token is considered whitespace.
*/
isWhitespace: boolean;
isWhitespace: boolean = false;
/**
* Contains all relevant correction-search data for use in generating
@ -67,22 +67,16 @@ export class ContextToken {
appliedTransitionId?: number;
/**
* Constructs a new, empty instance for use with the specified LexicalModel.
* Constructs a new instance based directly on a pre-constructed SearchQuotientNode.
* @param model
*/
constructor(model: LexicalModel);
/**
* Constructs a new instance with pre-existing text for use with the specified LexicalModel.
* @param model
* @param rawText
*/
constructor(model: LexicalModel, rawText: string, isPartial?: boolean);
constructor(quotientNode: SearchQuotientNode, isPartial?: boolean);
/**
* This constructor deep-copies the specified instance.
* @param baseToken
*/
constructor(baseToken: ContextToken);
constructor(param: ContextToken | LexicalModel, rawText?: string, isPartial?: boolean) {
constructor(param: ContextToken | SearchQuotientNode, isPartial?: boolean) {
if(param instanceof ContextToken) {
const priorToken = param;
Object.assign(this, priorToken);
@ -93,32 +87,37 @@ export class ContextToken {
// we need to ensure that only fully-utilized keystrokes are considered.
this._searchModule = priorToken.searchModule;
} else {
const model = param;
// May be altered outside of the constructor.
this.isWhitespace = false;
this.isPartial = !!isPartial;
rawText ||= '';
// Supports the old pathway for: updateWithBackspace(tokenText: string, transformId: number)
// Build a token that represents the current text with no ambiguity - probability at max (1.0)
let searchModule: SearchQuotientNode = new LegacyQuotientRoot(model);
const BASE_PROBABILITY = 1;
textToCharTransforms(rawText).forEach((transform) => {
let inputMetadata: PathInputProperties = {
segment: {
start: 0,
transitionId: undefined
},
bestProbFromSet: BASE_PROBABILITY,
subsetId: generateSubsetId()
};
searchModule = new LegacyQuotientSpur(searchModule, [{sample: transform, p: BASE_PROBABILITY}], inputMetadata);
});
this._searchModule = searchModule;
this._searchModule = param;
}
this.isPartial = !!isPartial;
}
/**
* Constructs a new instance with pre-existing text for use with the specified LexicalModel.
* @param model
* @param rawText
*/
static fromRawText(model: LexicalModel, rawText: string, isPartial?: boolean) {
rawText ||= '';
// Supports the old pathway for: updateWithBackspace(tokenText: string, transformId: number)
// Build a token that represents the current text with no ambiguity - probability at max (1.0)
let searchModule: SearchQuotientNode = new LegacyQuotientRoot(model);
const BASE_PROBABILITY = 1;
textToCharTransforms(rawText).forEach((transform) => {
let inputMetadata: PathInputProperties = {
segment: {
start: 0,
transitionId: undefined
},
bestProbFromSet: BASE_PROBABILITY,
subsetId: generateSubsetId()
};
searchModule = new LegacyQuotientSpur(searchModule, [{sample: transform, p: BASE_PROBABILITY}], inputMetadata);
});
return new ContextToken(searchModule, isPartial);
}
/**

9
web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
View file

@ -20,6 +20,7 @@ import { TransitionEdge } from './tokenization-subsets.js';
import LexicalModel = LexicalModelTypes.LexicalModel;
import Transform = LexicalModelTypes.Transform;
import { LegacyQuotientRoot } from './legacy-quotient-root.js';
// May be able to "get away" with 2 & 5 or so, but having extra will likely help
// with edit path stability.
@ -227,7 +228,7 @@ export class ContextTokenization {
// token for it, but don't try to preserve its fat-finger data any more.
// (To do so may be a complex problem for little return.)
if(editBoundary.text) {
preservedTokens.unshift(new ContextToken(lexicalModel, editBoundary.text, editBoundary.isPartial));
preservedTokens.unshift(ContextToken.fromRawText(lexicalModel, editBoundary.text, editBoundary.isPartial));
}
// And done. Why retokenize? We already had a proper tokenization;
@ -314,7 +315,7 @@ export class ContextTokenization {
// fallthrough;
case 'insert':
case 'substitute':
tokensToPrefix.push(new ContextToken(lexicalModel, slidAndRetokenized[match], i - mergeOffset == 0));
tokensToPrefix.push(ContextToken.fromRawText(lexicalModel, slidAndRetokenized[match], i - mergeOffset == 0));
break;
default:
// do nothing.
@ -602,11 +603,11 @@ export class ContextTokenization {
affectedToken = tailTokenization[tokenIndex];
if(!affectedToken) {
affectedToken = new ContextToken(lexicalModel);
affectedToken = new ContextToken(new LegacyQuotientRoot(lexicalModel));
tailTokenization.push(affectedToken);
} else if(KMWString.length(affectedToken.exampleInput) == distribution[0].sample.deleteLeft) {
// If the entire token will be replaced, throw out the old one and start anew.
affectedToken = new ContextToken(lexicalModel);
affectedToken = new ContextToken(new LegacyQuotientRoot(lexicalModel));
// Replace the token at the affected index with a brand-new token.
tailTokenization.splice(tokenIndex, 1, affectedToken);
}

44
web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts
View file

@ -15,7 +15,7 @@ import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'
import { LexicalModelTypes } from '@keymanapp/common-types';
import { KMWString } from '@keymanapp/web-utils';
import { ContextToken, correction, generateSubsetId, getBestMatches, InputSegment, models, SearchQuotientSpur } from '@keymanapp/lm-worker/test-index';
import { ContextToken, correction, generateSubsetId, getBestMatches, InputSegment, LegacyQuotientRoot, models, SearchQuotientSpur } from '@keymanapp/lm-worker/test-index';
import { quotientPathHasInputs } from "../../helpers/quotientPathHasInputs.js";
@ -54,7 +54,7 @@ describe('ContextToken', function() {
describe("<constructor>", () => {
it("(model: LexicalModel)", async () => {
let token = new ContextToken(plainModel);
let token = new ContextToken(new LegacyQuotientRoot(plainModel));
assert.equal(token.searchModule.inputCount, 0);
assert.isEmpty(token.exampleInput);
@ -67,7 +67,7 @@ describe('ContextToken', function() {
});
it("(model: LexicalModel, text: string)", () => {
let token = new ContextToken(plainModel, "and");
let token = ContextToken.fromRawText(plainModel, "and");
assert.equal(token.searchModule.bestExample.text, 'and');
assert.equal(token.exampleInput, 'and');
@ -85,7 +85,7 @@ describe('ContextToken', function() {
it("(token: ContextToken", () => {
// Same as in a test above, since we verified that it works correctly.
let baseToken = new ContextToken(plainModel, "and");
let baseToken = ContextToken.fromRawText(plainModel, "and");
let clonedToken = new ContextToken(baseToken);
assert.equal(clonedToken.searchModule, baseToken.searchModule);
@ -103,9 +103,9 @@ describe('ContextToken', function() {
describe("merge()", () => {
it("merges three tokens without previously-split transforms", () => {
const token1 = new ContextToken(plainModel, "can");
const token2 = new ContextToken(plainModel, "'");
const token3 = new ContextToken(plainModel, "t");
const token1 = ContextToken.fromRawText(plainModel, "can");
const token2 = ContextToken.fromRawText(plainModel, "'");
const token3 = ContextToken.fromRawText(plainModel, "t");
const merged = ContextToken.merge([token1, token2, token3]);
assert.equal(merged.exampleInput, "can't");
@ -127,9 +127,9 @@ describe('ContextToken', function() {
const srcTransform = { insert: "can't", deleteLeft: 0, deleteRight: 0, id: 1 };
const srcSubsetId = generateSubsetId();
const token1 = new ContextToken(plainModel);
const token2 = new ContextToken(plainModel);
const token3 = new ContextToken(plainModel);
const token1 = new ContextToken(new LegacyQuotientRoot(plainModel));
const token2 = new ContextToken(new LegacyQuotientRoot(plainModel));
const token3 = new ContextToken(new LegacyQuotientRoot(plainModel));
token1.addInput({
segment: {
@ -185,13 +185,13 @@ describe('ContextToken', function() {
];
// apples
const token1 = new ContextToken(plainModel);
const token1 = new ContextToken(new LegacyQuotientRoot(plainModel));
// and
const token2 = new ContextToken(plainModel);
const token2 = new ContextToken(new LegacyQuotientRoot(plainModel));
// sour
const token3 = new ContextToken(plainModel);
const token3 = new ContextToken(new LegacyQuotientRoot(plainModel));
// grapes
const token4 = new ContextToken(plainModel);
const token4 = new ContextToken(new LegacyQuotientRoot(plainModel));
const tokensToMerge = [token1, token2, token3, token4]
token1.addInput({
@ -275,13 +275,13 @@ describe('ContextToken', function() {
];
// apples
const token1 = new ContextToken(plainModel);
const token1 = new ContextToken(new LegacyQuotientRoot(plainModel));
// and
const token2 = new ContextToken(plainModel);
const token2 = new ContextToken(new LegacyQuotientRoot(plainModel));
// sour
const token3 = new ContextToken(plainModel);
const token3 = new ContextToken(new LegacyQuotientRoot(plainModel));
// grapes
const token4 = new ContextToken(plainModel);
const token4 = new ContextToken(new LegacyQuotientRoot(plainModel));
const tokensToMerge = [token1, token2, token3, token4]
token1.addInput({
@ -371,7 +371,7 @@ describe('ContextToken', function() {
]
]
const tokenToSplit = new ContextToken(plainModel);
const tokenToSplit = new ContextToken(new LegacyQuotientRoot(plainModel));
for(let i = 0; i < keystrokeDistributions.length; i++) {
tokenToSplit.addInput({
segment: {
@ -414,7 +414,7 @@ describe('ContextToken', function() {
const splitTextArray = ['big', 'large', 'transform'];
const subsetId = generateSubsetId();
const tokenToSplit = new ContextToken(plainModel);
const tokenToSplit = new ContextToken(new LegacyQuotientRoot(plainModel));
for(let i = 0; i < keystrokeDistributions.length; i++) {
tokenToSplit.addInput({
segment: {
@ -485,7 +485,7 @@ describe('ContextToken', function() {
generateSubsetId()
];
const tokenToSplit = new ContextToken(plainModel);
const tokenToSplit = new ContextToken(new LegacyQuotientRoot(plainModel));
for(let i = 0; i < keystrokeDistributions.length; i++) {
tokenToSplit.addInput({
segment: {
@ -612,7 +612,7 @@ describe('ContextToken', function() {
generateSubsetId()
];
const tokenToSplit = new ContextToken(plainModel);
const tokenToSplit = new ContextToken(new LegacyQuotientRoot(plainModel));
for(let i = 0; i < keystrokeDistributions.length; i++) {
tokenToSplit.addInput({
segment: {

4
web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts
View file

@ -39,7 +39,7 @@ var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'),
function toToken(text: string) {
let isWhitespace = text == ' ';
let token = new ContextToken(plainModel, text);
let token = ContextToken.fromRawText(plainModel, text);
token.isWhitespace = isWhitespace;
return token;
}
@ -48,7 +48,7 @@ let TOKEN_TRANSFORM_SEED = 0;
function toTransformToken(text: string, transformId?: number) {
let idSeed = transformId === undefined ? TOKEN_TRANSFORM_SEED++ : transformId;
let isWhitespace = text == ' ';
let token = new ContextToken(plainModel);
let token = ContextToken.fromRawText(plainModel, '');
const textAsTransform = { insert: text, deleteLeft: 0, id: idSeed };
token.addInput({
segment: {

10
web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts
View file

@ -36,7 +36,7 @@ var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'),
function toToken(text: string) {
let isWhitespace = text == ' ';
let token = new ContextToken(plainModel, text);
let token = ContextToken.fromRawText(plainModel, text);
token.isWhitespace = isWhitespace;
return token;
}
@ -179,7 +179,7 @@ describe('precomputationSubsetKeyer', function() {
edgeWindow: {
...buildEdgeWindow(
[...tokenization.tokens, (() => {
const token = new ContextToken(plainModel, 'da');
const token = ContextToken.fromRawText(plainModel, 'da');
// source text: 'date'
token.addInput({
segment: {
@ -211,7 +211,7 @@ describe('precomputationSubsetKeyer', function() {
precomputation2.alignment.edgeWindow = {
...buildEdgeWindow(
[...tokenization.tokens, (() => {
const token = new ContextToken(plainModel, 'da');
const token = ContextToken.fromRawText(plainModel, 'da');
// source text: 'date'
token.addInput({
segment: {
@ -256,7 +256,7 @@ describe('precomputationSubsetKeyer', function() {
edgeWindow: {
...buildEdgeWindow(
[...tokenization.tokens, (() => {
const token = new ContextToken(plainModel, 'da');
const token = ContextToken.fromRawText(plainModel, 'da');
token.isPartial = true;
// source text: 'dat'
token.addInput({
@ -288,7 +288,7 @@ describe('precomputationSubsetKeyer', function() {
precomputation2.alignment.edgeWindow = {
...buildEdgeWindow(
[...tokenization.tokens, (() => {
const token = new ContextToken(plainModel, 'da');
const token = ContextToken.fromRawText(plainModel, 'da');
token.isPartial = true;
// source text: 'dat'
token.addInput({