From b4c9b4091656115add28c537ba9af0e9392cdc07 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 11 Aug 2026 17:17:38 -0500 Subject: [PATCH 1/2] fix(web): annotate transition ID on reconstructed context tokens Build-bot: skip build:web Test-bot: skip --- .../src/main/correction/context-token.ts | 7 ++- .../main/correction/context-tokenization.ts | 3 ++ .../context/context-token.tests.ts | 52 +++++++++++++------ .../context/transition-helpers.tests.ts | 34 ++++++++++++ 4 files changed, 79 insertions(+), 17 deletions(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts index 90cdcabb77..e1265b4b63 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts @@ -17,6 +17,7 @@ import { generateSubsetId } from './tokenization-subsets.js'; import Distribution = LexicalModelTypes.Distribution; import LexicalModel = LexicalModelTypes.LexicalModel; +import ProbabilityMass = LexicalModelTypes.ProbabilityMass; import Transform = LexicalModelTypes.Transform; /** @@ -113,7 +114,11 @@ export class ContextToken { bestProbFromSet: BASE_PROBABILITY, subsetId: generateSubsetId() }; - searchModule = new LegacyQuotientSpur(searchModule, [{sample: transform, p: BASE_PROBABILITY}], inputMetadata); + const edge: ProbabilityMass = {sample: transform, p: BASE_PROBABILITY}; + if(transitionId !== undefined) { + edge.sample.id = transitionId; + } + searchModule = new LegacyQuotientSpur(searchModule, [edge], inputMetadata); }); return new ContextToken(searchModule, isPartial); diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 42dd97e3e8..39bdd5f61a 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -527,6 +527,9 @@ export class ContextTokenization { // Mutates stackedInserts, stackedDeletes. const baseRemovedTokenCount = Math.max(0, stackedDeletes.length - stackedInserts.length); const transformMap = assembleTransforms(stackedInserts, stackedDeletes, tailIndex); + if(transform.id !== undefined) { + [...transformMap.values()].forEach((v) => v.id = transform.id); + } // If there's an empty transform in the 0 position and we already know we're // dropping tokens - and only deleting - we're dropping an diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts index b9bebb4d35..b7ff05135b 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts @@ -51,21 +51,8 @@ describe('ContextToken', function() { KMWString.enableSupplementaryPlane(true); }); - describe("", () => { - it("(model: LexicalModel)", async () => { - let token = new ContextToken(new LegacyQuotientRoot(plainModel)); - - assert.equal(token.searchModule.inputCount, 0); - assert.isEmpty(token.exampleInput); - assert.isFalse(token.isWhitespace); - - // While searchSpace has no inputs, it _can_ match lexicon entries (via insertions). - let searchIterator = getBestMatches([token.searchModule], new ExecutionTimer(Number.POSITIVE_INFINITY, Number.POSITIVE_INFINITY)); - let firstEntry = await searchIterator.next(); - assert.isFalse(firstEntry.done); - }); - - it("(model: LexicalModel, text: string)", () => { + describe('fromRawText', () => { + it("may construct a fresh ContextToken from a model and plain text", () => { let token = ContextToken.fromRawText(plainModel, "and"); assert.equal(token.searchModule.bestExample.text, 'and'); @@ -82,7 +69,40 @@ describe('ContextToken', function() { assert.isFalse(token.isWhitespace); }); - it("(token: ContextToken", () => { + it("may assign a transition ID to the constructed ContextToken", () => { + const TRANSITION_ID = 6; + let token = ContextToken.fromRawText(plainModel, "and", false, TRANSITION_ID); + + assert.equal(token.searchModule.bestExample.text, 'and'); + assert.equal(token.exampleInput, 'and'); + + assert.equal(token.searchModule.inputCount, 3); + assert.isTrue(quotientPathHasInputs( + token.searchModule, [ + [{sample: { insert: 'a', deleteLeft: 0, id: TRANSITION_ID }, p: 1}], + [{sample: { insert: 'n', deleteLeft: 0, id: TRANSITION_ID }, p: 1}], + [{sample: { insert: 'd', deleteLeft: 0, id: TRANSITION_ID }, p: 1}] + ])); + + assert.isFalse(token.isWhitespace); + }); + }); + + describe("", () => { + it("constructs from SearchQuotient root nodes", async () => { + let token = new ContextToken(new LegacyQuotientRoot(plainModel)); + + assert.equal(token.searchModule.inputCount, 0); + assert.isEmpty(token.exampleInput); + assert.isFalse(token.isWhitespace); + + // While searchSpace has no inputs, it _can_ match lexicon entries (via insertions). + let searchIterator = getBestMatches([token.searchModule], new ExecutionTimer(Number.POSITIVE_INFINITY, Number.POSITIVE_INFINITY)); + let firstEntry = await searchIterator.next(); + assert.isFalse(firstEntry.done); + }); + + it("may clone a previously-existing ContextToken", () => { // Same as in a test above, since we verified that it works correctly. let baseToken = ContextToken.fromRawText(plainModel, "and"); let clonedToken = new ContextToken(baseToken); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/transition-helpers.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/transition-helpers.tests.ts index 3e689365a2..ec67c4e77d 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/transition-helpers.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/transition-helpers.tests.ts @@ -13,8 +13,11 @@ import { default as defaultBreaker } from '@keymanapp/models-wordbreakers'; import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; import { LexicalModelTypes } from '@keymanapp/common-types'; +import { KMWString } from 'keyman/common/web-utils'; + import { buildEdgeWindow, + ContextState, ContextToken, ContextTokenization, generateSubsetId, @@ -99,6 +102,9 @@ function buildTokenizationForSimpleInputs ( // Only the 'insert' and 'deleteLeft' fields are set during transform // tokenization at this time. map.set(0, { insert: sample.insert, deleteLeft: sample.deleteLeft }); + if(sample.id !== undefined) { + map.get(0).id = sample.id; + } return { sample: map, p }; }) }; @@ -501,4 +507,32 @@ describe('transitionTokenizations', () => { assertMatchingTokenization(actual, expected, msg); } }); + + it('handles backspace transitions correctly for a recently reset context', () => { + const baseState = new ContextState({ + left: 'an apple a', + startOfBuffer: true, + endOfBuffer: true + }, plainModel); + + const dist: Distribution = [{ + sample: { + insert: '', + deleteLeft: 3 + }, + p: 1 + }] + const precomputedTransition = precomputeTransitions([baseState.tokenization], dist); + + const result = transitionTokenizations(precomputedTransition.subsets, dist); + + const resultTokenization = result.get(precomputedTransition.keyMatchingUserContext); + + assert.isOk(resultTokenization); + const resultTail = resultTokenization.tail; + + assert.equal(resultTail.exampleInput, 'appl'); + assert.equal(resultTail.inputCount, resultTail.searchModule.codepointLength); + assert.equal(resultTail.searchModule.codepointLength, KMWString.length(resultTail.exampleInput)); + }); }); \ No newline at end of file From 4c120b9137f8b3994265685d5dff6fc8a34bb7b3 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Wed, 12 Aug 2026 10:57:04 -0500 Subject: [PATCH 2/2] change(web): clean up id-tagging transform-map for-each --- .../worker-thread/src/main/correction/context-tokenization.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 39bdd5f61a..4b798854ea 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -528,7 +528,7 @@ export class ContextTokenization { const baseRemovedTokenCount = Math.max(0, stackedDeletes.length - stackedInserts.length); const transformMap = assembleTransforms(stackedInserts, stackedDeletes, tailIndex); if(transform.id !== undefined) { - [...transformMap.values()].forEach((v) => v.id = transform.id); + transformMap.forEach((v) => v.id = transform.id); } // If there's an empty transform in the 0 position and we already know we're