From 1ffed0991a31e3efd449baa79c0ab27a035b6639 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 29 Sep 2025 10:22:58 -0500 Subject: [PATCH 1/2] fix(web): fix edge-window construction handling of non-BMP chars --- .../main/correction/context-tokenization.ts | 2 +- .../context/context-tokenization.tests.ts | 61 +++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 4b13d9d7fa..e5e04e8b74 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -474,7 +474,7 @@ export function buildEdgeWindow( // token, we hit the boundary; note the boundary text. if(deleteCnt == 0 && tokenDeleteLength != tokenLen) { editBoundary = { - text: applyAtFront ? KMWString.slice(token, tokenDeleteLength) : KMWString.slice(token, 0, tokenLen - tokenDeleteLength), + text: applyAtFront ? KMWString.substring(token, tokenDeleteLength) : KMWString.substring(token, 0, tokenLen - tokenDeleteLength), tokenIndex: i, isPartial: tokenDeleteLength != 0 || tokenIsPartial } diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index 6f34a10a3b..c9e06b156d 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -13,6 +13,7 @@ import { assert } from 'chai'; import { default as defaultBreaker } from '@keymanapp/models-wordbreakers'; import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; import { LexicalModelTypes } from '@keymanapp/common-types'; +import { KMWString } from '@keymanapp/web-utils'; import { buildEdgeWindow, ContextStateAlignment, ContextToken, ContextTokenization, models } from '@keymanapp/lm-worker/test-index'; @@ -29,7 +30,31 @@ function toToken(text: string) { return token; } +// https://www.compart.com/en/unicode/block/U+1D400 +const mathBoldUpperA = 0x1D400; // Mathematical Bold Capital A +const mathBoldLowerA = 0x1D41A; // Small A + +function toMathematicalSMP(text: string) { + const chars = [...text]; + + const asSMP = chars.map((c) => { + if(c >= 'a' && c <= 'z') { + return String.fromCodePoint(mathBoldLowerA + (c.charCodeAt(0) - 'a'.charCodeAt(0))); + } else if(c >= 'A' && c <= 'Z') { + return String.fromCodePoint(mathBoldUpperA + (c.charCodeAt(0) - 'A'.charCodeAt(0))); + } else { + return c; + } + }); + + return asSMP.join(''); +} + describe('ContextTokenization', function() { + before(() => { + KMWString.enableSupplementaryPlane(true); + }); + describe("", () => { it("constructs from just a token array", () => { const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; @@ -696,6 +721,24 @@ describe('ContextTokenization', function() { }); }); + it('builds edge windows for the start of context with no edits - SMP strings', () => { + const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'].map(s => toMathematicalSMP(s)); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t)), null); + + const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 0 }, true, editWindowSpec); + assert.deepEqual(results, { + retokenizationText: toMathematicalSMP('an apple'), + editBoundary: { + isPartial: false, + omitsEmptyToken: false, + text: toMathematicalSMP('an'), + tokenIndex: 0 + }, + deleteLengths: [0], + sliceIndex: 3 + }); + }); + it('builds edge windows for the start of context with deletion edits (1)', () => { const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t)), null); @@ -714,6 +757,24 @@ describe('ContextTokenization', function() { }); }); + it('builds edge windows for the start of context with deletion edits (1) - SMP strings', () => { + const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'].map(s => toMathematicalSMP(s)); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t)), null); + + const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 2 }, true, editWindowSpec); + assert.deepEqual(results, { + retokenizationText: toMathematicalSMP(' apple a'), + editBoundary: { + isPartial: false, + omitsEmptyToken: false, + text: ' ', + tokenIndex: 1 + }, + deleteLengths: [2, 0], + sliceIndex: 5 + }); + }); + it('builds edge windows for the start of context with deletion edits (2)', () => { const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t)), null); From 10ccc954efe801a8e4b6f18d1037a4cf14fee076 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 29 Sep 2025 10:27:25 -0500 Subject: [PATCH 2/2] feat(web): add non-BMP test for edge-window-slide suite --- .../context/context-tokenization.tests.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index 4c0426db3b..4c3ebc10e5 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -868,6 +868,17 @@ describe('ContextTokenization', function() { assert.isTrue(resultTokenization.tokens[0].isPartial); }); + it('preserves tokenization patterns when word slides partially out of window - SMP strings', () => { + const baseTokens = ['apples', ' ', 'and', ' ', 'bananas'].map(s => toMathematicalSMP(s)); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t)), null); + + const resultTokenization = baseTokenization.applyContextSlide(plainModel, { insert: '', deleteLeft: 0, deleteRight: 2}); + + assert.notStrictEqual(resultTokenization, baseTokenization); + assert.deepEqual(resultTokenization.exampleInput, ['ples', ' ', 'and', ' ', 'bananas'].map(s => toMathematicalSMP(s))); + assert.isTrue(resultTokenization.tokens[0].isPartial); + }); + it('does not preserve deleted tokens', () => { const baseTokens = ['apples', ' ', 'and', ' ', 'bananas']; const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t)), null);