diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 43ce302b5a..706b6d246a 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -624,7 +624,7 @@ export function buildEdgeWindow( // token, we hit the boundary; note the boundary text. if(deleteCnt == 0 && tokenDeleteLength != tokenLen) { editBoundary = { - text: applyAtFront ? KMWString.slice(token, tokenDeleteLength) : KMWString.slice(token, 0, tokenLen - tokenDeleteLength), + text: applyAtFront ? KMWString.substring(token, tokenDeleteLength) : KMWString.substring(token, 0, tokenLen - tokenDeleteLength), tokenIndex: i, isPartial: tokenDeleteLength != 0 || tokenIsPartial } diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index 5f7e7bc9a5..5d680a89aa 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -13,6 +13,7 @@ import { assert } from 'chai'; import { default as defaultBreaker } from '@keymanapp/models-wordbreakers'; import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; import { LexicalModelTypes } from '@keymanapp/common-types'; +import { KMWString } from '@keymanapp/web-utils'; import { buildEdgeWindow, ContextStateAlignment, ContextToken, ContextTokenization, models, traceInsertEdits } from '@keymanapp/lm-worker/test-index'; @@ -29,7 +30,31 @@ function toToken(text: string) { return token; } +// https://www.compart.com/en/unicode/block/U+1D400 +const mathBoldUpperA = 0x1D400; // Mathematical Bold Capital A +const mathBoldLowerA = 0x1D41A; // Small A + +function toMathematicalSMP(text: string) { + const chars = [...text]; + + const asSMP = chars.map((c) => { + if(c >= 'a' && c <= 'z') { + return String.fromCodePoint(mathBoldLowerA + (c.charCodeAt(0) - 'a'.charCodeAt(0))); + } else if(c >= 'A' && c <= 'Z') { + return String.fromCodePoint(mathBoldUpperA + (c.charCodeAt(0) - 'A'.charCodeAt(0))); + } else { + return c; + } + }); + + return asSMP.join(''); +} + describe('ContextTokenization', function() { + before(() => { + KMWString.enableSupplementaryPlane(true); + }); + describe("", () => { it("constructs from just a token array", () => { const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; @@ -696,6 +721,24 @@ describe('ContextTokenization', function() { }); }); + it('builds edge windows for the start of context with no edits - SMP strings', () => { + const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'].map(s => toMathematicalSMP(s)); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t)), null); + + const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 0 }, true, editWindowSpec); + assert.deepEqual(results, { + retokenizationText: toMathematicalSMP('an apple'), + editBoundary: { + isPartial: false, + omitsEmptyToken: false, + text: toMathematicalSMP('an'), + tokenIndex: 0 + }, + deleteLengths: [0], + sliceIndex: 3 + }); + }); + it('builds edge windows for the start of context with deletion edits (1)', () => { const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t)), null); @@ -714,6 +757,24 @@ describe('ContextTokenization', function() { }); }); + it('builds edge windows for the start of context with deletion edits (1) - SMP strings', () => { + const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'].map(s => toMathematicalSMP(s)); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t)), null); + + const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 2 }, true, editWindowSpec); + assert.deepEqual(results, { + retokenizationText: toMathematicalSMP(' apple a'), + editBoundary: { + isPartial: false, + omitsEmptyToken: false, + text: ' ', + tokenIndex: 1 + }, + deleteLengths: [2, 0], + sliceIndex: 5 + }); + }); + it('builds edge windows for the start of context with deletion edits (2)', () => { const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t)), null); @@ -807,6 +868,17 @@ describe('ContextTokenization', function() { assert.isTrue(resultTokenization.tokens[0].isPartial); }); + it('preserves tokenization patterns when word slides partially out of window - SMP strings', () => { + const baseTokens = ['apples', ' ', 'and', ' ', 'bananas'].map(s => toMathematicalSMP(s)); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t)), null); + + const resultTokenization = baseTokenization.applyContextSlide(plainModel, { insert: '', deleteLeft: 0, deleteRight: 2}); + + assert.notStrictEqual(resultTokenization, baseTokenization); + assert.deepEqual(resultTokenization.exampleInput, ['ples', ' ', 'and', ' ', 'bananas'].map(s => toMathematicalSMP(s))); + assert.isTrue(resultTokenization.tokens[0].isPartial); + }); + it('does not preserve deleted tokens', () => { const baseTokens = ['apples', ' ', 'and', ' ', 'bananas']; const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t)), null);