From 10ccc954efe801a8e4b6f18d1037a4cf14fee076 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 29 Sep 2025 10:27:25 -0500 Subject: [PATCH] feat(web): add non-BMP test for edge-window-slide suite --- .../context/context-tokenization.tests.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index 4c0426db3b..4c3ebc10e5 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -868,6 +868,17 @@ describe('ContextTokenization', function() { assert.isTrue(resultTokenization.tokens[0].isPartial); }); + it('preserves tokenization patterns when word slides partially out of window - SMP strings', () => { + const baseTokens = ['apples', ' ', 'and', ' ', 'bananas'].map(s => toMathematicalSMP(s)); + const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t)), null); + + const resultTokenization = baseTokenization.applyContextSlide(plainModel, { insert: '', deleteLeft: 0, deleteRight: 2}); + + assert.notStrictEqual(resultTokenization, baseTokenization); + assert.deepEqual(resultTokenization.exampleInput, ['ples', ' ', 'and', ' ', 'bananas'].map(s => toMathematicalSMP(s))); + assert.isTrue(resultTokenization.tokens[0].isPartial); + }); + it('does not preserve deleted tokens', () => { const baseTokens = ['apples', ' ', 'and', ' ', 'bananas']; const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t)), null);