feat(web): add extra unit test

web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts

@@ -376,5 +376,44 @@ describe('ContextTokenization', function() {
       assert.equal(tokenization.tail.searchSpace.inputSequence.length, "thin".length);
       assert.sameOrderedMembers(tokenization.tokens.map(t => t.exampleInput), targetTexts);
     });
+
+    it('handles word-break boundary shifts at both ends during backward context-window slide', () => {
+      const baseTexts = [
+        // Without any preceding adjacent char in view, we can only interpret
+        // the leading `'` as an opening single-quote.
+        /*isn*/ "'", "t", " ", "orange", " ", "juice", " ", "tasty", "?", "  ", "I", " ", "find", " ", ""
+      ];
+
+      const baseTokenization = new ContextTokenization(baseTexts.map(t => toToken(t)), null);
+
+      const targetTexts = [
+        // With one preceding adjacent non-whitespace char in view, we now
+        // realize it was part of a word... and remove a wordbreak!
+        /*is"*/ "n't", " ", "orange", " ", "juice", " ", "tasty", "?", "  ", "I", " ", "find"
+      ];
+      const targetTokens = targetTexts.map((t) => ({text: t, isWhitespace: t == ' '}));
+      const inputTransformMap: Map<number, Transform> = new Map();
+
+      inputTransformMap.set(0, { insert: '', deleteLeft: 1 });
+
+      const tokenization = baseTokenization.transitionTo(
+        targetTokens, {
+          canAlign: true,
+          leadTokenShift: -1, // "'",
+          leadEditLength: 1, // "t" / "n't"
+          matchLength: baseTexts.length - 4,
+          tailEditLength: 0,
+          tailTokenShift: -2 // " ", ""
+        },
+        plainModel,
+        [{ sample: inputTransformMap, p: 1}]
+      );
+
+      assert.isOk(tokenization);
+      assert.equal(tokenization.tokens.length, targetTokens.length);
+      assert.equal(tokenization.tail.exampleInput, "find");
+      assert.equal(tokenization.tail.searchSpace.inputSequence.length, "find".length);
+      assert.sameOrderedMembers(tokenization.tokens.map(t => t.exampleInput), targetTexts);
+    });
   });
 });