Merge remote-tracking branch 'origin/fix/web/sliding-tokenization-alignment' into fix/web/sliding-tokenization-alignment

This commit is contained in:
Joshua Horton 2025-08-19 12:06:21 -05:00
commit 307da5fef1

View file

@ -563,7 +563,7 @@ describe('ContextTokenization', function() {
"they'd", " ", "make", " ", "for", " ", "the", " ", "best", " ", "breakf"
];
// 73 chars above, vs a sliding window of length 64.
// 74 chars above, vs a sliding window of length 64.
assert.equal(baseContext3.reduce((accum, curr) => accum + curr.length, 0), 74);
// Actual window + one newly-typed character
assert.equal(incomingContext3.reduce((accum, curr) => accum + curr.length, 0), 65);