diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index 3fc239eb1f..81f997dea5 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -15,7 +15,7 @@ import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs' import { LexicalModelTypes } from '@keymanapp/common-types'; import { KMWString } from '@keymanapp/web-utils'; -import { analyzePathMergesAndSplits, assembleTransforms, buildEdgeWindow, ContextToken, ContextTokenization, EditOperation, EditTuple, ExtendedEditOperation, models, traceInsertEdits } from '@keymanapp/lm-worker/test-index'; +import { analyzePathMergesAndSplits, assembleTransforms, buildEdgeWindow, ContextToken, ContextTokenization, EditOperation, EditTuple, ExtendedEditOperation, models, PendingTokenization, traceInsertEdits } from '@keymanapp/lm-worker/test-index'; import Transform = LexicalModelTypes.Transform; import TrieModel = models.TrieModel; @@ -61,6 +61,11 @@ function toMathematicalSMP(text: string) { return asSMP.join(''); } +const testEdgeWindowSpec = { + minTokens: 3, + minChars: 8 +}; + describe('ContextTokenization', function() { before(() => { KMWString.enableSupplementaryPlane(true); @@ -78,71 +83,76 @@ describe('ContextTokenization', function() { }); it("constructs from a token array + alignment data", () => { - // const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; - // let alignment: ContextStateAlignment = { - // canAlign: true, - // editPath: [ - // {op: 'match', input: 0, match: 0}, - // {op: 'match', input: 1, match: 1}, - // {op: 'match', input: 2, match: 2}, - // {op: 'match', input: 3, match: 3}, - // {op: 'match', input: 4, match: 4}, - // {op: 'match', input: 5, match: 5}, - // {op: 'match', input: 6, match: 6} - // ], - // leadTokenShift: 0, - // leadEditLength: 0, - // matchLength: 6, - // tailEditLength: 1, - // tailTokenShift: 0 - // }; + const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; + const tokens = rawTextTokens.map((text => toTransformToken(text))); + const emptyTransform = { insert: '', deleteLeft: 0, deleteRight: 0 }; - // let tokenization = new ContextTokenization(rawTextTokens.map((text => toToken(text))), alignment); + // We _could_ flesh this out a bit more... but it's not really needed for this test. + const edgeWindow = buildEdgeWindow(tokens, emptyTransform, false, testEdgeWindowSpec); + let transitionEdits: PendingTokenization = { + alignment: { + merges: [], + splits: [], + unmappedEdits: [], + edgeWindow: {...edgeWindow, retokenization: rawTextTokens.slice(edgeWindow.sliceIndex)}, + removedTokenCount: 0 + }, + inputs: [{sample: (() => { + const map = new Map(); + map.set(0, emptyTransform); + return map; + })(), p: 1}] + }; - // assert.deepEqual(tokenization.tokens.map((entry) => entry.exampleInput), rawTextTokens); - // assert.deepEqual(tokenization.tokens.map((entry) => entry.isWhitespace), rawTextTokens.map((entry) => entry == ' ')); - // assert.isOk(tokenization.alignment); - // assert.deepEqual(tokenization.alignment, alignment); - // assert.equal(tokenization.tail.exampleInput, 'day'); - // assert.isFalse(tokenization.tail.isWhitespace); + let tokenization = new ContextTokenization(tokens, transitionEdits, null /* dummy val */); + + assert.deepEqual(tokenization.tokens.map((entry) => entry.exampleInput), rawTextTokens); + assert.deepEqual(tokenization.tokens.map((entry) => entry.isWhitespace), rawTextTokens.map((entry) => entry == ' ')); + assert.isOk(tokenization.transitionEdits); + assert.deepEqual(tokenization.transitionEdits, transitionEdits); + assert.equal(tokenization.tail.exampleInput, 'day'); + assert.isFalse(tokenization.tail.isWhitespace); }); it('clones', () => { - // const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; + const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; + const tokens = rawTextTokens.map((text => toTransformToken(text))); + const emptyTransform = { insert: '', deleteLeft: 0, deleteRight: 0 }; - // let baseTokenization = new ContextTokenization(rawTextTokens.map((text => toToken(text))), { - // canAlign: true, - // editPath: [ - // {op: 'match', input: 0, match: 0}, - // {op: 'match', input: 1, match: 1}, - // {op: 'match', input: 2, match: 2}, - // {op: 'match', input: 3, match: 3}, - // {op: 'match', input: 4, match: 4}, - // {op: 'match', input: 5, match: 5}, - // {op: 'match', input: 6, match: 6} - // ], - // leadTokenShift: 0, - // leadEditLength: 0, - // matchLength: 6, - // tailEditLength: 1, - // tailTokenShift: 0 - // }); + // We _could_ flesh this out a bit more... but it's not really needed for this test. + const edgeWindow = buildEdgeWindow(tokens, emptyTransform, false, testEdgeWindowSpec); + let transitionEdits: PendingTokenization = { + alignment: { + merges: [], + splits: [], + unmappedEdits: [], + edgeWindow: {...edgeWindow, retokenization: rawTextTokens.slice(edgeWindow.sliceIndex)}, + removedTokenCount: 0 + }, + inputs: [{sample: (() => { + const map = new Map(); + map.set(0, emptyTransform); + return map; + })(), p: 1}] + }; - // let cloned = new ContextTokenization(baseTokenization); + let baseTokenization = new ContextTokenization(tokens, transitionEdits, null /* dummy val */); - // assert.notDeepEqual(cloned, baseTokenization); - // assert.deepEqual(cloned.tokens.map((token) => token.searchSpace.inputSequence), - // baseTokenization.tokens.map((token) => token.searchSpace.inputSequence)); + let cloned = new ContextTokenization(baseTokenization); - // // The `.searchSpace` instances will not be deep-equal; there are class properties - // // that hold functions with closures, configured at runtime. + assert.notDeepEqual(cloned, baseTokenization); + assert.deepEqual(cloned.tokens.map((token) => token.searchSpace.inputSequence), + baseTokenization.tokens.map((token) => token.searchSpace.inputSequence)); - // // @ts-ignore - TS2704 b/c deleting a readonly property. - // baseTokenization.tokens.forEach((token) => delete token.searchSpace); - // // @ts-ignore - TS2704 b/c deleting a readonly property. - // cloned.tokens.forEach((token) => delete token.searchSpace); + // The `.searchSpace` instances will not be deep-equal; there are class properties + // that hold functions with closures, configured at runtime. - // assert.deepEqual(cloned, baseTokenization); + // @ts-ignore - TS2704 b/c deleting a readonly property. + baseTokenization.tokens.forEach((token) => delete token.searchSpace); + // @ts-ignore - TS2704 b/c deleting a readonly property. + cloned.tokens.forEach((token) => delete token.searchSpace); + + assert.deepEqual(cloned, baseTokenization); }); }); @@ -154,11 +164,6 @@ describe('ContextTokenization', function() { }); describe('evaluateTransition', () => { - const testEdgeWindowSpec = { - minTokens: 3, - minChars: 8 - }; - it('handles simple case - new whitespace + new empty token', () => { const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day']; const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t))); @@ -631,17 +636,12 @@ describe('ContextTokenization', function() { describe('buildEdgeWindow', () => { describe('with min token count 3, char count 8', () => { - const editWindowSpec = { - minTokens: 3, - minChars: 8 - } - it('handles empty contexts', () => { const baseTokens = ['']; const idSeed = TOKEN_TRANSFORM_SEED; const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); - const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 0 }, true, editWindowSpec); + const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 0 }, true, testEdgeWindowSpec); assert.deepEqual(results, { retokenizationText: '', editBoundary: { @@ -661,7 +661,7 @@ describe('ContextTokenization', function() { const idSeed = TOKEN_TRANSFORM_SEED; const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); - const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 2 }, true, editWindowSpec); + const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 2 }, true, testEdgeWindowSpec); assert.deepEqual(results, { retokenizationText: '', editBoundary: { @@ -681,7 +681,7 @@ describe('ContextTokenization', function() { const idSeed = TOKEN_TRANSFORM_SEED; const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); - const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 0 }, true, editWindowSpec); + const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 0 }, true, testEdgeWindowSpec); assert.deepEqual(results, { retokenizationText: 'an apple', editBoundary: { @@ -700,7 +700,7 @@ describe('ContextTokenization', function() { const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'].map(s => toMathematicalSMP(s)); const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t))); - const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 0 }, true, editWindowSpec); + const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 0 }, true, testEdgeWindowSpec); assert.deepEqual(results, { retokenizationText: toMathematicalSMP('an apple'), editBoundary: { @@ -721,7 +721,7 @@ describe('ContextTokenization', function() { const idSeed = TOKEN_TRANSFORM_SEED; const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); - const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 2 }, true, editWindowSpec); + const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 2 }, true, testEdgeWindowSpec); assert.deepEqual(results, { retokenizationText: ' apple a', editBoundary: { @@ -740,7 +740,7 @@ describe('ContextTokenization', function() { const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'].map(s => toMathematicalSMP(s)); const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t))); - const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 2 }, true, editWindowSpec); + const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 2 }, true, testEdgeWindowSpec); assert.deepEqual(results, { retokenizationText: toMathematicalSMP(' apple a'), editBoundary: { @@ -761,7 +761,7 @@ describe('ContextTokenization', function() { const idSeed = TOKEN_TRANSFORM_SEED; const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); - const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 4 }, true, editWindowSpec); + const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 4 }, true, testEdgeWindowSpec); assert.deepEqual(results, { retokenizationText: 'pple a day', editBoundary: { @@ -782,7 +782,7 @@ describe('ContextTokenization', function() { const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); baseTokenization.tail.isPartial = true; - const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 0 }, false, editWindowSpec); + const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 0 }, false, testEdgeWindowSpec); assert.deepEqual(results, { retokenizationText: 'apple a day', editBoundary: { @@ -802,7 +802,7 @@ describe('ContextTokenization', function() { const idSeed = TOKEN_TRANSFORM_SEED; const baseTokenization = new ContextTokenization(baseTokens.map(t => toTransformToken(t))); - const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 0 }, false, editWindowSpec); + const results = buildEdgeWindow(baseTokenization.tokens, { insert: '', deleteLeft: 0, deleteRight: 0 }, false, testEdgeWindowSpec); assert.deepEqual(results, { retokenizationText: 'apple a day ', editBoundary: { diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts index d23cb2068c..e5c511adb7 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts @@ -104,7 +104,7 @@ describe('determineContextTransition', () => { assert.equal(transition, tracker.latest); assert.isFalse(warningEmitterSpy.called); assert.sameOrderedMembers(transition.final.tokenization.exampleInput, ['this', ' ', 'is', ' ', 'for', ' ', 'techn']); - // assert.isOk(transition.final.tokenization.alignment); + assert.isOk(transition.final.tokenization.transitionEdits); assert.equal(transition.final.context.left, targetContext.left); assert.equal(transition.final.context.right ?? "", targetContext.right ?? ""); assert.sameDeepOrderedMembers(transition.inputDistribution, inputDistribution);