diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts index 3db2f82b12..1a8cfc8538 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts @@ -8,7 +8,6 @@ */ import { LexicalModelTypes } from '@keymanapp/common-types'; -import { deepCopy, KMWString } from "@keymanapp/web-utils"; import { SearchQuotientNode, PathInputProperties } from "./search-quotient-node.js"; import { TokenSplitMap } from "./context-tokenization.js"; @@ -109,7 +108,6 @@ export class ContextToken { textToCharTransforms(rawText).forEach((transform) => { let inputMetadata: PathInputProperties = { segment: { - trueTransform: transform, start: 0, transitionId: undefined }, @@ -235,27 +233,4 @@ export class ContextToken { return tokensFromSplit; } -} - -export function preprocessInputSources(inputSources: ReadonlyArray) { - const alteredSources = deepCopy(inputSources); - let trickledDeleteLeft = 0; - for(let i = alteredSources.length - 1; i >= 0; i--) { - const source = alteredSources[i]; - if(trickledDeleteLeft) { - const insLen = KMWString.length(source.segment.trueTransform.insert); - if(insLen <= trickledDeleteLeft) { - source.segment.trueTransform.insert = ''; - trickledDeleteLeft -= insLen; - } else { - source.segment.trueTransform.insert = KMWString.substring(source.segment.trueTransform.insert, 0, insLen - trickledDeleteLeft); - trickledDeleteLeft = 0; - } - } - trickledDeleteLeft += source.segment.trueTransform.deleteLeft; - source.segment.trueTransform.deleteLeft = 0; - } - - alteredSources[0].segment.trueTransform.deleteLeft = trickledDeleteLeft; - return alteredSources; } \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 89857fecc6..0fbc3494a6 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -590,7 +590,6 @@ export class ContextTokenization { const inputSource: PathInputProperties = { segment: { - trueTransform: sourceInput, transitionId: sourceInput.id, start: appliedLength }, diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-node.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-node.ts index c8f6241fa4..780bdadaa9 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-node.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-node.ts @@ -42,14 +42,6 @@ type CompleteSearchPath = { export type PathResult = NullPath | IntermediateSearchPath | CompleteSearchPath; export interface InputSegment { - /** - * The Transform corresponding to the keystroke applied to the true context - * for this input event. - * - * @deprecated Slated for removal within epic/autocorrect. - */ - trueTransform: Transform; - /** * The transform / transition ID of the corresponding input event. */ @@ -191,12 +183,11 @@ export interface SearchQuotientNode { readonly bestExample: { text: string, p: number }; /** - * Gets components useful for building a string-based representation of the - * keystroke range corrected by this search space. - * - * TODO: will return only the `inputSegment` part of each entry in the future. + * Gets components representing the keystroke range corrected by this search + * space. If only part of any keystroke's effects are used, this will also + * be noted. */ - readonly inputSegments: PathInputProperties[]; + readonly inputSegments: InputSegment[]; /** * Gets a compact string-based representation of `inputRange` that diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-root.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-root.ts index 2b263a4d85..2af16de8c8 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-root.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-root.ts @@ -2,7 +2,7 @@ import { LexicalModelTypes } from '@keymanapp/common-types'; import { SearchNode, SearchResult } from './distance-modeler.js'; -import { generateSpaceSeed, PathInputProperties, PathResult, SearchQuotientNode } from './search-quotient-node.js'; +import { generateSpaceSeed, InputSegment, PathResult, SearchQuotientNode } from './search-quotient-node.js'; import { SearchQuotientSpur } from './search-quotient-spur.js'; import LexicalModel = LexicalModelTypes.LexicalModel; @@ -90,7 +90,7 @@ export class SearchQuotientRoot implements SearchQuotientNode { } // Return a new array each time; avoid aliasing potential! - get inputSegments(): PathInputProperties[] { + get inputSegments(): InputSegment[] { return []; } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts index 0f88e012c2..e1b1c87598 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts @@ -13,7 +13,7 @@ import { LexicalModelTypes } from '@keymanapp/common-types'; import { buildMergedTransform } from '@keymanapp/models-templates'; import { EDIT_DISTANCE_COST_SCALE, SearchNode, SearchResult } from './distance-modeler.js'; -import { generateSpaceSeed, PathResult, SearchQuotientNode, PathInputProperties } from './search-quotient-node.js'; +import { generateSpaceSeed, InputSegment, PathInputProperties, PathResult, SearchQuotientNode } from './search-quotient-node.js'; import { generateSubsetId } from './tokenization-subsets.js'; import { SearchQuotientRoot } from './search-quotient-root.js'; import { LegacyQuotientRoot } from './legacy-quotient-root.js'; @@ -78,7 +78,6 @@ export abstract class SearchQuotientSpur implements SearchQuotientNode { const keystroke = inputSource as ProbabilityMass; inputSource = { segment: { - trueTransform: keystroke.sample, transitionId: keystroke.sample.id, start: 0 }, @@ -422,29 +421,29 @@ export abstract class SearchQuotientSpur implements SearchQuotientNode { return Object.values(this.returnedValues ?? {}).map(v => new SearchResult(v)); } - public get inputSegments(): PathInputProperties[] { + public get inputSegments(): InputSegment[] { if(!this.parentNode) { return []; } - const parentSources = this.parentNode.inputSegments; + const segments = this.parentNode.inputSegments; if(this.inputSource) { const inputId = this.inputSource.segment.transitionId; - if(inputId !== undefined && parentSources.length > 0 && parentSources[parentSources.length - 1].segment.transitionId == inputId) { + if(inputId !== undefined && segments.length > 0 && segments[segments.length - 1].transitionId == inputId) { // Fuse the input sources! - const tailSrc = parentSources.pop(); + const tailSegment = segments.pop(); // Deep-copy the object and replace the segment end value. - const extendedTailSrc = {...tailSrc, segment: {...tailSrc.segment, end: this.inputSource.segment.end}}; - if(extendedTailSrc.segment.end) { - delete extendedTailSrc.segment.end; + const extendedTailSegment = {...tailSegment, end: this.inputSource.segment.end}; + if(extendedTailSegment.end) { + delete extendedTailSegment.end; } - parentSources.push(extendedTailSrc); + segments.push(extendedTailSegment); } else { - parentSources.push(this.inputSource); + segments.push(this.inputSource.segment); } } - return parentSources; + return segments; } /** @@ -453,17 +452,17 @@ export abstract class SearchQuotientSpur implements SearchQuotientNode { */ get sourceRangeKey(): string { const components: string[] = []; - const sources = this.inputSegments; + const segments = this.inputSegments; - for(const source of sources) { - const i = source.segment.start; - const j = source.segment.end; - let component = (`T${source.segment.transitionId}`); + for(const segment of segments) { + const i = segment.start; + const j = segment.end; + let component = (`T${segment.transitionId}`); const parentSegs = this.parentNode.inputSegments; // It is possible for an .end to be 0 after a split - if an input's // left-deletions are applied without applying any of its insert string. - const midInputStart = i != 0 || parentSegs[parentSegs.length - 1]?.segment.end !== undefined; + const midInputStart = i != 0 || parentSegs[parentSegs.length - 1]?.end !== undefined; // If there's an entry for end, always include the start position if(j !== undefined) { diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts index cdb4bb7b54..cfb9fa0903 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts @@ -15,7 +15,7 @@ import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs' import { LexicalModelTypes } from '@keymanapp/common-types'; import { KMWString } from '@keymanapp/web-utils'; -import { ContextToken, correction, generateSubsetId, getBestMatches, models, PathInputProperties, preprocessInputSources, SearchQuotientSpur, unitTestEndpoints } from '@keymanapp/lm-worker/test-index'; +import { ContextToken, correction, generateSubsetId, getBestMatches, InputSegment, models, SearchQuotientSpur, unitTestEndpoints } from '@keymanapp/lm-worker/test-index'; import Distribution = LexicalModelTypes.Distribution; import ExecutionTimer = correction.ExecutionTimer; @@ -133,7 +133,6 @@ describe('ContextToken', function() { token1.addInput({ segment: { - trueTransform: srcTransform, transitionId: srcTransform.id, start: 0 }, @@ -143,7 +142,6 @@ describe('ContextToken', function() { token2.addInput({ segment: { - trueTransform: srcTransform, transitionId: srcTransform.id, start: 3 }, @@ -153,7 +151,6 @@ describe('ContextToken', function() { token3.addInput({ segment: { - trueTransform: srcTransform, transitionId: srcTransform.id, start: 4 }, @@ -164,12 +161,8 @@ describe('ContextToken', function() { const merged = ContextToken.merge([token1, token2, token3]); assert.equal(merged.exampleInput, "can't"); assert.deepEqual(merged.inputSegments, [ { - segment: { - trueTransform: srcTransform, - transitionId: srcTransform.id, - start: 0 - }, bestProbFromSet: 1, - subsetId: srcSubsetId + transitionId: srcTransform.id, + start: 0 } ]); assert.equal(merged.searchModule.inputCount, 1); assert.deepEqual((merged.searchModule as SearchQuotientSpur).lastInput, [{sample: srcTransform, p: 1}]); @@ -203,7 +196,6 @@ describe('ContextToken', function() { token1.addInput({ segment: { - trueTransform: srcTransforms[0], transitionId: srcTransforms[0].id, start: 0 }, @@ -212,7 +204,6 @@ describe('ContextToken', function() { }, [{sample: srcTransforms[0], p: 1}]); token1.addInput({ segment: { - trueTransform: srcTransforms[1], transitionId: srcTransforms[1].id, start: 0 }, @@ -222,7 +213,6 @@ describe('ContextToken', function() { token2.addInput({ segment: { - trueTransform: srcTransforms[1], transitionId: srcTransforms[1].id, start: 1 }, @@ -232,7 +222,6 @@ describe('ContextToken', function() { token3.addInput({ segment: { - trueTransform: srcTransforms[1], transitionId: srcTransforms[1].id, start: 4 }, @@ -241,7 +230,6 @@ describe('ContextToken', function() { }, [{sample: {insert: 's', deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}]); token3.addInput({ segment: { - trueTransform: srcTransforms[2], transitionId: srcTransforms[2].id, start: 0 }, @@ -251,7 +239,6 @@ describe('ContextToken', function() { token4.addInput({ segment: { - trueTransform: srcTransforms[3], transitionId: srcTransforms[3].id, start: 0 }, @@ -261,13 +248,9 @@ describe('ContextToken', function() { const merged = ContextToken.merge(tokensToMerge); assert.equal(merged.exampleInput, "applesandsourgrapes"); - assert.deepEqual(merged.inputSegments, srcTransforms.map((t, i) => ({ - segment: { - trueTransform: t, - transitionId: t.id, - start: 0 - }, bestProbFromSet: 1, - subsetId: srcSubsetIds[i] + assert.deepEqual(merged.inputSegments, srcTransforms.map((t, i) => ({ + transitionId: t.id, + start: 0 }))); assert.isTrue(quotientPathHasInputs( merged.searchModule, @@ -303,7 +286,6 @@ describe('ContextToken', function() { token1.addInput({ segment: { - trueTransform: srcTransforms[0], transitionId: srcTransforms[0].id, start: 0 }, @@ -312,7 +294,6 @@ describe('ContextToken', function() { }, [{sample: srcTransforms[0], p: 1}]); token1.addInput({ segment: { - trueTransform: srcTransforms[1], transitionId: srcTransforms[1].id, start: 0 }, @@ -322,7 +303,6 @@ describe('ContextToken', function() { token2.addInput({ segment: { - trueTransform: srcTransforms[1], transitionId: srcTransforms[1].id, start: 1 }, @@ -332,7 +312,6 @@ describe('ContextToken', function() { token3.addInput({ segment: { - trueTransform: srcTransforms[1], transitionId: srcTransforms[1].id, start: 4 }, @@ -341,7 +320,6 @@ describe('ContextToken', function() { }, [{sample: {insert: toMathematicalSMP('s'), deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}]); token3.addInput({ segment: { - trueTransform: srcTransforms[2], transitionId: srcTransforms[2].id, start: 0 }, @@ -351,7 +329,6 @@ describe('ContextToken', function() { token4.addInput({ segment: { - trueTransform: srcTransforms[3], transitionId: srcTransforms[3].id, start: 0 }, @@ -362,12 +339,8 @@ describe('ContextToken', function() { const merged = ContextToken.merge(tokensToMerge); assert.equal(merged.exampleInput, toMathematicalSMP("applesandsourgrapes")); assert.deepEqual(merged.inputSegments, srcTransforms.map((t, i) => ({ - segment: { - trueTransform: t, - transitionId: t.id, - start: 0 - }, bestProbFromSet: 1, - subsetId: srcSubsetIds[i] + transitionId: t.id, + start: 0 }))); assert.isTrue(quotientPathHasInputs( merged.searchModule, @@ -402,7 +375,6 @@ describe('ContextToken', function() { for(let i = 0; i < keystrokeDistributions.length; i++) { tokenToSplit.addInput({ segment: { - trueTransform: keystrokeDistributions[i][0].sample, transitionId: keystrokeDistributions[i][0].sample.id, start: 0 }, bestProbFromSet: .75, @@ -446,7 +418,6 @@ describe('ContextToken', function() { for(let i = 0; i < keystrokeDistributions.length; i++) { tokenToSplit.addInput({ segment: { - trueTransform: keystrokeDistributions[i][0].sample, transitionId: keystrokeDistributions[i][0].sample.id, start: 0 }, @@ -476,26 +447,16 @@ describe('ContextToken', function() { assert.sameOrderedMembers(resultsOfSplit.map(t => t.exampleInput), splitTextArray); const offsets = [0, 3, 8]; assert.sameDeepOrderedMembers(resultsOfSplit.map(t => t.inputSegments[0]), [0, 1, 2].map(i => { - const inputSource: PathInputProperties = { - segment: { - trueTransform: { - insert: 'biglargetransform', - id: keystrokeDistributions[0][0].sample.id, - deleteLeft: 0, - deleteRight: 0 - }, - transitionId: keystrokeDistributions[0][0].sample.id, - start: offsets[i] - }, - bestProbFromSet: 1, - subsetId + const segment: InputSegment = { + transitionId: keystrokeDistributions[0][0].sample.id, + start: offsets[i] }; if(offsets[i+1] !== undefined) { - inputSource.segment.end = offsets[i+1]; + segment.end = offsets[i+1]; } - return inputSource; + return segment; })); for(let i = 0; i < resultsOfSplit.length; i++) { @@ -528,7 +489,6 @@ describe('ContextToken', function() { for(let i = 0; i < keystrokeDistributions.length; i++) { tokenToSplit.addInput({ segment: { - trueTransform: keystrokeDistributions[i][0].sample, transitionId: keystrokeDistributions[i][0].sample.id, start: 0 }, @@ -558,53 +518,28 @@ describe('ContextToken', function() { assert.sameOrderedMembers(resultsOfSplit.map(t => t.exampleInput), splitTextArray); assert.deepEqual(resultsOfSplit[0].inputSegments, [ { - segment: { - trueTransform: keystrokeDistributions[0][0].sample, - transitionId: keystrokeDistributions[0][0].sample.id, - start: 0 - }, - bestProbFromSet: 1, - subsetId: subsetIds[0] + transitionId: keystrokeDistributions[0][0].sample.id, + start: 0 }, { - segment: { - trueTransform: keystrokeDistributions[1][0].sample, - transitionId: keystrokeDistributions[1][0].sample.id, - start: 0, - end: 'arge'.length - }, - bestProbFromSet: 1, - subsetId: subsetIds[1] + transitionId: keystrokeDistributions[1][0].sample.id, + start: 0, + end: 'arge'.length }, ]); assert.deepEqual(resultsOfSplit[1].inputSegments, [ { - segment: { - trueTransform: keystrokeDistributions[1][0].sample, - transitionId: keystrokeDistributions[1][0].sample.id, - start: 'arge'.length - }, - bestProbFromSet: 1, - subsetId: subsetIds[1] + transitionId: keystrokeDistributions[1][0].sample.id, + start: 'arge'.length }, { - segment: { - trueTransform: keystrokeDistributions[2][0].sample, - transitionId: keystrokeDistributions[2][0].sample.id, - start: 0, - end: 'ng'.length - }, - bestProbFromSet: 1, - subsetId: subsetIds[2] + transitionId: keystrokeDistributions[2][0].sample.id, + start: 0, + end: 'ng'.length }, ]); assert.deepEqual(resultsOfSplit[2].inputSegments, [ { - segment: { - trueTransform: keystrokeDistributions[2][0].sample, - transitionId: keystrokeDistributions[2][0].sample.id, - start: 'ng'.length, - }, - bestProbFromSet: 1, - subsetId: subsetIds[2] + transitionId: keystrokeDistributions[2][0].sample.id, + start: 'ng'.length, } ]); @@ -681,7 +616,6 @@ describe('ContextToken', function() { for(let i = 0; i < keystrokeDistributions.length; i++) { tokenToSplit.addInput({ segment: { - trueTransform: keystrokeDistributions[i][0].sample, transitionId: keystrokeDistributions[i][0].sample.id, start: 0 }, @@ -710,51 +644,26 @@ describe('ContextToken', function() { assert.equal(resultsOfSplit.length, 3); assert.sameOrderedMembers(resultsOfSplit.map(t => t.exampleInput), splitTextArray); assert.deepEqual(resultsOfSplit[0].inputSegments, [{ - segment: { - trueTransform: keystrokeDistributions[0][0].sample, - transitionId: keystrokeDistributions[0][0].sample.id, - start: 0 - }, - bestProbFromSet: 1, - subsetId: subsetIds[0] + transitionId: keystrokeDistributions[0][0].sample.id, + start: 0 }, { - segment: { - trueTransform: keystrokeDistributions[1][0].sample, - transitionId: keystrokeDistributions[1][0].sample.id, - start: 0, - end: 'arge'.length - }, - bestProbFromSet: 1, - subsetId: subsetIds[1] + transitionId: keystrokeDistributions[1][0].sample.id, + start: 0, + end: 'arge'.length }, ]); assert.deepEqual(resultsOfSplit[1].inputSegments, [{ - segment: { - trueTransform: keystrokeDistributions[1][0].sample, - transitionId: keystrokeDistributions[1][0].sample.id, - start: 'arge'.length - }, - bestProbFromSet: 1, - subsetId: subsetIds[1] + transitionId: keystrokeDistributions[1][0].sample.id, + start: 'arge'.length }, { - segment: { - trueTransform: keystrokeDistributions[2][0].sample, - transitionId: keystrokeDistributions[2][0].sample.id, - start: 0, - end: 'ng'.length - }, - bestProbFromSet: 1, - subsetId: subsetIds[2] + transitionId: keystrokeDistributions[2][0].sample.id, + start: 0, + end: 'ng'.length } ]); assert.deepEqual(resultsOfSplit[2].inputSegments, [{ - segment: { - trueTransform: keystrokeDistributions[2][0].sample, - transitionId: keystrokeDistributions[2][0].sample.id, - start: 'ng'.length - }, - bestProbFromSet: 1, - subsetId: subsetIds[2] + transitionId: keystrokeDistributions[2][0].sample.id, + start: 'ng'.length }]); assert.isTrue(quotientPathHasInputs( @@ -808,34 +717,4 @@ describe('ContextToken', function() { )); }); }); -}); - -describe('preprocessInputSources', () => { - it('properly preprocesses deleteLefts in the transforms', () => { - const transforms: Transform[] = [ - { insert: 'long', deleteLeft: 0, deleteRight: 0, id: 11 }, - { insert: 'argelovely', deleteLeft: 3, deleteRight: 0, id: 12 }, - { insert: 'ngtransforms', deleteLeft: 4, deleteRight: 0, id: 13 } - ]; - - const subsetIds = [ - generateSubsetId(), - generateSubsetId(), - generateSubsetId() - ]; - - const results = preprocessInputSources(transforms.map((t, i) => ({ - segment: { - trueTransform: t, - transitionId: t.id, - start: 0 - }, - bestProbFromSet: 1, - subsetId: subsetIds[i] - }))); - - assert.equal(results.length, transforms.length); - assert.sameOrderedMembers(results.map((entry) => entry.segment.trueTransform.insert), ['l', 'argelo', 'ngtransforms']); - assert.sameOrderedMembers(results.map((entry) => entry.segment.trueTransform.deleteLeft), [0, 0, 0]); - }); }); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index 0f2ff333ce..925412f12b 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -52,7 +52,6 @@ function toTransformToken(text: string, transformId?: number) { const textAsTransform = { insert: text, deleteLeft: 0, id: idSeed }; token.addInput({ segment: { - trueTransform: textAsTransform, transitionId: textAsTransform.id, start: 0 }, bestProbFromSet: 1, @@ -476,34 +475,21 @@ describe('ContextTokenization', function() { const boundaryToken = tokenization.tokens[tokenization.tokens.length-3]; const boundaryTailInput = boundaryToken.inputSegments[boundaryToken.inputSegments.length - 1]; assert.deepEqual(boundaryTailInput, { - segment: { - trueTransform: inputTransform, - transitionId: inputTransform.id, - start: 0, - end: 0 - }, bestProbFromSet: 1, - subsetId + transitionId: inputTransform.id, + start: 0, + end: 0 }); // The new tail tokens should not include anything from the original tail; // the token should be replaced. assert.deepEqual(tokenization.tokens[tokenization.tokens.length-2].inputSegments, [{ - segment: { - trueTransform: inputTransform, - transitionId: inputTransform.id, - start: 0, - end: 1 // captured the leading whitespace insert - }, bestProbFromSet: 1, - subsetId + transitionId: inputTransform.id, + start: 0, + end: 1 // captured the leading whitespace insert }]); assert.deepEqual(tokenization.tokens[tokenization.tokens.length-1].inputSegments, [{ - segment: { - trueTransform: inputTransform, - transitionId: inputTransform.id, - start: 1 - }, - bestProbFromSet: 1, - subsetId + transitionId: inputTransform.id, + start: 1 }]); const tailIndex = tokenization.tokens.length - 1; diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts index a710b4cbd1..986fa61ea5 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts @@ -183,11 +183,7 @@ describe('precomputationSubsetKeyer', function() { // source text: 'date' token.addInput({ segment: { - trueTransform: { - insert: 'te', - deleteLeft: 0, - id: 13 - }, transitionId: 13, + transitionId: 13, start: 0 }, bestProbFromSet: 1, subsetId: generateSubsetId() @@ -219,11 +215,7 @@ describe('precomputationSubsetKeyer', function() { // source text: 'date' token.addInput({ segment: { - trueTransform: { - insert: 'te', - deleteLeft: 0, - id: 13 - }, transitionId: 13, + transitionId: 13, start: 0 }, bestProbFromSet: 1, subsetId: generateSubsetId() @@ -269,11 +261,7 @@ describe('precomputationSubsetKeyer', function() { // source text: 'dat' token.addInput({ segment: { - trueTransform: { - insert: 't', - deleteLeft: 0, - id: 13 - }, transitionId: 13, + transitionId: 13, start: 0 }, bestProbFromSet: 1, subsetId: generateSubsetId() @@ -305,11 +293,7 @@ describe('precomputationSubsetKeyer', function() { // source text: 'dat' token.addInput({ segment: { - trueTransform: { - insert: 't', - deleteLeft: 0, - id: 13 - }, transitionId: 13, + transitionId: 13, start: 0 }, bestProbFromSet: 1, subsetId: generateSubsetId() @@ -766,11 +750,7 @@ describe('TokenizationSubsetBuilder', function() { const fourCharTailToken = new ContextToken(baseTokenization.tail); fourCharTailToken.addInput({ segment: { - trueTransform: { - insert: 'é', - deleteLeft: 1, - id: 13 - }, transitionId: 13, + transitionId: 13, start: 0 }, bestProbFromSet: 1, subsetId: generateSubsetId() @@ -781,11 +761,7 @@ describe('TokenizationSubsetBuilder', function() { const fiveCharTailToken = new ContextToken(baseTokenization.tail); fiveCharTailToken.addInput({ segment: { - trueTransform: { - insert: 'é', - deleteLeft: 1, - id: 13 - }, transitionId: 13, + transitionId: 13, start: 0 }, bestProbFromSet: 1, subsetId: generateSubsetId() @@ -823,11 +799,7 @@ describe('TokenizationSubsetBuilder', function() { const twoCharTailToken = new ContextToken(baseTokenization.tail); twoCharTailToken.addInput({ segment: { - trueTransform: { - insert: 'é', - deleteLeft: 1, - id: 13 - }, transitionId: 13, + transitionId: 13, start: 0 }, bestProbFromSet: .6, subsetId: generateSubsetId() @@ -838,10 +810,7 @@ describe('TokenizationSubsetBuilder', function() { const threeCharTailToken = new ContextToken(baseTokenization.tail); threeCharTailToken.addInput({ segment: { - trueTransform: { - insert: 'é', - deleteLeft: 1 - }, transitionId: 13, + transitionId: 13, start: 0 }, bestProbFromSet: .6, subsetId: generateSubsetId() diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-spur.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-spur.tests.ts index e5f1eb6088..b3931bad9d 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-spur.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-spur.tests.ts @@ -116,14 +116,8 @@ describe('SearchQuotientSpur', () => { assert.deepEqual(extendedPath.inputs, leadEdgeDistribution); assert.deepEqual(extendedPath.inputSegments, [ { - segment: { - trueTransform: leadEdgeDistribution[0].sample, - transitionId: leadEdgeDistribution[0].sample.id, - start: 0 - }, - bestProbFromSet: leadEdgeDistribution[0].p, - // Just write in the variable-value entry; the rest should match perfectly. - subsetId: extendedPath.inputSegments[0].subsetId + transitionId: leadEdgeDistribution[0].sample.id, + start: 0 } ]); @@ -174,23 +168,11 @@ describe('SearchQuotientSpur', () => { assert.deepEqual(length2Path.inputs, tailEdgeDistribution); assert.deepEqual(length2Path.inputSegments, [ { - segment: { - trueTransform: leadEdgeDistribution[0].sample, - transitionId: leadEdgeDistribution[0].sample.id, - start: 0 - }, - bestProbFromSet: leadEdgeDistribution[0].p, - // Just write in the variable-value entry; the rest should match perfectly. - subsetId: length2Path.inputSegments[0].subsetId + transitionId: leadEdgeDistribution[0].sample.id, + start: 0 }, { - segment: { - trueTransform: tailEdgeDistribution[0].sample, - transitionId: tailEdgeDistribution[0].sample.id, - start: 0 - }, - bestProbFromSet: tailEdgeDistribution[0].p, - // Just write in the variable-value entry; the rest should match perfectly. - subsetId: length2Path.inputSegments[1].subsetId + transitionId: tailEdgeDistribution[0].sample.id, + start: 0 } ]); @@ -258,23 +240,11 @@ describe('SearchQuotientSpur', () => { assert.deepEqual(length2Path.inputs, tailEdgeDistribution); assert.deepEqual(length2Path.inputSegments, [ { - segment: { - trueTransform: leadEdgeDistribution[0].sample, - transitionId: leadEdgeDistribution[0].sample.id, - start: 0 - }, - bestProbFromSet: leadEdgeDistribution[0].p, - // Just write in the variable-value entry; the rest should match perfectly. - subsetId: length2Path.inputSegments[0].subsetId + transitionId: leadEdgeDistribution[0].sample.id, + start: 0 }, { - segment: { - trueTransform: tailEdgeDistribution[0].sample, - transitionId: tailEdgeDistribution[0].sample.id, - start: 0 - }, - bestProbFromSet: tailEdgeDistribution[0].p, - // Just write in the variable-value entry; the rest should match perfectly. - subsetId: length2Path.inputSegments[1].subsetId + transitionId: tailEdgeDistribution[0].sample.id, + start: 0 } ]); @@ -1285,7 +1255,6 @@ describe('SearchQuotientSpur', () => { const headTarget = new LegacyQuotientSpur( path, headDistributionSplit, { segment: { - trueTransform: inputDistribution[0].sample, start: 0, transitionId: inputDistribution[0].sample.id }, @@ -1304,7 +1273,6 @@ describe('SearchQuotientSpur', () => { const tailTarget = new LegacyQuotientSpur( new LegacyQuotientRoot(testModel), tailDistributionSplit, { segment: { - trueTransform: inputDistribution[0].sample, start: 2, transitionId: inputDistribution[0].sample.id }, @@ -1377,7 +1345,6 @@ describe('SearchQuotientSpur', () => { const originalInputBase: PathInputProperties = { segment: { - trueTransform: {insert: 'biglargetransform', deleteLeft: 0, id: 11}, start: 0, transitionId: 11 }, @@ -1868,7 +1835,6 @@ describe('SearchQuotientSpur', () => { path, headDistributionSplit, { segment: { start: 0, - trueTransform: inputDistribution[0].sample, transitionId: inputDistribution[0].sample.id }, bestProbFromSet: inputDistribution[0].p, @@ -1887,7 +1853,6 @@ describe('SearchQuotientSpur', () => { new LegacyQuotientRoot(testModel), tailDistributionSplit, { segment: { start: 2, - trueTransform: inputDistribution[0].sample, transitionId: inputDistribution[0].sample.id }, bestProbFromSet: inputDistribution[0].p,