erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-08-29 11:47:39 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 11

refactor(web): simplify inputSegment spec + related SearchQuotientNode property

Browse source 
Build-bot: skip build:web build:android build:ios
Test-bot: test
This commit is contained in:
Joshua Horton 2025-11-11 17:00:13 -06:00
parent f3fed20926
commit cff287fe9d
9 changed files with 86 additions and 323 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts
View file

@ -8,7 +8,6 @@
*/
import { LexicalModelTypes } from '@keymanapp/common-types';
import { deepCopy, KMWString } from "@keymanapp/web-utils";
import { SearchQuotientNode, PathInputProperties } from "./search-quotient-node.js";
import { TokenSplitMap } from "./context-tokenization.js";
@ -109,7 +108,6 @@ export class ContextToken {
textToCharTransforms(rawText).forEach((transform) => {
let inputMetadata: PathInputProperties = {
segment: {
trueTransform: transform,
start: 0,
transitionId: undefined
},
@ -235,27 +233,4 @@ export class ContextToken {
return tokensFromSplit;
}
}
export function preprocessInputSources(inputSources: ReadonlyArray<PathInputProperties>) {
const alteredSources = deepCopy(inputSources);
let trickledDeleteLeft = 0;
for(let i = alteredSources.length - 1; i >= 0; i--) {
const source = alteredSources[i];
if(trickledDeleteLeft) {
const insLen = KMWString.length(source.segment.trueTransform.insert);
if(insLen <= trickledDeleteLeft) {
source.segment.trueTransform.insert = '';
trickledDeleteLeft -= insLen;
} else {
source.segment.trueTransform.insert = KMWString.substring(source.segment.trueTransform.insert, 0, insLen - trickledDeleteLeft);
trickledDeleteLeft = 0;
}
}
trickledDeleteLeft += source.segment.trueTransform.deleteLeft;
source.segment.trueTransform.deleteLeft = 0;
}
alteredSources[0].segment.trueTransform.deleteLeft = trickledDeleteLeft;
return alteredSources;
}

1
web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
View file

@ -590,7 +590,6 @@ export class ContextTokenization {
const inputSource: PathInputProperties = {
segment: {
trueTransform: sourceInput,
transitionId: sourceInput.id,
start: appliedLength
},

17
web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-node.ts
View file

@ -42,14 +42,6 @@ type CompleteSearchPath = {
export type PathResult = NullPath | IntermediateSearchPath | CompleteSearchPath;
export interface InputSegment {
/**
* The Transform corresponding to the keystroke applied to the true context
* for this input event.
*
* @deprecated Slated for removal within epic/autocorrect.
*/
trueTransform: Transform;
/**
* The transform / transition ID of the corresponding input event.
*/
@ -191,12 +183,11 @@ export interface SearchQuotientNode {
readonly bestExample: { text: string, p: number };
/**
* Gets components useful for building a string-based representation of the
* keystroke range corrected by this search space.
*
* TODO: will return only the `inputSegment` part of each entry in the future.
* Gets components representing the keystroke range corrected by this search
* space. If only part of any keystroke's effects are used, this will also
* be noted.
*/
readonly inputSegments: PathInputProperties[];
readonly inputSegments: InputSegment[];
/**
* Gets a compact string-based representation of `inputRange` that

4
web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-root.ts
View file

@ -2,7 +2,7 @@
import { LexicalModelTypes } from '@keymanapp/common-types';
import { SearchNode, SearchResult } from './distance-modeler.js';
import { generateSpaceSeed, PathInputProperties, PathResult, SearchQuotientNode } from './search-quotient-node.js';
import { generateSpaceSeed, InputSegment, PathResult, SearchQuotientNode } from './search-quotient-node.js';
import { SearchQuotientSpur } from './search-quotient-spur.js';
import LexicalModel = LexicalModelTypes.LexicalModel;
@ -90,7 +90,7 @@ export class SearchQuotientRoot implements SearchQuotientNode {
}
// Return a new array each time; avoid aliasing potential!
get inputSegments(): PathInputProperties[] {
get inputSegments(): InputSegment[] {
return [];
}

35
web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts
View file

@ -13,7 +13,7 @@ import { LexicalModelTypes } from '@keymanapp/common-types';
import { buildMergedTransform } from '@keymanapp/models-templates';
import { EDIT_DISTANCE_COST_SCALE, SearchNode, SearchResult } from './distance-modeler.js';
import { generateSpaceSeed, PathResult, SearchQuotientNode, PathInputProperties } from './search-quotient-node.js';
import { generateSpaceSeed, InputSegment, PathInputProperties, PathResult, SearchQuotientNode } from './search-quotient-node.js';
import { generateSubsetId } from './tokenization-subsets.js';
import { SearchQuotientRoot } from './search-quotient-root.js';
import { LegacyQuotientRoot } from './legacy-quotient-root.js';
@ -78,7 +78,6 @@ export abstract class SearchQuotientSpur implements SearchQuotientNode {
const keystroke = inputSource as ProbabilityMass<Transform>;
inputSource = {
segment: {
trueTransform: keystroke.sample,
transitionId: keystroke.sample.id,
start: 0
},
@ -422,29 +421,29 @@ export abstract class SearchQuotientSpur implements SearchQuotientNode {
return Object.values(this.returnedValues ?? {}).map(v => new SearchResult(v));
}
public get inputSegments(): PathInputProperties[] {
public get inputSegments(): InputSegment[] {
if(!this.parentNode) {
return [];
}
const parentSources = this.parentNode.inputSegments;
const segments = this.parentNode.inputSegments;
if(this.inputSource) {
const inputId = this.inputSource.segment.transitionId;
if(inputId !== undefined && parentSources.length > 0 && parentSources[parentSources.length - 1].segment.transitionId == inputId) {
if(inputId !== undefined && segments.length > 0 && segments[segments.length - 1].transitionId == inputId) {
// Fuse the input sources!
const tailSrc = parentSources.pop();
const tailSegment = segments.pop();
// Deep-copy the object and replace the segment end value.
const extendedTailSrc = {...tailSrc, segment: {...tailSrc.segment, end: this.inputSource.segment.end}};
if(extendedTailSrc.segment.end) {
delete extendedTailSrc.segment.end;
const extendedTailSegment = {...tailSegment, end: this.inputSource.segment.end};
if(extendedTailSegment.end) {
delete extendedTailSegment.end;
}
parentSources.push(extendedTailSrc);
segments.push(extendedTailSegment);
} else {
parentSources.push(this.inputSource);
segments.push(this.inputSource.segment);
}
}
return parentSources;
return segments;
}
/**
@ -453,17 +452,17 @@ export abstract class SearchQuotientSpur implements SearchQuotientNode {
*/
get sourceRangeKey(): string {
const components: string[] = [];
const sources = this.inputSegments;
const segments = this.inputSegments;
for(const source of sources) {
const i = source.segment.start;
const j = source.segment.end;
let component = (`T${source.segment.transitionId}`);
for(const segment of segments) {
const i = segment.start;
const j = segment.end;
let component = (`T${segment.transitionId}`);
const parentSegs = this.parentNode.inputSegments;
// It is possible for an .end to be 0 after a split - if an input's
// left-deletions are applied without applying any of its insert string.
const midInputStart = i != 0 || parentSegs[parentSegs.length - 1]?.segment.end !== undefined;
const midInputStart = i != 0 || parentSegs[parentSegs.length - 1]?.end !== undefined;
// If there's an entry for end, always include the start position
if(j !== undefined) {

195
web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-token.tests.ts
View file

@ -15,7 +15,7 @@ import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'
import { LexicalModelTypes } from '@keymanapp/common-types';
import { KMWString } from '@keymanapp/web-utils';
import { ContextToken, correction, generateSubsetId, getBestMatches, models, PathInputProperties, preprocessInputSources, SearchQuotientSpur, unitTestEndpoints } from '@keymanapp/lm-worker/test-index';
import { ContextToken, correction, generateSubsetId, getBestMatches, InputSegment, models, SearchQuotientSpur, unitTestEndpoints } from '@keymanapp/lm-worker/test-index';
import Distribution = LexicalModelTypes.Distribution;
import ExecutionTimer = correction.ExecutionTimer;
@ -133,7 +133,6 @@ describe('ContextToken', function() {
token1.addInput({
segment: {
trueTransform: srcTransform,
transitionId: srcTransform.id,
start: 0
},
@ -143,7 +142,6 @@ describe('ContextToken', function() {
token2.addInput({
segment: {
trueTransform: srcTransform,
transitionId: srcTransform.id,
start: 3
},
@ -153,7 +151,6 @@ describe('ContextToken', function() {
token3.addInput({
segment: {
trueTransform: srcTransform,
transitionId: srcTransform.id,
start: 4
},
@ -164,12 +161,8 @@ describe('ContextToken', function() {
const merged = ContextToken.merge([token1, token2, token3]);
assert.equal(merged.exampleInput, "can't");
assert.deepEqual(merged.inputSegments, [ {
segment: {
trueTransform: srcTransform,
transitionId: srcTransform.id,
start: 0
}, bestProbFromSet: 1,
subsetId: srcSubsetId
transitionId: srcTransform.id,
start: 0
} ]);
assert.equal(merged.searchModule.inputCount, 1);
assert.deepEqual((merged.searchModule as SearchQuotientSpur).lastInput, [{sample: srcTransform, p: 1}]);
@ -203,7 +196,6 @@ describe('ContextToken', function() {
token1.addInput({
segment: {
trueTransform: srcTransforms[0],
transitionId: srcTransforms[0].id,
start: 0
},
@ -212,7 +204,6 @@ describe('ContextToken', function() {
}, [{sample: srcTransforms[0], p: 1}]);
token1.addInput({
segment: {
trueTransform: srcTransforms[1],
transitionId: srcTransforms[1].id,
start: 0
},
@ -222,7 +213,6 @@ describe('ContextToken', function() {
token2.addInput({
segment: {
trueTransform: srcTransforms[1],
transitionId: srcTransforms[1].id,
start: 1
},
@ -232,7 +222,6 @@ describe('ContextToken', function() {
token3.addInput({
segment: {
trueTransform: srcTransforms[1],
transitionId: srcTransforms[1].id,
start: 4
},
@ -241,7 +230,6 @@ describe('ContextToken', function() {
}, [{sample: {insert: 's', deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}]);
token3.addInput({
segment: {
trueTransform: srcTransforms[2],
transitionId: srcTransforms[2].id,
start: 0
},
@ -251,7 +239,6 @@ describe('ContextToken', function() {
token4.addInput({
segment: {
trueTransform: srcTransforms[3],
transitionId: srcTransforms[3].id,
start: 0
},
@ -261,13 +248,9 @@ describe('ContextToken', function() {
const merged = ContextToken.merge(tokensToMerge);
assert.equal(merged.exampleInput, "applesandsourgrapes");
assert.deepEqual(merged.inputSegments, srcTransforms.map((t, i) => ({
segment: {
trueTransform: t,
transitionId: t.id,
start: 0
}, bestProbFromSet: 1,
subsetId: srcSubsetIds[i]
assert.deepEqual(merged.inputSegments, srcTransforms.map((t, i) => ({
transitionId: t.id,
start: 0
})));
assert.isTrue(quotientPathHasInputs(
merged.searchModule,
@ -303,7 +286,6 @@ describe('ContextToken', function() {
token1.addInput({
segment: {
trueTransform: srcTransforms[0],
transitionId: srcTransforms[0].id,
start: 0
},
@ -312,7 +294,6 @@ describe('ContextToken', function() {
}, [{sample: srcTransforms[0], p: 1}]);
token1.addInput({
segment: {
trueTransform: srcTransforms[1],
transitionId: srcTransforms[1].id,
start: 0
},
@ -322,7 +303,6 @@ describe('ContextToken', function() {
token2.addInput({
segment: {
trueTransform: srcTransforms[1],
transitionId: srcTransforms[1].id,
start: 1
},
@ -332,7 +312,6 @@ describe('ContextToken', function() {
token3.addInput({
segment: {
trueTransform: srcTransforms[1],
transitionId: srcTransforms[1].id,
start: 4
},
@ -341,7 +320,6 @@ describe('ContextToken', function() {
}, [{sample: {insert: toMathematicalSMP('s'), deleteLeft: 0, deleteRight: 0, id: 2}, p: 1}]);
token3.addInput({
segment: {
trueTransform: srcTransforms[2],
transitionId: srcTransforms[2].id,
start: 0
},
@ -351,7 +329,6 @@ describe('ContextToken', function() {
token4.addInput({
segment: {
trueTransform: srcTransforms[3],
transitionId: srcTransforms[3].id,
start: 0
},
@ -362,12 +339,8 @@ describe('ContextToken', function() {
const merged = ContextToken.merge(tokensToMerge);
assert.equal(merged.exampleInput, toMathematicalSMP("applesandsourgrapes"));
assert.deepEqual(merged.inputSegments, srcTransforms.map((t, i) => ({
segment: {
trueTransform: t,
transitionId: t.id,
start: 0
}, bestProbFromSet: 1,
subsetId: srcSubsetIds[i]
transitionId: t.id,
start: 0
})));
assert.isTrue(quotientPathHasInputs(
merged.searchModule,
@ -402,7 +375,6 @@ describe('ContextToken', function() {
for(let i = 0; i < keystrokeDistributions.length; i++) {
tokenToSplit.addInput({
segment: {
trueTransform: keystrokeDistributions[i][0].sample,
transitionId: keystrokeDistributions[i][0].sample.id,
start: 0
}, bestProbFromSet: .75,
@ -446,7 +418,6 @@ describe('ContextToken', function() {
for(let i = 0; i < keystrokeDistributions.length; i++) {
tokenToSplit.addInput({
segment: {
trueTransform: keystrokeDistributions[i][0].sample,
transitionId: keystrokeDistributions[i][0].sample.id,
start: 0
},
@ -476,26 +447,16 @@ describe('ContextToken', function() {
assert.sameOrderedMembers(resultsOfSplit.map(t => t.exampleInput), splitTextArray);
const offsets = [0, 3, 8];
assert.sameDeepOrderedMembers(resultsOfSplit.map(t => t.inputSegments[0]), [0, 1, 2].map(i => {
const inputSource: PathInputProperties = {
segment: {
trueTransform: {
insert: 'biglargetransform',
id: keystrokeDistributions[0][0].sample.id,
deleteLeft: 0,
deleteRight: 0
},
transitionId: keystrokeDistributions[0][0].sample.id,
start: offsets[i]
},
bestProbFromSet: 1,
subsetId
const segment: InputSegment = {
transitionId: keystrokeDistributions[0][0].sample.id,
start: offsets[i]
};
if(offsets[i+1] !== undefined) {
inputSource.segment.end = offsets[i+1];
segment.end = offsets[i+1];
}
return inputSource;
return segment;
}));
for(let i = 0; i < resultsOfSplit.length; i++) {
@ -528,7 +489,6 @@ describe('ContextToken', function() {
for(let i = 0; i < keystrokeDistributions.length; i++) {
tokenToSplit.addInput({
segment: {
trueTransform: keystrokeDistributions[i][0].sample,
transitionId: keystrokeDistributions[i][0].sample.id,
start: 0
},
@ -558,53 +518,28 @@ describe('ContextToken', function() {
assert.sameOrderedMembers(resultsOfSplit.map(t => t.exampleInput), splitTextArray);
assert.deepEqual(resultsOfSplit[0].inputSegments, [
{
segment: {
trueTransform: keystrokeDistributions[0][0].sample,
transitionId: keystrokeDistributions[0][0].sample.id,
start: 0
},
bestProbFromSet: 1,
subsetId: subsetIds[0]
transitionId: keystrokeDistributions[0][0].sample.id,
start: 0
}, {
segment: {
trueTransform: keystrokeDistributions[1][0].sample,
transitionId: keystrokeDistributions[1][0].sample.id,
start: 0,
end: 'arge'.length
},
bestProbFromSet: 1,
subsetId: subsetIds[1]
transitionId: keystrokeDistributions[1][0].sample.id,
start: 0,
end: 'arge'.length
},
]);
assert.deepEqual(resultsOfSplit[1].inputSegments, [
{
segment: {
trueTransform: keystrokeDistributions[1][0].sample,
transitionId: keystrokeDistributions[1][0].sample.id,
start: 'arge'.length
},
bestProbFromSet: 1,
subsetId: subsetIds[1]
transitionId: keystrokeDistributions[1][0].sample.id,
start: 'arge'.length
}, {
segment: {
trueTransform: keystrokeDistributions[2][0].sample,
transitionId: keystrokeDistributions[2][0].sample.id,
start: 0,
end: 'ng'.length
},
bestProbFromSet: 1,
subsetId: subsetIds[2]
transitionId: keystrokeDistributions[2][0].sample.id,
start: 0,
end: 'ng'.length
},
]);
assert.deepEqual(resultsOfSplit[2].inputSegments, [
{
segment: {
trueTransform: keystrokeDistributions[2][0].sample,
transitionId: keystrokeDistributions[2][0].sample.id,
start: 'ng'.length,
},
bestProbFromSet: 1,
subsetId: subsetIds[2]
transitionId: keystrokeDistributions[2][0].sample.id,
start: 'ng'.length,
}
]);
@ -681,7 +616,6 @@ describe('ContextToken', function() {
for(let i = 0; i < keystrokeDistributions.length; i++) {
tokenToSplit.addInput({
segment: {
trueTransform: keystrokeDistributions[i][0].sample,
transitionId: keystrokeDistributions[i][0].sample.id,
start: 0
},
@ -710,51 +644,26 @@ describe('ContextToken', function() {
assert.equal(resultsOfSplit.length, 3);
assert.sameOrderedMembers(resultsOfSplit.map(t => t.exampleInput), splitTextArray);
assert.deepEqual(resultsOfSplit[0].inputSegments, [{
segment: {
trueTransform: keystrokeDistributions[0][0].sample,
transitionId: keystrokeDistributions[0][0].sample.id,
start: 0
},
bestProbFromSet: 1,
subsetId: subsetIds[0]
transitionId: keystrokeDistributions[0][0].sample.id,
start: 0
}, {
segment: {
trueTransform: keystrokeDistributions[1][0].sample,
transitionId: keystrokeDistributions[1][0].sample.id,
start: 0,
end: 'arge'.length
},
bestProbFromSet: 1,
subsetId: subsetIds[1]
transitionId: keystrokeDistributions[1][0].sample.id,
start: 0,
end: 'arge'.length
},
]);
assert.deepEqual(resultsOfSplit[1].inputSegments, [{
segment: {
trueTransform: keystrokeDistributions[1][0].sample,
transitionId: keystrokeDistributions[1][0].sample.id,
start: 'arge'.length
},
bestProbFromSet: 1,
subsetId: subsetIds[1]
transitionId: keystrokeDistributions[1][0].sample.id,
start: 'arge'.length
}, {
segment: {
trueTransform: keystrokeDistributions[2][0].sample,
transitionId: keystrokeDistributions[2][0].sample.id,
start: 0,
end: 'ng'.length
},
bestProbFromSet: 1,
subsetId: subsetIds[2]
transitionId: keystrokeDistributions[2][0].sample.id,
start: 0,
end: 'ng'.length
}
]);
assert.deepEqual(resultsOfSplit[2].inputSegments, [{
segment: {
trueTransform: keystrokeDistributions[2][0].sample,
transitionId: keystrokeDistributions[2][0].sample.id,
start: 'ng'.length
},
bestProbFromSet: 1,
subsetId: subsetIds[2]
transitionId: keystrokeDistributions[2][0].sample.id,
start: 'ng'.length
}]);
assert.isTrue(quotientPathHasInputs(
@ -808,34 +717,4 @@ describe('ContextToken', function() {
));
});
});
});
describe('preprocessInputSources', () => {
it('properly preprocesses deleteLefts in the transforms', () => {
const transforms: Transform[] = [
{ insert: 'long', deleteLeft: 0, deleteRight: 0, id: 11 },
{ insert: 'argelovely', deleteLeft: 3, deleteRight: 0, id: 12 },
{ insert: 'ngtransforms', deleteLeft: 4, deleteRight: 0, id: 13 }
];
const subsetIds = [
generateSubsetId(),
generateSubsetId(),
generateSubsetId()
];
const results = preprocessInputSources(transforms.map((t, i) => ({
segment: {
trueTransform: t,
transitionId: t.id,
start: 0
},
bestProbFromSet: 1,
subsetId: subsetIds[i]
})));
assert.equal(results.length, transforms.length);
assert.sameOrderedMembers(results.map((entry) => entry.segment.trueTransform.insert), ['l', 'argelo', 'ngtransforms']);
assert.sameOrderedMembers(results.map((entry) => entry.segment.trueTransform.deleteLeft), [0, 0, 0]);
});
});

30
web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts
View file

@ -52,7 +52,6 @@ function toTransformToken(text: string, transformId?: number) {
const textAsTransform = { insert: text, deleteLeft: 0, id: idSeed };
token.addInput({
segment: {
trueTransform: textAsTransform,
transitionId: textAsTransform.id,
start: 0
}, bestProbFromSet: 1,
@ -476,34 +475,21 @@ describe('ContextTokenization', function() {
const boundaryToken = tokenization.tokens[tokenization.tokens.length-3];
const boundaryTailInput = boundaryToken.inputSegments[boundaryToken.inputSegments.length - 1];
assert.deepEqual(boundaryTailInput, {
segment: {
trueTransform: inputTransform,
transitionId: inputTransform.id,
start: 0,
end: 0
}, bestProbFromSet: 1,
subsetId
transitionId: inputTransform.id,
start: 0,
end: 0
});
// The new tail tokens should not include anything from the original tail;
// the token should be replaced.
assert.deepEqual(tokenization.tokens[tokenization.tokens.length-2].inputSegments, [{
segment: {
trueTransform: inputTransform,
transitionId: inputTransform.id,
start: 0,
end: 1 // captured the leading whitespace insert
}, bestProbFromSet: 1,
subsetId
transitionId: inputTransform.id,
start: 0,
end: 1 // captured the leading whitespace insert
}]);
assert.deepEqual(tokenization.tokens[tokenization.tokens.length-1].inputSegments, [{
segment: {
trueTransform: inputTransform,
transitionId: inputTransform.id,
start: 1
},
bestProbFromSet: 1,
subsetId
transitionId: inputTransform.id,
start: 1
}]);
const tailIndex = tokenization.tokens.length - 1;

47
web/src/test/auto/headless/engine/predictive-text/worker-thread/context/tokenization-subsets.tests.ts
View file

@ -183,11 +183,7 @@ describe('precomputationSubsetKeyer', function() {
// source text: 'date'
token.addInput({
segment: {
trueTransform: {
insert: 'te',
deleteLeft: 0,
id: 13
}, transitionId: 13,
transitionId: 13,
start: 0
}, bestProbFromSet: 1,
subsetId: generateSubsetId()
@ -219,11 +215,7 @@ describe('precomputationSubsetKeyer', function() {
// source text: 'date'
token.addInput({
segment: {
trueTransform: {
insert: 'te',
deleteLeft: 0,
id: 13
}, transitionId: 13,
transitionId: 13,
start: 0
}, bestProbFromSet: 1,
subsetId: generateSubsetId()
@ -269,11 +261,7 @@ describe('precomputationSubsetKeyer', function() {
// source text: 'dat'
token.addInput({
segment: {
trueTransform: {
insert: 't',
deleteLeft: 0,
id: 13
}, transitionId: 13,
transitionId: 13,
start: 0
}, bestProbFromSet: 1,
subsetId: generateSubsetId()
@ -305,11 +293,7 @@ describe('precomputationSubsetKeyer', function() {
// source text: 'dat'
token.addInput({
segment: {
trueTransform: {
insert: 't',
deleteLeft: 0,
id: 13
}, transitionId: 13,
transitionId: 13,
start: 0
}, bestProbFromSet: 1,
subsetId: generateSubsetId()
@ -766,11 +750,7 @@ describe('TokenizationSubsetBuilder', function() {
const fourCharTailToken = new ContextToken(baseTokenization.tail);
fourCharTailToken.addInput({
segment: {
trueTransform: {
insert: 'é',
deleteLeft: 1,
id: 13
}, transitionId: 13,
transitionId: 13,
start: 0
}, bestProbFromSet: 1,
subsetId: generateSubsetId()
@ -781,11 +761,7 @@ describe('TokenizationSubsetBuilder', function() {
const fiveCharTailToken = new ContextToken(baseTokenization.tail);
fiveCharTailToken.addInput({
segment: {
trueTransform: {
insert: 'é',
deleteLeft: 1,
id: 13
}, transitionId: 13,
transitionId: 13,
start: 0
}, bestProbFromSet: 1,
subsetId: generateSubsetId()
@ -823,11 +799,7 @@ describe('TokenizationSubsetBuilder', function() {
const twoCharTailToken = new ContextToken(baseTokenization.tail);
twoCharTailToken.addInput({
segment: {
trueTransform: {
insert: 'é',
deleteLeft: 1,
id: 13
}, transitionId: 13,
transitionId: 13,
start: 0
}, bestProbFromSet: .6,
subsetId: generateSubsetId()
@ -838,10 +810,7 @@ describe('TokenizationSubsetBuilder', function() {
const threeCharTailToken = new ContextToken(baseTokenization.tail);
threeCharTailToken.addInput({
segment: {
trueTransform: {
insert: 'é',
deleteLeft: 1
}, transitionId: 13,
transitionId: 13,
start: 0
}, bestProbFromSet: .6,
subsetId: generateSubsetId()

55
web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/search-quotient-spur.tests.ts
View file

@ -116,14 +116,8 @@ describe('SearchQuotientSpur', () => {
assert.deepEqual(extendedPath.inputs, leadEdgeDistribution);
assert.deepEqual(extendedPath.inputSegments, [
{
segment: {
trueTransform: leadEdgeDistribution[0].sample,
transitionId: leadEdgeDistribution[0].sample.id,
start: 0
},
bestProbFromSet: leadEdgeDistribution[0].p,
// Just write in the variable-value entry; the rest should match perfectly.
subsetId: extendedPath.inputSegments[0].subsetId
transitionId: leadEdgeDistribution[0].sample.id,
start: 0
}
]);
@ -174,23 +168,11 @@ describe('SearchQuotientSpur', () => {
assert.deepEqual(length2Path.inputs, tailEdgeDistribution);
assert.deepEqual(length2Path.inputSegments, [
{
segment: {
trueTransform: leadEdgeDistribution[0].sample,
transitionId: leadEdgeDistribution[0].sample.id,
start: 0
},
bestProbFromSet: leadEdgeDistribution[0].p,
// Just write in the variable-value entry; the rest should match perfectly.
subsetId: length2Path.inputSegments[0].subsetId
transitionId: leadEdgeDistribution[0].sample.id,
start: 0
}, {
segment: {
trueTransform: tailEdgeDistribution[0].sample,
transitionId: tailEdgeDistribution[0].sample.id,
start: 0
},
bestProbFromSet: tailEdgeDistribution[0].p,
// Just write in the variable-value entry; the rest should match perfectly.
subsetId: length2Path.inputSegments[1].subsetId
transitionId: tailEdgeDistribution[0].sample.id,
start: 0
}
]);
@ -258,23 +240,11 @@ describe('SearchQuotientSpur', () => {
assert.deepEqual(length2Path.inputs, tailEdgeDistribution);
assert.deepEqual(length2Path.inputSegments, [
{
segment: {
trueTransform: leadEdgeDistribution[0].sample,
transitionId: leadEdgeDistribution[0].sample.id,
start: 0
},
bestProbFromSet: leadEdgeDistribution[0].p,
// Just write in the variable-value entry; the rest should match perfectly.
subsetId: length2Path.inputSegments[0].subsetId
transitionId: leadEdgeDistribution[0].sample.id,
start: 0
}, {
segment: {
trueTransform: tailEdgeDistribution[0].sample,
transitionId: tailEdgeDistribution[0].sample.id,
start: 0
},
bestProbFromSet: tailEdgeDistribution[0].p,
// Just write in the variable-value entry; the rest should match perfectly.
subsetId: length2Path.inputSegments[1].subsetId
transitionId: tailEdgeDistribution[0].sample.id,
start: 0
}
]);
@ -1285,7 +1255,6 @@ describe('SearchQuotientSpur', () => {
const headTarget = new LegacyQuotientSpur(
path, headDistributionSplit, {
segment: {
trueTransform: inputDistribution[0].sample,
start: 0,
transitionId: inputDistribution[0].sample.id
},
@ -1304,7 +1273,6 @@ describe('SearchQuotientSpur', () => {
const tailTarget = new LegacyQuotientSpur(
new LegacyQuotientRoot(testModel), tailDistributionSplit, {
segment: {
trueTransform: inputDistribution[0].sample,
start: 2,
transitionId: inputDistribution[0].sample.id
},
@ -1377,7 +1345,6 @@ describe('SearchQuotientSpur', () => {
const originalInputBase: PathInputProperties = {
segment: {
trueTransform: {insert: 'biglargetransform', deleteLeft: 0, id: 11},
start: 0,
transitionId: 11
},
@ -1868,7 +1835,6 @@ describe('SearchQuotientSpur', () => {
path, headDistributionSplit, {
segment: {
start: 0,
trueTransform: inputDistribution[0].sample,
transitionId: inputDistribution[0].sample.id
},
bestProbFromSet: inputDistribution[0].p,
@ -1887,7 +1853,6 @@ describe('SearchQuotientSpur', () => {
new LegacyQuotientRoot(testModel), tailDistributionSplit, {
segment: {
start: 2,
trueTransform: inputDistribution[0].sample,
transitionId: inputDistribution[0].sample.id
},
bestProbFromSet: inputDistribution[0].p,