mirror of
https://github.com/keymanapp/keyman.git
synced 2026-11-08 17:07:32 +00:00
fix(web): adjusts existing unit tests to match new intermediate-prediction-data format
This commit is contained in:
Joshua Horton
parent
b12c912211
commit
6b27134eb8
5 changed files with 681 additions and 419 deletions
|
|
@ -1,15 +1,15 @@
|
|||
import { assert } from 'chai';
|
||||
|
||||
import { CORRECTION_SEARCH_THRESHOLDS, IntermediateCompositedPrediction, ModelCompositor, shouldStopSearchingEarly } from "@keymanapp/lm-worker/test-index";
|
||||
import { CORRECTION_SEARCH_THRESHOLDS, IntermediateTokenizedPrediction, ModelCompositor, shouldStopSearchingEarly } from "@keymanapp/lm-worker/test-index";
|
||||
|
||||
function mockIntermediatePrediction(value: number) {
|
||||
function mockTokenizedPrediction(value: number) {
|
||||
return {
|
||||
metadata: {
|
||||
probabilities: {
|
||||
total: value
|
||||
}
|
||||
}
|
||||
} as IntermediateCompositedPrediction
|
||||
} as IntermediateTokenizedPrediction
|
||||
}
|
||||
|
||||
describe('correction-search: shouldStopSearchingEarly', () => {
|
||||
|
|
@ -22,7 +22,7 @@ describe('correction-search: shouldStopSearchingEarly', () => {
|
|||
assert.equal(predictionProbs.length, ModelCompositor.MAX_SUGGESTIONS, "test setup no longer valid");
|
||||
|
||||
// The only part for each entry we actually care about here: .totalProb.
|
||||
const predictions = predictionProbs.map((entry) => mockIntermediatePrediction(entry));
|
||||
const predictions = predictionProbs.map((entry) => mockTokenizedPrediction(entry));
|
||||
|
||||
// Thresholding is performed in log-space.
|
||||
// 0.0501 and 0.0499 are offset on each side of 0.05, the last value in the array defined above.
|
||||
|
|
@ -38,8 +38,8 @@ describe('correction-search: shouldStopSearchingEarly', () => {
|
|||
//
|
||||
// Can technically run the method with an empty array, but the actual scenario would have
|
||||
// at least one prediction present in the "found predictions" array.
|
||||
assert.isFalse(shouldStopSearchingEarly(baseCost, baseCost + expectedThreshold - 0.01, [mockIntermediatePrediction(Math.exp(-1))]));
|
||||
assert.isTrue(shouldStopSearchingEarly( baseCost, baseCost + expectedThreshold + 0.01, [mockIntermediatePrediction(Math.exp(-1))]));
|
||||
assert.isFalse(shouldStopSearchingEarly(baseCost, baseCost + expectedThreshold - 0.01, [mockTokenizedPrediction(Math.exp(-1))]));
|
||||
assert.isTrue(shouldStopSearchingEarly( baseCost, baseCost + expectedThreshold + 0.01, [mockTokenizedPrediction(Math.exp(-1))]));
|
||||
});
|
||||
|
||||
it('stops checking corrections earlier when enough predictions have been found', () => {
|
||||
|
|
@ -48,7 +48,7 @@ describe('correction-search: shouldStopSearchingEarly', () => {
|
|||
|
||||
// The only part for each entry we actually care about here: .totalProb.
|
||||
/** @type {import('#./predict-helpers.js').CorrectionPredictionTuple[]} */
|
||||
const predictions = predictionProbs.map((entry) => mockIntermediatePrediction(entry));
|
||||
const predictions = predictionProbs.map((entry) => mockTokenizedPrediction(entry));
|
||||
|
||||
const baseCost = 1;
|
||||
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ import {
|
|||
ContextState,
|
||||
ContextToken,
|
||||
ContextTokenization,
|
||||
IntermediateCompositedPrediction,
|
||||
IntermediateTokenizedPrediction,
|
||||
ModelCompositor,
|
||||
TokenizationResultMapping
|
||||
} from "@keymanapp/lm-worker/test-index";
|
||||
|
|
@ -78,13 +78,17 @@ describe('determineTokenizedCorrectionSequence', () => {
|
|||
endOfBuffer: true
|
||||
});
|
||||
|
||||
assert.deepEqual(results.tokenizedCorrection, [
|
||||
assert.deepEqual(results.tokens, [
|
||||
{
|
||||
sample: {
|
||||
insert: 'fo',
|
||||
deleteLeft: 0
|
||||
correction: {
|
||||
sample: {
|
||||
insert: 'fo',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: trueInput.p
|
||||
},
|
||||
p: trueInput.p
|
||||
casingRoot: 'fo',
|
||||
autoSelectable: true
|
||||
}
|
||||
]);
|
||||
});
|
||||
|
|
@ -129,12 +133,20 @@ describe('determineTokenizedCorrectionSequence', () => {
|
|||
endOfBuffer: true
|
||||
});
|
||||
|
||||
assert.equal(results.tokenizedCorrection.length, 1);
|
||||
assert.deepEqual(results.tokenizedCorrection[0].sample, {
|
||||
insert: ' ',
|
||||
deleteLeft: 0
|
||||
});
|
||||
assert.approximately(results.tokenizedCorrection[0].p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000);
|
||||
assert.equal(results.tokens.length, 1);
|
||||
assert.approximately(results.tokens[0].correction.p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000);
|
||||
|
||||
assert.deepEqual(results.tokens, [{
|
||||
correction: {
|
||||
sample: {
|
||||
insert: ' ',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: results.tokens[0].correction.p
|
||||
},
|
||||
casingRoot: ' ',
|
||||
autoSelectable: false
|
||||
}]);
|
||||
});
|
||||
|
||||
it(`properly analyzes common-case word-start - beginning a new token`, () => {
|
||||
|
|
@ -178,12 +190,20 @@ describe('determineTokenizedCorrectionSequence', () => {
|
|||
});
|
||||
|
||||
|
||||
assert.equal(results.tokenizedCorrection.length, 1);
|
||||
assert.deepEqual(results.tokenizedCorrection[0].sample, {
|
||||
insert: 'f',
|
||||
deleteLeft: 0
|
||||
});
|
||||
assert.approximately(results.tokenizedCorrection[0].p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000);
|
||||
assert.equal(results.tokens.length, 1);
|
||||
assert.approximately(results.tokens[0].correction.p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000);
|
||||
|
||||
assert.deepEqual(results.tokens, [{
|
||||
correction: {
|
||||
sample: {
|
||||
insert: 'f',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: results.tokens[0].correction.p
|
||||
},
|
||||
casingRoot: 'f',
|
||||
autoSelectable: true
|
||||
}]);
|
||||
});
|
||||
|
||||
it(`properly analyzes post-merge case`, () => {
|
||||
|
|
@ -232,15 +252,17 @@ describe('determineTokenizedCorrectionSequence', () => {
|
|||
endOfBuffer: true
|
||||
});
|
||||
|
||||
assert.deepEqual(results.tokenizedCorrection, [
|
||||
{
|
||||
assert.deepEqual(results.tokens, [{
|
||||
correction: {
|
||||
sample: {
|
||||
insert: 'can\'t',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: trueInput.p
|
||||
}
|
||||
]);
|
||||
},
|
||||
casingRoot: 'can\'t',
|
||||
autoSelectable: true
|
||||
}]);
|
||||
});
|
||||
|
||||
// Will be handled far better after resolving multi-tokenization handling.
|
||||
|
|
@ -285,12 +307,20 @@ describe('determineTokenizedCorrectionSequence', () => {
|
|||
});
|
||||
|
||||
|
||||
assert.equal(results.tokenizedCorrection.length, 1);
|
||||
assert.deepEqual(results.tokenizedCorrection[0].sample, {
|
||||
insert: ' ',
|
||||
deleteLeft: 0
|
||||
});
|
||||
assert.approximately(results.tokenizedCorrection[0].p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000);
|
||||
assert.equal(results.tokens.length, 1);
|
||||
assert.approximately(results.tokens[0].correction.p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000);
|
||||
|
||||
assert.deepEqual(results.tokens, [{
|
||||
correction: {
|
||||
sample: {
|
||||
insert: ' ',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: results.tokens[0].correction.p
|
||||
},
|
||||
casingRoot: ' ',
|
||||
autoSelectable: false
|
||||
}]);
|
||||
});
|
||||
|
||||
it(`properly analyzes complex transition - multi-token replacement`, () => {
|
||||
|
|
@ -334,21 +364,34 @@ describe('determineTokenizedCorrectionSequence', () => {
|
|||
|
||||
// Coming up next - actually providing ALL correction elements, not just the final one.
|
||||
// We're not _quite_ ready for that yet, though.
|
||||
assert.equal(results.tokenizedCorrection.length, 1);
|
||||
assert.deepEqual(results.tokenizedCorrection[0].sample, {
|
||||
assert.equal(results.tokens.length, 1);
|
||||
assert.deepEqual(results.tokens[0].correction.sample, {
|
||||
insert: 'd',
|
||||
deleteLeft: 0
|
||||
});
|
||||
assert.approximately(results.tokenizedCorrection[0].p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000);
|
||||
assert.approximately(results.tokens[0].correction.p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000);
|
||||
|
||||
const dummiedTuple: IntermediateCompositedPrediction = {
|
||||
components: {
|
||||
assert.deepEqual(results.tokens, [{
|
||||
correction: {
|
||||
sample: {
|
||||
insert: 'd',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: results.tokens[0].correction.p
|
||||
},
|
||||
casingRoot: 'd',
|
||||
autoSelectable: true
|
||||
}]);
|
||||
|
||||
const dummiedTuple: IntermediateTokenizedPrediction = {
|
||||
components: [{
|
||||
prediction: {
|
||||
transform: { insert: 'dog', deleteLeft: 0 },
|
||||
displayAs: 'dog'
|
||||
},
|
||||
correction: 'd'
|
||||
},
|
||||
correction: 'd',
|
||||
casingRoot: 'd'
|
||||
}],
|
||||
metadata: {
|
||||
probabilities: {
|
||||
prediction: .25,
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import { LexicalModelTypes } from "@keymanapp/common-types";
|
|||
import * as wordBreakers from '@keymanapp/models-wordbreakers';
|
||||
import { KMWString } from 'keyman/common/web-utils';
|
||||
|
||||
import { IntermediateCompositedPrediction, ModelCompositor, determineTraversallessCorrectionSequences, models } from "@keymanapp/lm-worker/test-index";
|
||||
import { determineTraversallessCorrectionSequences, IntermediateTokenizedPrediction, ModelCompositor, models } from "@keymanapp/lm-worker/test-index";
|
||||
|
||||
import Context = LexicalModelTypes.Context;
|
||||
import DummyModel = models.DummyModel;
|
||||
|
|
@ -80,12 +80,16 @@ describe('determineTraversallessCorrectionSequences', () => {
|
|||
}
|
||||
);
|
||||
|
||||
assert.deepEqual(entry.tokenizedCorrection, [{
|
||||
sample: {
|
||||
insert: 'appl',
|
||||
deleteLeft: 0
|
||||
assert.deepEqual(entry.tokens, [{
|
||||
correction: {
|
||||
sample: {
|
||||
insert: 'appl',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: trueInput.p
|
||||
},
|
||||
p: trueInput.p
|
||||
casingRoot: 'appl',
|
||||
autoSelectable: true
|
||||
}]);
|
||||
});
|
||||
|
||||
|
|
@ -122,13 +126,19 @@ describe('determineTraversallessCorrectionSequences', () => {
|
|||
}
|
||||
);
|
||||
|
||||
assert.deepEqual(entry.tokenizedCorrection, [{
|
||||
sample: {
|
||||
insert: 'iPhone',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: trueInput.p
|
||||
}]);
|
||||
assert.deepEqual(predictionRootEntries[0].tokens, [
|
||||
{
|
||||
correction: {
|
||||
sample: {
|
||||
insert: 'iPhone',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: trueInput.p
|
||||
},
|
||||
casingRoot: 'iPhone',
|
||||
autoSelectable: true
|
||||
}
|
||||
]);
|
||||
});
|
||||
|
||||
it(`properly analyzes common-case token-extension - adding a letter to an existing word`, () => {
|
||||
|
|
@ -163,13 +173,19 @@ describe('determineTraversallessCorrectionSequences', () => {
|
|||
}
|
||||
);
|
||||
|
||||
assert.deepEqual(entry.tokenizedCorrection, [{
|
||||
sample: {
|
||||
insert: 'fo',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: trueInput.p
|
||||
}]);
|
||||
assert.deepEqual(entry.tokens, [
|
||||
{
|
||||
correction: {
|
||||
sample: {
|
||||
insert: 'fo',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: trueInput.p
|
||||
},
|
||||
casingRoot: 'fo',
|
||||
autoSelectable: true
|
||||
}
|
||||
]);
|
||||
});
|
||||
|
||||
it(`properly analyzes common-case whitespace - ending a token and adding a new one`, () => {
|
||||
|
|
@ -204,12 +220,20 @@ describe('determineTraversallessCorrectionSequences', () => {
|
|||
}
|
||||
);
|
||||
|
||||
assert.equal(entry.tokenizedCorrection.length, 1);
|
||||
assert.deepEqual(entry.tokenizedCorrection[0].sample, {
|
||||
insert: '',
|
||||
deleteLeft: 0
|
||||
});
|
||||
assert.approximately(entry.tokenizedCorrection[0].p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000);
|
||||
assert.equal(entry.tokens.length, 1);
|
||||
assert.approximately(entry.tokens[0].correction.p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000);
|
||||
|
||||
assert.deepEqual(entry.tokens, [{
|
||||
correction: {
|
||||
sample: {
|
||||
insert: '',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: entry.tokens[0].correction.p
|
||||
},
|
||||
casingRoot: '',
|
||||
autoSelectable: false
|
||||
}]);
|
||||
});
|
||||
|
||||
|
||||
|
|
@ -245,12 +269,20 @@ describe('determineTraversallessCorrectionSequences', () => {
|
|||
}
|
||||
);
|
||||
|
||||
assert.equal(entry.tokenizedCorrection.length, 1);
|
||||
assert.deepEqual(entry.tokenizedCorrection[0].sample, {
|
||||
insert: 'f',
|
||||
deleteLeft: 0
|
||||
});
|
||||
assert.approximately(entry.tokenizedCorrection[0].p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000);
|
||||
assert.equal(entry.tokens.length, 1);
|
||||
assert.approximately(entry.tokens[0].correction.p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000);
|
||||
|
||||
assert.deepEqual(entry.tokens, [{
|
||||
correction: {
|
||||
sample: {
|
||||
insert: 'f',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: entry.tokens[0].correction.p
|
||||
},
|
||||
casingRoot: 'f',
|
||||
autoSelectable: true
|
||||
}]);
|
||||
});
|
||||
|
||||
it(`properly analyzes post-merge case`, () => {
|
||||
|
|
@ -285,12 +317,16 @@ describe('determineTraversallessCorrectionSequences', () => {
|
|||
}
|
||||
);
|
||||
|
||||
assert.deepEqual(entry.tokenizedCorrection, [{
|
||||
sample: {
|
||||
insert: 'can\'t',
|
||||
deleteLeft: 0
|
||||
assert.deepEqual(entry.tokens, [{
|
||||
correction: {
|
||||
sample: {
|
||||
insert: 'can\'t',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: trueInput.p
|
||||
},
|
||||
p: trueInput.p
|
||||
casingRoot: 'can\'t',
|
||||
autoSelectable: true
|
||||
}]);
|
||||
});
|
||||
|
||||
|
|
@ -328,12 +364,20 @@ describe('determineTraversallessCorrectionSequences', () => {
|
|||
// }
|
||||
// );
|
||||
|
||||
assert.equal(entry.tokenizedCorrection.length, 1);
|
||||
assert.deepEqual(entry.tokenizedCorrection[0].sample, {
|
||||
insert: '',
|
||||
deleteLeft: 0
|
||||
});
|
||||
assert.approximately(entry.tokenizedCorrection[0].p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000);
|
||||
assert.equal(entry.tokens.length, 1);
|
||||
assert.approximately(entry.tokens[0].correction.p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000);
|
||||
|
||||
assert.deepEqual(entry.tokens, [{
|
||||
correction: {
|
||||
sample: {
|
||||
insert: '',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: entry.tokens[0].correction.p
|
||||
},
|
||||
casingRoot: '',
|
||||
autoSelectable: false
|
||||
}]);
|
||||
});
|
||||
|
||||
it(`properly analyzes complex transition - multi-token replacement`, () => {
|
||||
|
|
@ -370,21 +414,34 @@ describe('determineTraversallessCorrectionSequences', () => {
|
|||
|
||||
// Coming up next - actually providing ALL correction elements, not just the final one.
|
||||
// We're not _quite_ ready for that yet, though.
|
||||
assert.equal(entry.tokenizedCorrection.length, 1);
|
||||
assert.deepEqual(entry.tokenizedCorrection[0].sample, {
|
||||
assert.equal(entry.tokens.length, 1);
|
||||
assert.deepEqual(entry.tokens[0].correction.sample, {
|
||||
insert: 'd',
|
||||
deleteLeft: 0
|
||||
});
|
||||
assert.approximately(entry.tokenizedCorrection[0].p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000);
|
||||
assert.approximately(entry.tokens[0].correction.p, Math.pow(trueInput.p, ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT), Number.EPSILON*1000);
|
||||
|
||||
const dummiedTuple: IntermediateCompositedPrediction = {
|
||||
components: {
|
||||
assert.deepEqual(entry.tokens, [{
|
||||
correction: {
|
||||
sample: {
|
||||
insert: 'd',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: entry.tokens[0].correction.p
|
||||
},
|
||||
casingRoot: 'd',
|
||||
autoSelectable: true
|
||||
}]);
|
||||
|
||||
const dummiedTuple: IntermediateTokenizedPrediction = {
|
||||
components: [{
|
||||
prediction: {
|
||||
transform: { insert: 'dog', deleteLeft: 0 },
|
||||
displayAs: 'dog'
|
||||
},
|
||||
correction: 'd'
|
||||
},
|
||||
correction: 'd',
|
||||
casingRoot: 'd'
|
||||
}],
|
||||
metadata: {
|
||||
probabilities: {
|
||||
prediction: .25,
|
||||
|
|
|
|||
|
|
@ -4,16 +4,12 @@ import { assert } from 'chai';
|
|||
import { deepCopy } from "keyman/common/web-utils";
|
||||
import { LexicalModelTypes } from '@keymanapp/common-types';
|
||||
|
||||
import { EDIT_DISTANCE_COST_SCALE, models, predictFromCorrectionSequence, tupleDisplayOrderSort } from "@keymanapp/lm-worker/test-index";
|
||||
import { EDIT_DISTANCE_COST_SCALE, PredictionParameters, models, predictFromCorrectionSequence, tupleDisplayOrderSort } from "@keymanapp/lm-worker/test-index";
|
||||
|
||||
import CasingFunction = LexicalModelTypes.CasingFunction;
|
||||
import Context = LexicalModelTypes.Context;
|
||||
import Distribution = LexicalModelTypes.Distribution;
|
||||
import DummyModel = models.DummyModel;
|
||||
import Outcome = LexicalModelTypes.Outcome;
|
||||
import ProbabilityMass = LexicalModelTypes.ProbabilityMass;
|
||||
import Suggestion = LexicalModelTypes.Suggestion;
|
||||
import Transform = LexicalModelTypes.Transform;
|
||||
|
||||
// See: developer/src/kmc-model/model-defaults.ts, defaultApplyCasing
|
||||
const applyCasing: CasingFunction = (casing, text) => {
|
||||
|
|
@ -75,21 +71,32 @@ const DUMMY_MODEL_CONFIG = {
|
|||
describe('predictFromCorrectionSequence', () => {
|
||||
describe('on a single correction', () => {
|
||||
it('constructs suggestions matching multiple lexical entries directly - no transform ID', () => {
|
||||
const context: Context = {
|
||||
left: '',
|
||||
right: '',
|
||||
startOfBuffer: true,
|
||||
endOfBuffer: true
|
||||
};
|
||||
const transitionID = 12345;
|
||||
|
||||
const correctionDistribution: Distribution<Transform> = [{
|
||||
sample: {
|
||||
insert: 'Its',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: 0.6
|
||||
}
|
||||
];
|
||||
const parameters: PredictionParameters = {
|
||||
rootContext: {
|
||||
left: '',
|
||||
right: '',
|
||||
startOfBuffer: true,
|
||||
endOfBuffer: true
|
||||
},
|
||||
tokens: [
|
||||
{
|
||||
correction: {
|
||||
sample: {
|
||||
insert: 'Its',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
p: 0.6
|
||||
},
|
||||
casingRoot: '',
|
||||
autoSelectable: true
|
||||
}
|
||||
],
|
||||
applyInPost: (x) => x,
|
||||
deleteLeft: 0
|
||||
};
|
||||
|
||||
const dummied_suggestions: Outcome<Suggestion>[] = [
|
||||
{
|
||||
|
|
@ -114,13 +121,12 @@ describe('predictFromCorrectionSequence', () => {
|
|||
futureSuggestions: [ dummied_suggestions ]
|
||||
});
|
||||
|
||||
const transitionID = 12345;
|
||||
const predictions = predictFromCorrectionSequence(model, correctionDistribution, context, transitionID);
|
||||
predictions.forEach((entry) => assert.equal(entry.components.correction, 'Its'));
|
||||
const predictions = predictFromCorrectionSequence(model, parameters);
|
||||
predictions.forEach((entry) => assert.equal(entry.components[0].correction, 'Its'));
|
||||
predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, 0.6));
|
||||
predictions.sort(tupleDisplayOrderSort);
|
||||
|
||||
assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components.prediction), dummied_suggestions.map((s) => {
|
||||
assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components[0].prediction), dummied_suggestions.map((s) => {
|
||||
delete s.p;
|
||||
s.transformId = transitionID;
|
||||
s.transform.id = transitionID;
|
||||
|
|
@ -132,23 +138,32 @@ describe('predictFromCorrectionSequence', () => {
|
|||
});
|
||||
|
||||
it('constructs suggestions matching multiple lexical entries directly - with transform ID', () => {
|
||||
const context: Context = {
|
||||
left: '',
|
||||
right: '',
|
||||
startOfBuffer: true,
|
||||
endOfBuffer: true
|
||||
};
|
||||
|
||||
const transitionID = 314159;
|
||||
const correctionDistribution: Distribution<Transform> = [{
|
||||
sample: {
|
||||
insert: 'Its',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
p: 0.6
|
||||
}
|
||||
];
|
||||
|
||||
const parameters: PredictionParameters = {
|
||||
rootContext: {
|
||||
left: '',
|
||||
right: '',
|
||||
startOfBuffer: true,
|
||||
endOfBuffer: true
|
||||
},
|
||||
tokens: [
|
||||
{
|
||||
correction: {
|
||||
sample: {
|
||||
insert: 'Its',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
p: 0.6
|
||||
},
|
||||
casingRoot: '',
|
||||
autoSelectable: true
|
||||
}
|
||||
],
|
||||
applyInPost: (x) => x,
|
||||
deleteLeft: 0
|
||||
};
|
||||
|
||||
const dummied_suggestions: Outcome<Suggestion>[] = [
|
||||
{
|
||||
|
|
@ -173,41 +188,51 @@ describe('predictFromCorrectionSequence', () => {
|
|||
futureSuggestions: [ dummied_suggestions ]
|
||||
});
|
||||
|
||||
const predictions = predictFromCorrectionSequence(model, correctionDistribution, context, transitionID);
|
||||
predictions.forEach((entry) => assert.equal(entry.components.correction, 'Its'));
|
||||
const predictions = predictFromCorrectionSequence(model, parameters);
|
||||
predictions.forEach((entry) => assert.equal(entry.components[0].correction, 'Its'));
|
||||
predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, 0.6));
|
||||
predictions.sort(tupleDisplayOrderSort);
|
||||
|
||||
assert.sameOrderedMembers(predictions.map((entry) => entry.components.prediction.displayAs), ["it's", "its"]);
|
||||
assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components.prediction), dummied_suggestions.map((entry) => {
|
||||
assert.sameOrderedMembers(predictions.map((entry) => entry.components[0].prediction.displayAs), ["it's", "its"]);
|
||||
assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components[0].prediction), dummied_suggestions.map((entry) => {
|
||||
entry = deepCopy(entry);
|
||||
entry.transformId = transitionID;
|
||||
entry.transform.id = transitionID;
|
||||
delete entry.p;
|
||||
return entry;
|
||||
}));
|
||||
|
||||
assert.approximately(predictions[0].metadata.probabilities.total, 0.18 * 0.6, 0.00001);
|
||||
assert.approximately(predictions[1].metadata.probabilities.total, 0.02 * 0.6, 0.00001);
|
||||
predictions.forEach((prediction) => assert.equal(prediction.components.prediction.transformId, transitionID));
|
||||
predictions.forEach((prediction) => assert.equal(prediction.components[0].prediction.transformId, transitionID));
|
||||
});
|
||||
|
||||
it('constructs suggestions without input (as if after a context reset)', () => {
|
||||
const context: Context = {
|
||||
left: 'appl',
|
||||
right: '',
|
||||
startOfBuffer: true,
|
||||
endOfBuffer: true
|
||||
};
|
||||
const transitionID = 271828;
|
||||
|
||||
const correctionDistribution: Distribution<Transform> = [{
|
||||
sample: {
|
||||
insert: 'appl',
|
||||
deleteLeft: 4
|
||||
},
|
||||
p: 1
|
||||
}
|
||||
];
|
||||
const parameters: PredictionParameters = {
|
||||
rootContext: {
|
||||
left: '',
|
||||
right: '',
|
||||
startOfBuffer: true,
|
||||
endOfBuffer: true
|
||||
},
|
||||
tokens: [
|
||||
{
|
||||
correction: {
|
||||
sample: {
|
||||
insert: 'appl',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
p: 1
|
||||
},
|
||||
casingRoot: 'appl',
|
||||
autoSelectable: true
|
||||
}
|
||||
],
|
||||
applyInPost: (x) => x,
|
||||
deleteLeft: 0
|
||||
};
|
||||
|
||||
const dummied_suggestions: Outcome<Suggestion>[] = [
|
||||
{
|
||||
|
|
@ -225,60 +250,79 @@ describe('predictFromCorrectionSequence', () => {
|
|||
futureSuggestions: [ dummied_suggestions ]
|
||||
});
|
||||
|
||||
const transitionID = 12345;
|
||||
const predictions = predictFromCorrectionSequence(model, correctionDistribution, context, transitionID);
|
||||
predictions.forEach((entry) => assert.equal(entry.components.correction, 'appl'));
|
||||
const predictions = predictFromCorrectionSequence(model, parameters);
|
||||
predictions.forEach((entry) => assert.deepEqual(entry.components.map((c => c.correction)), ['appl']));
|
||||
predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, 1));
|
||||
predictions.sort(tupleDisplayOrderSort);
|
||||
|
||||
assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components.prediction), dummied_suggestions.map((s) => {
|
||||
assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components.map((c) => c.prediction)), [dummied_suggestions.map((s) => {
|
||||
delete s.p;
|
||||
s.transformId = transitionID;
|
||||
s.transform.id = transitionID;
|
||||
return s;
|
||||
}));
|
||||
})]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('on a sequence of corrections', () => {
|
||||
it('returns results even if some correction tokens lack predictions', () => {
|
||||
const context: Context = {
|
||||
left: 'i want to eat a ',
|
||||
right: '',
|
||||
startOfBuffer: true,
|
||||
endOfBuffer: true
|
||||
};
|
||||
const transitionID = 101;
|
||||
|
||||
const correctionSequence: Distribution<Transform> = [
|
||||
{
|
||||
sample: {
|
||||
insert: 'golden',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: 0.1
|
||||
}, {
|
||||
sample: {
|
||||
insert: ' ',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: 0.2
|
||||
}, {
|
||||
sample: {
|
||||
insert: 'app',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: 0.2
|
||||
}
|
||||
];
|
||||
const parameters: PredictionParameters = {
|
||||
rootContext: {
|
||||
left: 'i want to eat a ',
|
||||
right: '',
|
||||
startOfBuffer: true,
|
||||
endOfBuffer: true
|
||||
},
|
||||
tokens: [
|
||||
{
|
||||
correction: {
|
||||
sample: {
|
||||
insert: 'g',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
p: 0.1
|
||||
},
|
||||
casingRoot: 'g',
|
||||
autoSelectable: true
|
||||
}, {
|
||||
correction: {
|
||||
sample: {
|
||||
insert: ' ',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
p: 0.2
|
||||
},
|
||||
casingRoot: ' ',
|
||||
autoSelectable: true
|
||||
}, {
|
||||
correction: {
|
||||
sample: {
|
||||
insert: 'apple',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
p: 0.2
|
||||
},
|
||||
casingRoot: 'apple',
|
||||
autoSelectable: true
|
||||
}
|
||||
],
|
||||
applyInPost: (x) => x,
|
||||
deleteLeft: 0
|
||||
};
|
||||
|
||||
const dummied_suggestion_sequences: Outcome<Suggestion>[][] = [
|
||||
[
|
||||
{
|
||||
transform: {
|
||||
insert: "golden",
|
||||
insert: "g",
|
||||
deleteLeft: 0
|
||||
},
|
||||
displayAs: "golden",
|
||||
displayAs: "g",
|
||||
p: 0.1
|
||||
}
|
||||
],
|
||||
|
|
@ -295,69 +339,104 @@ describe('predictFromCorrectionSequence', () => {
|
|||
]
|
||||
];
|
||||
|
||||
const transitionID = 101;
|
||||
const expected_prediction: ProbabilityMass<Suggestion> = {
|
||||
sample: {
|
||||
const expected_predictions: Suggestion[] = [
|
||||
{
|
||||
transform: {
|
||||
insert: 'golden apple',
|
||||
insert: 'g',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
displayAs: 'golden apple',
|
||||
displayAs: 'g',
|
||||
transformId: transitionID,
|
||||
}, {
|
||||
transform: {
|
||||
insert: ' ',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
displayAs: ' ',
|
||||
transformId: transitionID
|
||||
}, p: dummied_suggestion_sequences.map((dist) => {
|
||||
}, {
|
||||
transform: {
|
||||
insert: 'apple',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
displayAs: 'apple',
|
||||
transformId: transitionID
|
||||
}
|
||||
];
|
||||
|
||||
const expected_prediction_p = dummied_suggestion_sequences.map((dist) => {
|
||||
return dist[0]
|
||||
}).reduce((accum, curr) => {
|
||||
return accum * (curr ? curr.p : Math.exp(-EDIT_DISTANCE_COST_SCALE))
|
||||
}, 1)
|
||||
}
|
||||
}, 1);
|
||||
|
||||
const model = new DummyModel({
|
||||
...DUMMY_MODEL_CONFIG,
|
||||
futureSuggestions: dummied_suggestion_sequences
|
||||
});
|
||||
|
||||
const predictions = predictFromCorrectionSequence(model, correctionSequence, context, transitionID);
|
||||
predictions.forEach((entry) => assert.equal(entry.components.correction, 'golden app'));
|
||||
predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, correctionSequence.reduce((accum, curr) => accum * curr.p, 1)));
|
||||
const predictions = predictFromCorrectionSequence(model, parameters);
|
||||
predictions.forEach((entry) => assert.deepEqual(entry.components.map((c) => c.correction), ['g', ' ', 'apple']));
|
||||
predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, parameters.tokens.reduce((accum, curr) => accum * curr.correction.p, 1)));
|
||||
predictions.sort(tupleDisplayOrderSort);
|
||||
|
||||
assert.equal(predictions[0].components.prediction.transform.insert, 'golden apple');
|
||||
assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components.prediction), [expected_prediction.sample]);
|
||||
assert.sameDeepOrderedMembers(predictions[0].components.map((c) => c.prediction), expected_predictions);
|
||||
|
||||
assert.approximately(predictions[0].metadata.probabilities.prediction, expected_prediction.p, 0.00001);
|
||||
assert.equal(predictions[0].components.prediction.transformId, transitionID);
|
||||
assert.approximately(predictions[0].metadata.probabilities.prediction, expected_prediction_p, 0.00001);
|
||||
});
|
||||
|
||||
it('returns no results if all correction tokens lack predictions', () => {
|
||||
const context: Context = {
|
||||
left: 'i want to eat a ',
|
||||
right: '',
|
||||
startOfBuffer: true,
|
||||
endOfBuffer: true
|
||||
};
|
||||
const transitionID = 3;
|
||||
|
||||
const correctionSequence: Distribution<Transform> = [
|
||||
{
|
||||
sample: {
|
||||
insert: 'golden',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: 0.1
|
||||
}, {
|
||||
sample: {
|
||||
insert: ' ',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: 0.2
|
||||
}, {
|
||||
sample: {
|
||||
insert: 'app',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: 0.2
|
||||
}
|
||||
];
|
||||
const parameters: PredictionParameters = {
|
||||
rootContext: {
|
||||
left: 'i want to eat a ',
|
||||
right: '',
|
||||
startOfBuffer: true,
|
||||
endOfBuffer: true
|
||||
},
|
||||
tokens: [
|
||||
{
|
||||
correction: {
|
||||
sample: {
|
||||
insert: 'golden',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
p: 0.1
|
||||
},
|
||||
casingRoot: 'golden',
|
||||
autoSelectable: true
|
||||
}, {
|
||||
correction: {
|
||||
sample: {
|
||||
insert: ' ',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
p: 0.2
|
||||
},
|
||||
casingRoot: ' ',
|
||||
autoSelectable: true
|
||||
}, {
|
||||
correction: {
|
||||
sample: {
|
||||
insert: 'app',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
p: 0.2
|
||||
},
|
||||
casingRoot: 'app',
|
||||
autoSelectable: true
|
||||
}
|
||||
],
|
||||
applyInPost: (x) => x,
|
||||
deleteLeft: 0
|
||||
};
|
||||
|
||||
const dummied_suggestion_sequences: Outcome<Suggestion>[][] = [
|
||||
[],
|
||||
|
|
@ -370,39 +449,59 @@ describe('predictFromCorrectionSequence', () => {
|
|||
futureSuggestions: dummied_suggestion_sequences
|
||||
});
|
||||
|
||||
const predictions = predictFromCorrectionSequence(model, correctionSequence, context, 3);
|
||||
const predictions = predictFromCorrectionSequence(model, parameters);
|
||||
assert.deepEqual(predictions, []);
|
||||
});
|
||||
|
||||
it('uses only the best suggestion for non-final corrected tokens', () => {
|
||||
const context: Context = {
|
||||
left: 'i want to eat a ',
|
||||
right: '',
|
||||
startOfBuffer: true,
|
||||
endOfBuffer: true
|
||||
};
|
||||
const transitionID = 42;
|
||||
|
||||
const correctionSequence: Distribution<Transform> = [
|
||||
{
|
||||
sample: {
|
||||
insert: 'g',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: 0.1
|
||||
}, {
|
||||
sample: {
|
||||
insert: ' ',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: 0.2
|
||||
}, {
|
||||
sample: {
|
||||
insert: 'app',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: 0.2
|
||||
}
|
||||
];
|
||||
const parameters: PredictionParameters = {
|
||||
rootContext: {
|
||||
left: 'i want to eat a ',
|
||||
right: '',
|
||||
startOfBuffer: true,
|
||||
endOfBuffer: true
|
||||
},
|
||||
tokens: [
|
||||
{
|
||||
correction: {
|
||||
sample: {
|
||||
insert: 'g',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
p: 0.1
|
||||
},
|
||||
casingRoot: 'g',
|
||||
autoSelectable: true
|
||||
}, {
|
||||
correction: {
|
||||
sample: {
|
||||
insert: ' ',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
p: 0.2
|
||||
},
|
||||
casingRoot: ' ',
|
||||
autoSelectable: true
|
||||
}, {
|
||||
correction: {
|
||||
sample: {
|
||||
insert: 'app',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
p: 0.2
|
||||
},
|
||||
casingRoot: 'app',
|
||||
autoSelectable: true
|
||||
}
|
||||
],
|
||||
applyInPost: (x) => x,
|
||||
deleteLeft: 0
|
||||
};
|
||||
|
||||
const dummied_suggestion_sequences: Outcome<Suggestion>[][] = [
|
||||
[
|
||||
|
|
@ -442,72 +541,109 @@ describe('predictFromCorrectionSequence', () => {
|
|||
]
|
||||
];
|
||||
|
||||
const transitionID = 42;
|
||||
const expected_prediction: ProbabilityMass<Suggestion> = {
|
||||
sample: {
|
||||
transform: {
|
||||
insert: 'golden apple',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
displayAs: 'golden apple',
|
||||
transformId: 42
|
||||
}, p: dummied_suggestion_sequences.map((dist) => {
|
||||
const expected_prediction_p = dummied_suggestion_sequences
|
||||
.map((dist) => {
|
||||
return dist[0]
|
||||
}).reduce((accum, curr) => {
|
||||
return accum * (curr ? curr.p : Math.exp(-EDIT_DISTANCE_COST_SCALE))
|
||||
}, 1)
|
||||
}
|
||||
}, 1);
|
||||
|
||||
const expected_predictions: Suggestion[] = [
|
||||
{
|
||||
transform: {
|
||||
insert: 'golden',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
displayAs: 'golden',
|
||||
transformId: transitionID
|
||||
}, {
|
||||
transform: {
|
||||
insert: ' ',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
displayAs: ' ',
|
||||
transformId: transitionID
|
||||
}, {
|
||||
transform: {
|
||||
insert: 'apple',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
displayAs: 'apple',
|
||||
transformId: transitionID
|
||||
}
|
||||
];
|
||||
|
||||
const model = new DummyModel({
|
||||
...DUMMY_MODEL_CONFIG,
|
||||
futureSuggestions: dummied_suggestion_sequences
|
||||
});
|
||||
|
||||
const predictions = predictFromCorrectionSequence(model, correctionSequence, context, transitionID);
|
||||
const predictions = predictFromCorrectionSequence(model, parameters);
|
||||
// There should be no variations with 'green' or 'gray' apples.
|
||||
assert.equal(predictions.length, 1);
|
||||
|
||||
predictions.forEach((entry) => assert.equal(entry.components.correction, 'g app'));
|
||||
predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, correctionSequence.reduce((accum, curr) => accum * curr.p, 1)));
|
||||
predictions.forEach((entry) => assert.deepEqual(entry.components.map((c) => c.correction), ['g', ' ', 'app']));
|
||||
predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, parameters.tokens.reduce((accum, curr) => accum * curr.correction.p, 1)));
|
||||
predictions.sort(tupleDisplayOrderSort);
|
||||
|
||||
assert.equal(predictions[0].components.prediction.transform.insert, 'golden apple');
|
||||
assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components.prediction), [expected_prediction.sample]);
|
||||
assert.deepEqual(predictions[0].components.map((c) => c.prediction.transform.insert), ['golden', ' ', 'apple']);
|
||||
assert.sameDeepOrderedMembers(predictions[0].components.map((entry) => entry.prediction), expected_predictions);
|
||||
|
||||
assert.approximately(predictions[0].metadata.probabilities.prediction, expected_prediction.p, 0.00001);
|
||||
assert.equal(predictions[0].components.prediction.transformId, transitionID);
|
||||
assert.approximately(predictions[0].metadata.probabilities.prediction, expected_prediction_p, 0.00001);
|
||||
});
|
||||
|
||||
it('uses all suggestions generated from context-final correction-tokens', () => {
|
||||
const context: Context = {
|
||||
left: 'i want to eat a ',
|
||||
right: '',
|
||||
startOfBuffer: true,
|
||||
endOfBuffer: true
|
||||
};
|
||||
const transitionID = 13;
|
||||
|
||||
const correctionSequence: Distribution<Transform> = [
|
||||
{
|
||||
sample: {
|
||||
insert: 'golden',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: 0.1
|
||||
}, {
|
||||
sample: {
|
||||
insert: ' ',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: 0.2
|
||||
}, {
|
||||
sample: {
|
||||
insert: 'app',
|
||||
deleteLeft: 0
|
||||
},
|
||||
p: 0.2
|
||||
}
|
||||
];
|
||||
const parameters: PredictionParameters = {
|
||||
rootContext: {
|
||||
left: 'i want to eat a ',
|
||||
right: '',
|
||||
startOfBuffer: true,
|
||||
endOfBuffer: true
|
||||
},
|
||||
tokens: [
|
||||
{
|
||||
correction: {
|
||||
sample: {
|
||||
insert: 'golden',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
p: 0.1
|
||||
},
|
||||
casingRoot: 'golden',
|
||||
autoSelectable: true
|
||||
}, {
|
||||
correction: {
|
||||
sample: {
|
||||
insert: ' ',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
p: 0.2
|
||||
},
|
||||
casingRoot: ' ',
|
||||
autoSelectable: true
|
||||
}, {
|
||||
correction: {
|
||||
sample: {
|
||||
insert: 'app',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
p: 0.2
|
||||
},
|
||||
casingRoot: 'app',
|
||||
autoSelectable: true
|
||||
}
|
||||
],
|
||||
applyInPost: (x) => x,
|
||||
deleteLeft: 0
|
||||
};
|
||||
|
||||
const dummied_suggestion_sequences: Outcome<Suggestion>[][] = [
|
||||
[
|
||||
|
|
@ -549,29 +685,40 @@ describe('predictFromCorrectionSequence', () => {
|
|||
|
||||
const tailIndex = dummied_suggestion_sequences.length - 1;
|
||||
|
||||
const transitionID = 13;
|
||||
const expected_predictions: ProbabilityMass<Suggestion>[] = dummied_suggestion_sequences[tailIndex].map((p) => {
|
||||
const expectedText = `golden ${p.transform.insert}`;
|
||||
const expected_prediction_prefix_p = dummied_suggestion_sequences
|
||||
.slice(0, dummied_suggestion_sequences.length - 1)
|
||||
.map((dist) => {
|
||||
return dist[0]
|
||||
}).reduce((accum, curr) => {
|
||||
return accum * (curr ? curr.p : Math.exp(-EDIT_DISTANCE_COST_SCALE))
|
||||
}, 1);
|
||||
|
||||
return {
|
||||
sample: {
|
||||
transform: {
|
||||
insert: expectedText,
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
displayAs: expectedText,
|
||||
transformId: transitionID
|
||||
}, p: dummied_suggestion_sequences.map((dist) => {
|
||||
return dist[0]
|
||||
}).reduce((accum, curr, index) => {
|
||||
if(tailIndex == index) {
|
||||
return accum * p.p;
|
||||
} else {
|
||||
return accum * (curr ? curr.p : Math.exp(-EDIT_DISTANCE_COST_SCALE))
|
||||
}
|
||||
}, 1)
|
||||
};
|
||||
const expected_prediction_prefix: Suggestion[] = [
|
||||
{
|
||||
transform: {
|
||||
insert: 'golden',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
displayAs: 'golden',
|
||||
transformId: transitionID
|
||||
}, {
|
||||
transform: {
|
||||
insert: ' ',
|
||||
deleteLeft: 0,
|
||||
id: transitionID
|
||||
},
|
||||
displayAs: ' ',
|
||||
transformId: transitionID
|
||||
}
|
||||
];
|
||||
|
||||
const expected_prediction_sequences: Suggestion[][] = dummied_suggestion_sequences[tailIndex].map((p) => {
|
||||
return [...expected_prediction_prefix, p];
|
||||
});
|
||||
|
||||
const expected_prediction_seq_probs: number[] = dummied_suggestion_sequences[tailIndex].map((p) => {
|
||||
return p.p * expected_prediction_prefix_p;
|
||||
});
|
||||
|
||||
const model = new DummyModel({
|
||||
|
|
@ -579,22 +726,17 @@ describe('predictFromCorrectionSequence', () => {
|
|||
futureSuggestions: dummied_suggestion_sequences
|
||||
});
|
||||
|
||||
const predictions = predictFromCorrectionSequence(model, correctionSequence, context, transitionID);
|
||||
const predictions = predictFromCorrectionSequence(model, parameters);
|
||||
assert.equal(predictions.length, dummied_suggestion_sequences[dummied_suggestion_sequences.length - 1].length);
|
||||
|
||||
predictions.forEach((entry) => assert.equal(entry.components.correction, 'golden app'));
|
||||
predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, correctionSequence.reduce((accum, curr) => accum * curr.p, 1)));
|
||||
predictions.forEach((entry) => assert.deepEqual(entry.components.map((c) => c.correction), ['golden', ' ', 'app']));
|
||||
predictions.forEach((entry) => assert.equal(entry.metadata.probabilities.correction, parameters.tokens.reduce((accum, curr) => accum * curr.correction.p, 1)));
|
||||
predictions.sort(tupleDisplayOrderSort);
|
||||
|
||||
assert.sameOrderedMembers(
|
||||
predictions.map((t) => t.components.prediction.transform.insert),
|
||||
['golden apple', 'golden application', 'golden appetizer']
|
||||
);
|
||||
assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components.prediction), expected_predictions.map((p => p.sample)));
|
||||
assert.sameDeepOrderedMembers(predictions.map((entry) => entry.components.map((c) => c.prediction)), expected_prediction_sequences);
|
||||
|
||||
for(let i = 0; i < predictions.length; i++) {
|
||||
assert.approximately(predictions[i].metadata.probabilities.prediction, expected_predictions[i].p, 0.00001, `Expected probabilty mismatch at index ${i}`);
|
||||
assert.equal(predictions[i].components.prediction.transformId, transitionID);
|
||||
assert.approximately(predictions[i].metadata.probabilities.prediction, expected_prediction_seq_probs[i], 0.00001, `Expected probabilty mismatch at index ${i}`);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import * as wordBreakers from '@keymanapp/models-wordbreakers';
|
|||
import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs';
|
||||
import { LexicalModelTypes } from '@keymanapp/common-types';
|
||||
|
||||
import { applySuggestionCasing, models } from '@keymanapp/lm-worker/test-index';
|
||||
import { TokenizedPredictionData, applySuggestionCasing, models } from '@keymanapp/lm-worker/test-index';
|
||||
|
||||
import CasingFunction = LexicalModelTypes.CasingFunction;
|
||||
import TrieModel = models.TrieModel;
|
||||
|
|
@ -45,117 +45,137 @@ describe('applySuggestionCasing', function() {
|
|||
);
|
||||
|
||||
it('properly cases suggestions with no suggestion root', function() {
|
||||
let suggestion = {
|
||||
transform: {
|
||||
insert: 'the',
|
||||
deleteLeft: 0
|
||||
let suggestion: TokenizedPredictionData[] = [{
|
||||
prediction: {
|
||||
transform: {
|
||||
insert: 'the',
|
||||
deleteLeft: 0
|
||||
},
|
||||
displayAs: 'the'
|
||||
},
|
||||
displayAs: 'the'
|
||||
};
|
||||
correction: '',
|
||||
casingRoot: 'th'
|
||||
}];
|
||||
|
||||
applySuggestionCasing(suggestion, '', plainCasedModel, 'initial');
|
||||
assert.equal(suggestion.displayAs, 'The');
|
||||
assert.equal(suggestion.transform.insert, 'The');
|
||||
applySuggestionCasing(suggestion[0], plainCasedModel);
|
||||
assert.equal(suggestion[0].prediction.displayAs, 'the');
|
||||
assert.equal(suggestion[0].prediction.transform.insert, 'the');
|
||||
|
||||
suggestion = {
|
||||
transform: {
|
||||
insert: 'thE',
|
||||
deleteLeft: 0
|
||||
suggestion = [{
|
||||
prediction: {
|
||||
transform: {
|
||||
insert: 'ThE',
|
||||
deleteLeft: 0
|
||||
},
|
||||
displayAs: 'ThE'
|
||||
},
|
||||
displayAs: 'thE'
|
||||
};
|
||||
correction: '',
|
||||
casingRoot: 'Th'
|
||||
}];
|
||||
|
||||
applySuggestionCasing(suggestion, '', plainCasedModel, 'initial');
|
||||
assert.equal(suggestion.displayAs, 'ThE');
|
||||
assert.equal(suggestion.transform.insert, 'ThE');
|
||||
|
||||
suggestion = {
|
||||
transform: {
|
||||
insert: 'the',
|
||||
deleteLeft: 0
|
||||
},
|
||||
displayAs: 'the'
|
||||
};
|
||||
|
||||
applySuggestionCasing(suggestion, '', plainCasedModel, 'upper');
|
||||
assert.equal(suggestion.displayAs, 'THE');
|
||||
assert.equal(suggestion.transform.insert, 'THE');
|
||||
applySuggestionCasing(suggestion[0], plainCasedModel);
|
||||
assert.equal(suggestion[0].prediction.displayAs, 'ThE');
|
||||
assert.equal(suggestion[0].prediction.transform.insert, 'ThE');
|
||||
});
|
||||
|
||||
it('properly cases suggestions that fully replace the suggestion root', function() {
|
||||
let suggestion = {
|
||||
transform: {
|
||||
insert: 'therefore',
|
||||
deleteLeft: 3
|
||||
let suggestion: TokenizedPredictionData[] = [{
|
||||
prediction: {
|
||||
transform: {
|
||||
insert: 'therefore',
|
||||
deleteLeft: 3
|
||||
},
|
||||
displayAs: 'therefore'
|
||||
},
|
||||
displayAs: 'therefore'
|
||||
};
|
||||
correction: 'The',
|
||||
casingRoot: 'Th'
|
||||
}];
|
||||
|
||||
applySuggestionCasing(suggestion, 'the', plainCasedModel, 'initial');
|
||||
assert.equal(suggestion.displayAs, 'Therefore');
|
||||
assert.equal(suggestion.transform.insert, 'Therefore');
|
||||
applySuggestionCasing(suggestion[0], plainCasedModel);
|
||||
assert.equal(suggestion[0].prediction.displayAs, 'Therefore');
|
||||
assert.equal(suggestion[0].prediction.transform.insert, 'Therefore');
|
||||
|
||||
suggestion = {
|
||||
transform: {
|
||||
insert: 'thereFore',
|
||||
deleteLeft: 3
|
||||
suggestion = [{
|
||||
prediction: {
|
||||
transform: {
|
||||
insert: 'thereFore',
|
||||
deleteLeft: 3
|
||||
},
|
||||
displayAs: 'thereFore'
|
||||
},
|
||||
displayAs: 'thereFore'
|
||||
};
|
||||
correction: 'The',
|
||||
casingRoot: 'Th'
|
||||
}];
|
||||
|
||||
applySuggestionCasing(suggestion, 'the', plainCasedModel, 'initial');
|
||||
assert.equal(suggestion.displayAs, 'ThereFore');
|
||||
assert.equal(suggestion.transform.insert, 'ThereFore');
|
||||
applySuggestionCasing(suggestion[0], plainCasedModel);
|
||||
assert.equal(suggestion[0].prediction.displayAs, 'ThereFore');
|
||||
assert.equal(suggestion[0].prediction.transform.insert, 'ThereFore');
|
||||
|
||||
suggestion = {
|
||||
transform: {
|
||||
insert: 'therefore',
|
||||
deleteLeft: 3
|
||||
suggestion = [{
|
||||
prediction: {
|
||||
transform: {
|
||||
insert: 'therefore',
|
||||
deleteLeft: 3
|
||||
},
|
||||
displayAs: 'therefore'
|
||||
},
|
||||
displayAs: 'therefore'
|
||||
};
|
||||
correction: 'THE',
|
||||
casingRoot: 'TH'
|
||||
}];
|
||||
|
||||
applySuggestionCasing(suggestion, 'the', plainCasedModel, 'upper');
|
||||
assert.equal(suggestion.displayAs, 'THEREFORE');
|
||||
assert.equal(suggestion.transform.insert, 'THEREFORE');
|
||||
applySuggestionCasing(suggestion[0], plainCasedModel);
|
||||
assert.equal(suggestion[0].prediction.displayAs, 'THEREFORE');
|
||||
assert.equal(suggestion[0].prediction.transform.insert, 'THEREFORE');
|
||||
});
|
||||
|
||||
it('properly cases suggestions that do not fully replace the suggestion root', function() {
|
||||
let suggestion = {
|
||||
transform: {
|
||||
insert: 'erefore',
|
||||
deleteLeft: 1
|
||||
let suggestion: TokenizedPredictionData[] = [{
|
||||
prediction: {
|
||||
transform: {
|
||||
insert: 'therefore',
|
||||
deleteLeft: 3
|
||||
},
|
||||
displayAs: 'therefore'
|
||||
},
|
||||
displayAs: 'therefore'
|
||||
};
|
||||
correction: 'The',
|
||||
casingRoot: 'Th'
|
||||
}];
|
||||
|
||||
// When integrated, the 'the' string comes from a wordbreak operation on the current context.
|
||||
applySuggestionCasing(suggestion, 'the', plainCasedModel, 'initial');
|
||||
assert.equal(suggestion.displayAs, 'Therefore');
|
||||
assert.equal(suggestion.transform.insert, 'Therefore');
|
||||
applySuggestionCasing(suggestion[0], plainCasedModel);
|
||||
assert.equal(suggestion[0].prediction.displayAs, 'Therefore');
|
||||
assert.equal(suggestion[0].prediction.transform.insert, 'Therefore');
|
||||
|
||||
suggestion = {
|
||||
transform: {
|
||||
insert: 'ereFore',
|
||||
deleteLeft: 1
|
||||
suggestion = [{
|
||||
prediction: {
|
||||
transform: {
|
||||
insert: 'ThereFore',
|
||||
deleteLeft: 3
|
||||
},
|
||||
displayAs: 'thereFore'
|
||||
},
|
||||
displayAs: 'thereFore'
|
||||
};
|
||||
correction: 'The',
|
||||
casingRoot: 'Th'
|
||||
}];
|
||||
|
||||
applySuggestionCasing(suggestion, 'the', plainCasedModel, 'initial');
|
||||
assert.equal(suggestion.displayAs, 'ThereFore');
|
||||
assert.equal(suggestion.transform.insert, 'ThereFore');
|
||||
applySuggestionCasing(suggestion[0], plainCasedModel);
|
||||
assert.equal(suggestion[0].prediction.displayAs, 'ThereFore');
|
||||
assert.equal(suggestion[0].prediction.transform.insert, 'ThereFore');
|
||||
|
||||
suggestion = {
|
||||
transform: {
|
||||
insert: 'erefore',
|
||||
deleteLeft: 1
|
||||
suggestion = [{
|
||||
prediction: {
|
||||
transform: {
|
||||
insert: 'therefore',
|
||||
deleteLeft: 3
|
||||
},
|
||||
displayAs: 'therefore'
|
||||
},
|
||||
displayAs: 'therefore'
|
||||
};
|
||||
correction: 'THE',
|
||||
casingRoot: 'TH'
|
||||
}];
|
||||
|
||||
applySuggestionCasing(suggestion, 'the', plainCasedModel, 'upper');
|
||||
assert.equal(suggestion.displayAs, 'THEREFORE');
|
||||
assert.equal(suggestion.transform.insert, 'THEREFORE');
|
||||
applySuggestionCasing(suggestion[0], plainCasedModel);
|
||||
assert.equal(suggestion[0].prediction.displayAs, 'THEREFORE');
|
||||
assert.equal(suggestion[0].prediction.transform.insert, 'THEREFORE');
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Reference in a new issue