diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts index 9a034904a5..82fa2acf50 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/search-quotient-spur.ts @@ -24,6 +24,7 @@ import Distribution = LexicalModelTypes.Distribution; import LexicalModel = LexicalModelTypes.LexicalModel; import ProbabilityMass = LexicalModelTypes.ProbabilityMass; import Transform = LexicalModelTypes.Transform; +import { LegacyQuotientSpur } from './legacy-quotient-spur.js'; export const QUEUE_NODE_COMPARATOR: QueueComparator = function(arg1, arg2) { return arg1.currentCost - arg2.currentCost; @@ -265,7 +266,8 @@ export abstract class SearchQuotientSpur extends SearchQuotientNode { // // stopgap: maybe go ahead and check each input for any that are longer? // won't matter shortly, though. - return [[this, new LegacyQuotientRoot(this.model)]]; + const rootConstructor = this instanceof LegacyQuotientSpur ? LegacyQuotientRoot : SearchQuotientRoot; + return [[this, new rootConstructor(this.model)]]; } else { const firstSet: Distribution = this.inputs.map((input) => ({ // keep insert head @@ -300,9 +302,10 @@ export abstract class SearchQuotientSpur extends SearchQuotientNode { }) : this.parentNode; // construct two SearchPath instances based on the two sets! + const rootConstructor = this instanceof LegacyQuotientSpur ? LegacyQuotientRoot : SearchQuotientRoot; return [[ parent, - this.construct(new LegacyQuotientRoot(this.model), secondSet, { + this.construct(new rootConstructor(this.model), secondSet, { ...this.inputSource, segment: { ...this.inputSource.segment, diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/substitution-quotient-spur.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/substitution-quotient-spur.ts index baa6d9f5e8..1b34afcb50 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/substitution-quotient-spur.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/substitution-quotient-spur.ts @@ -20,8 +20,8 @@ import ProbabilityMass = LexicalModelTypes.ProbabilityMass; import Transform = LexicalModelTypes.Transform; export class SubstitutionQuotientSpur extends SearchQuotientSpur { - public insertLength: number; - public leftDeleteLength: number; + public readonly insertLength: number; + public readonly leftDeleteLength: number; constructor( parentNode: SearchQuotientNode, diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/quotient-node-finalizer.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/quotient-node-finalizer.tests.ts index 4feb701f38..e614867c9c 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/quotient-node-finalizer.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/quotient-node-finalizer.tests.ts @@ -15,14 +15,15 @@ import { default as defaultBreaker } from '@keymanapp/models-wordbreakers'; import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; import { + EDIT_DISTANCE_COST_SCALE, generateSubsetId, - LegacyQuotientSpur, models, PathInputProperties, PathResult, QuotientNodeFinalizer, SearchQuotientNode, SearchQuotientRoot, + SubstitutionQuotientSpur, TokenResultMapping } from '@keymanapp/lm-worker/test-index'; @@ -76,10 +77,9 @@ function buildFixture_therefore() { }; }); - // TODO: Use SubstitutionQuotientSpur instead! let quotientNodes: SearchQuotientNode[] = [new SearchQuotientRoot(plainModel)]; for(let i=0; i < 9; i++) { - quotientNodes.push(new LegacyQuotientSpur(quotientNodes[i], distributions[i], inputSources[i])); + quotientNodes.push(new SubstitutionQuotientSpur(quotientNodes[i], distributions[i], inputSources[i])); } return quotientNodes; @@ -99,7 +99,7 @@ describe('QuotientNodeFinalizer', () => { assert.equal(searchResult.type, 'complete'); if(searchResult.type == 'complete') { - assert.equal(searchResult.mapping.totalCost, -Math.log(therefo.bestExample.p)); + assert.approximately(searchResult.mapping.totalCost, -Math.log(therefo.bestExample.p), Number.EPSILON * 1000); assert.isNotNaN(searchResult.cost); assert.equal(searchResult.cost, searchResult.mapping.totalCost); } else { @@ -107,14 +107,16 @@ describe('QuotientNodeFinalizer', () => { } searchResult = therefo.handleNextNode(); - // There should be more results that may be found. + // There should be more searching to perform before aborting. assert.notEqual(searchResult.type, 'none'); + // However, no other valid results are within correction range + // while rooted on 6 input transforms. do { searchResult = therefo.handleNextNode(); } while(searchResult.type == 'intermediate'); - assert.notEqual(searchResult.type, 'none'); + assert.equal(searchResult.type, 'none'); }); it('finds only corrections when predictions are forbidden', () => { @@ -130,6 +132,11 @@ describe('QuotientNodeFinalizer', () => { assert.equal(searchResult.type, 'complete'); if(searchResult.type == 'complete') { assert.isAbove(searchResult.mapping.totalCost, -Math.log(therefo.bestExample.p)); + + // There are two codepoints missing that are necessary to complete a + // full word with the represented prefix. Check that the penalty is set + // appropriately, accounting for floating-point precision issues. + assert.isAtLeast(searchResult.mapping.totalCost, -Math.log(therefo.bestExample.p) + EDIT_DISTANCE_COST_SCALE * 1.99); assert.isNotNaN(searchResult.cost); assert.equal(searchResult.cost, searchResult.mapping.totalCost); } else { @@ -137,14 +144,16 @@ describe('QuotientNodeFinalizer', () => { } searchResult = therefo.handleNextNode(); - // There should be more results that may be found. + // There should be more searching to perform before aborting. assert.notEqual(searchResult.type, 'none'); do { searchResult = therefo.handleNextNode(); } while(searchResult.type == 'intermediate'); - assert.notEqual(searchResult.type, 'none'); + // However, no other valid results are within correction range + // while rooted on 6 input transforms. + assert.equal(searchResult.type, 'none'); }); }); }); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts index 59ae32846f..435dfd66b6 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts @@ -21,12 +21,12 @@ import { correctionValidForAutoSelect, generateSubsetId, getBestMatches, - LegacyQuotientSpur, models, PathInputProperties, PathResult, SearchQuotientNode, SearchQuotientRoot, + SubstitutionQuotientSpur, TokenizationCorrector, TokenResult, TokenizationResultMapping @@ -86,22 +86,21 @@ function buildFixture_therefore() { const therefTokens: ContextToken[] = []; // as in "therefore" const the_efTokens: ContextToken[] = []; // as in "the effect" - // TODO: Use SubstitutionQuotientSpur instead! let firstTokenNode: SearchQuotientNode = new SearchQuotientRoot(plainModel); for(let i=0; i < 3; i++) { - firstTokenNode = new LegacyQuotientSpur(firstTokenNode, distributions[i], inputSources[i]); + firstTokenNode = new SubstitutionQuotientSpur(firstTokenNode, distributions[i], inputSources[i]); } the_efTokens.push(new ContextToken(firstTokenNode, false)); - firstTokenNode = new LegacyQuotientSpur(firstTokenNode, [distributions[3][1]], { + firstTokenNode = new SubstitutionQuotientSpur(firstTokenNode, [distributions[3][1]], { ...inputSources[3], subsetId: generateSubsetId() }); // whitespace token alternate - using the ' ' input instead. const whitespaceToken = new ContextToken( - new LegacyQuotientSpur( + new SubstitutionQuotientSpur( new SearchQuotientRoot(plainModel), [distributions[3][0]], { ...inputSources[3], subsetId: generateSubsetId() } @@ -112,11 +111,11 @@ function buildFixture_therefore() { let secondTokenNode: SearchQuotientNode = new SearchQuotientRoot(plainModel); for(let i=4; i < distributions.length; i++) { - firstTokenNode = new LegacyQuotientSpur(firstTokenNode, distributions[i], { + firstTokenNode = new SubstitutionQuotientSpur(firstTokenNode, distributions[i], { ...inputSources[i], subsetId: generateSubsetId() }); - secondTokenNode = new LegacyQuotientSpur(secondTokenNode, distributions[i], { + secondTokenNode = new SubstitutionQuotientSpur(secondTokenNode, distributions[i], { ...inputSources[i], subsetId: generateSubsetId() }) @@ -171,17 +170,16 @@ function buildFixture_terminalWhitespace() { const fullTokens: ContextToken[] = []; const lastToken: ContextToken[] = []; - // TODO: Use SubstitutionQuotientSpur instead! let firstTokenNode: SearchQuotientNode = new SearchQuotientRoot(plainModel); for(let i=0; i < 5; i++) { - firstTokenNode = new LegacyQuotientSpur(firstTokenNode, distributions[i], inputSources[i]); + firstTokenNode = new SubstitutionQuotientSpur(firstTokenNode, distributions[i], inputSources[i]); } fullTokens.push(new ContextToken(firstTokenNode, false)); // whitespace token alternate - using the ' ' input instead. const whitespaceToken = new ContextToken( - new LegacyQuotientSpur( + new SubstitutionQuotientSpur( new SearchQuotientRoot(plainModel), distributions[5], inputSources[5], @@ -316,14 +314,16 @@ describe('TokenizationCorrector', () => { } searchResult = instance.handleNextNode(); - // There should be more results that may be found. + // There should be more searching to perform before aborting. assert.notEqual(searchResult.type, 'none'); do { searchResult = instance.handleNextNode(); } while(searchResult.type == 'intermediate'); - assert.notEqual(searchResult.type, 'none'); + // However, no other valid results are within correction range + // while rooted on 6 input transforms. + assert.equal(searchResult.type, 'none'); }); it('finds corrections for a group of tokens with two correctable', () => {