diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts index 516ac779fb..29cd455668 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts @@ -346,6 +346,92 @@ interface ContextMatchResult { } export class ContextTracker extends CircularArray { + /** + * Aligns two tokens on a character-by-character basis as needed for higher, token-level alignment + * operations. + * @param incomingToken The incoming token value + * @param matchingToken The pre-existing token value to use for comparison and alignment + * @param forNearCaret If `false`, disallows any substitutions and activates a leading-edge alignment + * validation mode. + * @returns + */ + static isSubstitutionAlignable( + incomingToken: string, + matchingToken: string, + forNearCaret?: boolean + ): boolean { + // 1 - Determine the edit path for the word. + let subEditPath = ClassicalDistanceCalculation.computeDistance( + [...matchingToken].map(value => ({key: value})), + [...incomingToken].map(value => ({key: value})), + // Diagonal width to consider must be at least 2, as adding a single + // whitespace after a token tends to add two tokens: one for whitespace, + // one for the empty token to follow it. + 3 + ).editPath(); + + const firstInsert = subEditPath.indexOf('insert'); + const firstDelete = subEditPath.indexOf('delete'); + + // 2 - deletions and insertions should be mutually exclusive. + // A fixed, unedited word can't slide across both 'left' and 'right' boundaries at the same time. + if(firstInsert != -1 && firstDelete != -1) { + return false; + }; + + // 3 - checks exclusive to leading-edge conditions + if(!forNearCaret) { + const firstSubstitute = subEditPath.indexOf('substitute'); + const firstMatch = subEditPath.indexOf('match'); + if(firstSubstitute > -1) { + return false; + } else if(firstMatch > -1) { + // Should not have inserts on both sides of matched text! + if(firstInsert > -1 && firstInsert < firstMatch && subEditPath.lastIndexOf('insert') > firstMatch) { + return false; + } else if(firstDelete > -1 && firstDelete < firstMatch && subEditPath.lastIndexOf('delete') > firstMatch) { + return false; + } + } + + // Further checks below are oriented for text/tokens at the caret. + return true; + } + + // 4 - check the stats for total edits of each type and validate that edits don't overly exceed + // original characters. + const editCount = { + matchMove: 0, + rawEdit: 0 + }; + + subEditPath.forEach((entry) => { + switch(entry) { + case 'transpose-end': + case 'transpose-start': + case 'match': + editCount.matchMove++; + break; + case 'insert': + case 'transpose-insert': + case 'delete': + case 'transpose-delete': + case 'substitute': + editCount.rawEdit++; + } + }); + + // We shouldn't have more raw substitutions, inserts, and deletes than matches + transposes, + // though allowing +1 as a fudge factor. + // The 'a' => 'à' pattern can be a reasonably common Keyman keyboard rule and + // is one substitution, zero matches in NFC. + if(editCount.matchMove + 1 < editCount.rawEdit) { + return false; + } + + return true; + } + static attemptMatchContext( tokenizedContext: Token[], matchState: TrackedContextState, diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js index 0c21a26b21..65b16c5fc2 100644 --- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js +++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tracker.js @@ -22,6 +22,79 @@ describe('ContextTracker', function() { }]; } + describe('isSubstitutionAlignable', () => { + it(`returns true: 'ca' => 'can'`, () => { + assert.isTrue(ContextTracker.isSubstitutionAlignable('can', 'ca')); + }); + + // Leading word in context window starts sliding out of said window. + it(`returns true: 'can' => 'an'`, () => { + assert.isTrue(ContextTracker.isSubstitutionAlignable('an', 'can')); + }); + + // Same edits on both sides: not valid. + it(`returns false: 'apple' => 'grapples'`, () => { + assert.isFalse(ContextTracker.isSubstitutionAlignable('grapples', 'apple')); + }); + + // Edits on one side: valid. + it(`returns true: 'apple' => 'grapple'`, () => { + assert.isTrue(ContextTracker.isSubstitutionAlignable('grapple', 'apple')); + }); + + // Edits on one side: valid. + it(`returns true: 'apple' => 'grapple'`, () => { + assert.isTrue(ContextTracker.isSubstitutionAlignable('apples', 'apple')); + }); + + // Same edits on both sides: not valid. + it(`returns false: 'grapples' => 'apple'`, () => { + assert.isFalse(ContextTracker.isSubstitutionAlignable('apple', 'grapples')); + }); + + // Substitution: not valid when not permitted via parameter. + it(`returns false: 'apple' => 'banana'`, () => { + // edit path: 'insert' ('b' of banana), 'match' (on leading a), rest are 'substitute'. + assert.isFalse(ContextTracker.isSubstitutionAlignable('banana', 'apple')); + }); + + // Substitution: not valid if too much is substituted, even if allowed via parameter. + it(`returns false: 'apple' => 'banana' (subs allowed)`, () => { + // edit path: 'insert' ('b' of banana), 'match' (on leading a), rest are 'substitute'. + // 1 match vs 4 substitute = no bueno. It'd require too niche of a keyboard rule. + assert.isFalse(ContextTracker.isSubstitutionAlignable('banana', 'apple', true)); + }); + + it(`returns true: 'a' => 'à' (subs allowed)`, () => { + assert.isTrue(ContextTracker.isSubstitutionAlignable('à', 'a', true)); + }); + + // Leading substitution: valid if enough of the remaining word matches. + // Could totally happen from a legit Keyman keyboard rule. + it(`returns true: 'can' => 'van' (subs allowed)`, () => { + assert.isTrue(ContextTracker.isSubstitutionAlignable('van', 'can', true)); + }); + + // Trailing substitution: invalid if not allowed. + it(`returns false: 'can' => 'cap' (subs not allowed)`, () => { + assert.isFalse(ContextTracker.isSubstitutionAlignable('cap', 'can')); + }); + + // Trailing substitution: valid. + it(`returns false: 'can' => 'cap' (subs allowed)`, () => { + assert.isTrue(ContextTracker.isSubstitutionAlignable('cap', 'can', true)); + }); + + it(`returns true: 'clasts' => 'clasps' (subs allowed)`, () => { + assert.isTrue(ContextTracker.isSubstitutionAlignable('clasps', 'clasts', true)); + }); + + // random deletion at the start + later substitution = still permitted + it(`returns false: 'clasts' => 'clasps' (subs allowed)`, () => { + assert.isTrue(ContextTracker.isSubstitutionAlignable('lasps', 'clasts', true)); + }); + }); + describe('attemptMatchContext', function() { it("properly matches and aligns when lead token is removed", function() { let existingContext = models.tokenize(defaultBreaker, {