From 49f7d265ccc2fd68133e0d8d1f8d32ad95700d38 Mon Sep 17 00:00:00 2001 From: jahorton Date: Mon, 26 Oct 2020 15:11:03 +0700 Subject: [PATCH 1/4] feat(common/models): compacts context tokens on bksp --- .../worker/correction/context-tracker.ts | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/common/predictive-text/worker/correction/context-tracker.ts b/common/predictive-text/worker/correction/context-tracker.ts index 7c482703cc..6a0dc90183 100644 --- a/common/predictive-text/worker/correction/context-tracker.ts +++ b/common/predictive-text/worker/correction/context-tracker.ts @@ -359,7 +359,28 @@ namespace correction { token.transformDistributions = [transformDistribution]; state.pushTail(token); } else { - state.updateTail(transformDistribution, tokenizedContext[0]); + // Consider backspace entry for this case? + let primaryInput = transformDistribution[0].sample; + if(primaryInput && primaryInput.insert == "" && primaryInput.deleteLeft > 0) { + // It's a backspace transform; time for special handling! + // + // For now, with 14.0, we simply compress all remaining Transforms for the token into a single + // one. Probabalistically modeling BKSP is quite complex, so we simplify by assuming everything + // remaining after a BKSP is 'true' and 'intended' text. + let compactedTransform: Transform = { + insert: tokenizedContext[0], + deleteLeft: 0, + id: primaryInput.id // We compact all previous transforms into the new one. + }; + state.tokens.pop(); + + let compactedToken = new TrackedContextToken(); + compactedToken.raw = tokenizedContext[0]; + compactedToken.transformDistributions = [[{sample: compactedTransform, p: 1.0}]]; + state.pushTail(compactedToken); + } else { + state.updateTail(transformDistribution, tokenizedContext[0]); + } } } return state; From cb3a8baa6ba228e3b9c21066071666136977810f Mon Sep 17 00:00:00 2001 From: jahorton Date: Wed, 23 Dec 2020 09:35:17 +0700 Subject: [PATCH 2/4] feat(common/models): paradigm shift; single-char transform sequence for workaround --- .../worker/correction/context-tracker.ts | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/common/predictive-text/worker/correction/context-tracker.ts b/common/predictive-text/worker/correction/context-tracker.ts index be94849022..3527cea648 100644 --- a/common/predictive-text/worker/correction/context-tracker.ts +++ b/common/predictive-text/worker/correction/context-tracker.ts @@ -363,19 +363,28 @@ namespace correction { if(primaryInput && primaryInput.insert == "" && primaryInput.deleteLeft > 0) { // It's a backspace transform; time for special handling! // - // For now, with 14.0, we simply compress all remaining Transforms for the token into a single - // one. Probabalistically modeling BKSP is quite complex, so we simplify by assuming everything - // remaining after a BKSP is 'true' and 'intended' text. - let compactedTransform: Transform = { - insert: tokenizedContext[0], - deleteLeft: 0, - id: primaryInput.id // We compact all previous transforms into the new one. - }; + // For now, with 14.0, we simply compress all remaining Transforms for the token into + // multiple single-char transforms. Probabalistically modeling BKSP is quite complex, + // so we simplify by assuming everything remaining after a BKSP is 'true' and 'intended' text. + // + // Note that we cannot just use a single, monolithic transform at this point b/c + // of our current edit-distance optimization strategy; diagonalization is currently... + // not very compatible with that. + let backspacedTokenContext = tokenizedContext[0].split('').map(function(char) { + let transform: Transform = { + insert: char, + deleteLeft: 0, + id: primaryInput.id // Not exactly optimal for every transform to have the same ID, + // but is actually accurate here. + }; + + return [{sample: transform, p: 1.0}]; + }); state.tokens.pop(); let compactedToken = new TrackedContextToken(); compactedToken.raw = tokenizedContext[0]; - compactedToken.transformDistributions = [[{sample: compactedTransform, p: 1.0}]]; + compactedToken.transformDistributions = backspacedTokenContext; state.pushTail(compactedToken); } else { state.updateTail(transformDistribution, tokenizedContext[0]); From f2250fdc52b73ace32009821bb272d666f18b2d4 Mon Sep 17 00:00:00 2001 From: jahorton Date: Wed, 23 Dec 2020 09:46:50 +0700 Subject: [PATCH 3/4] fix(common/models): null guard for prior commits --- common/predictive-text/worker/correction/context-tracker.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/predictive-text/worker/correction/context-tracker.ts b/common/predictive-text/worker/correction/context-tracker.ts index 3527cea648..d31363f999 100644 --- a/common/predictive-text/worker/correction/context-tracker.ts +++ b/common/predictive-text/worker/correction/context-tracker.ts @@ -359,7 +359,8 @@ namespace correction { state.pushTail(token); } else { // Consider backspace entry for this case? - let primaryInput = transformDistribution[0].sample; + let hasDistribution = transformDistribution && Array.isArray(transformDistribution); + let primaryInput = hasDistribution ? transformDistribution[0].sample : null; if(primaryInput && primaryInput.insert == "" && primaryInput.deleteLeft > 0) { // It's a backspace transform; time for special handling! // From b677b68c31e7fd841527da6db2ee42e76f784c66 Mon Sep 17 00:00:00 2001 From: jahorton Date: Wed, 23 Dec 2020 09:47:26 +0700 Subject: [PATCH 4/4] change(common/models): diagonal width boost to help multi-char transforms --- .../predictive-text/worker/correction/classical-calculation.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/predictive-text/worker/correction/classical-calculation.ts b/common/predictive-text/worker/correction/classical-calculation.ts index 233324a786..3ac31ecd54 100644 --- a/common/predictive-text/worker/correction/classical-calculation.ts +++ b/common/predictive-text/worker/correction/classical-calculation.ts @@ -52,7 +52,8 @@ namespace correction { * Otherwise, this object represents a heuristic that _may_ overestimate the true edit distance. Note that it will * never underestimate. */ - diagonalWidth: number = 1; + diagonalWidth: number = 2; // TODO: Ideally, should start at 1... but we'll start at 2 for now + // as a naive workaround for multi-char transform limitations. // The sequence of characters input so far. inputSequence: TInput[] = [];