diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 174f185c0e..e65d30ebea 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -13,6 +13,7 @@ import { KMWString } from 'keyman/common/web-utils'; import { ContextToken } from './context-token.js'; import { TransformUtils } from '../transformUtils.js'; import { computeDistance, EditOperation, EditTuple } from './classical-calculation.js'; +import { LegacyQuotientRoot } from './legacy-quotient-root.js'; import { determineModelTokenizer } from '../model-helpers.js'; import { ExtendedEditOperation, SegmentableDistanceCalculation } from './segmentable-calculation.js'; import { PathInputProperties } from './search-quotient-node.js'; @@ -20,7 +21,6 @@ import { TransitionEdge } from './tokenization-subsets.js'; import LexicalModel = LexicalModelTypes.LexicalModel; import Transform = LexicalModelTypes.Transform; -import { LegacyQuotientRoot } from './legacy-quotient-root.js'; // May be able to "get away" with 2 & 5 or so, but having extra will likely help // with edit path stability. @@ -565,7 +565,7 @@ export class ContextTokenization { removedTokenCount }, tokenizedTransform: transformMap, - isBksp: transform.insert == '' && transform.deleteLeft == 1 + isBksp: TransformUtils.isBackspace(transform) }; } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/transition-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/transition-helpers.ts index 7d78f20609..754b30e40a 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/transition-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/transition-helpers.ts @@ -13,6 +13,7 @@ import { ContextToken } from './context-token.js'; import { ContextTokenization } from './context-tokenization.js'; import { SearchQuotientCluster } from './search-quotient-cluster.js'; import { legacySubsetKeyer, TokenizationSubset, TokenizationSubsetBuilder } from './tokenization-subsets.js'; +import { TransformUtils } from '#./transformUtils.js'; import Distribution = LexicalModelTypes.Distribution; import Transform = LexicalModelTypes.Transform; @@ -53,7 +54,7 @@ export function precomputeTransitions( const trueInput = transformDistribution[0].sample; const lexicalModel = startTokenizations[0]?.tail.searchModule.model; - if(trueInput.insert == '' && trueInput.deleteLeft == 0) { + if(TransformUtils.isBackspace(trueInput)) { transformDistribution = [transformDistribution[0]]; } @@ -139,23 +140,26 @@ export function transitionTokenizations( // If the last token is empty and has no flag for a revertable transition, // attempt to copy the previous token's revertable transition flag. const tokens = remadeTokenization.tokens; - const lastTokenIndex = tokens.length - 1; - if(tokens[lastTokenIndex].isEmptyToken && tokens[lastTokenIndex-1]) { - tokens[lastTokenIndex].appliedTransitionId ??= tokens[lastTokenIndex-1].appliedTransitionId - } + // If we have a pure backspace operation, we should forget fat-finger data + // and reconstruct the token without corrective data. if(precomp[1].isBksp) { const appliedEdge = precomp[1]; - const affectedTokenCount = appliedEdge.inputs[0].sample.size; + const affectedTokenCount = appliedEdge.inputs[0].sample.size - appliedEdge.alignment.removedTokenCount; for(let i = 0; i < affectedTokenCount; i++) { const index = tokens.length - affectedTokenCount + i; const token = tokens[index]; - remadeTokenization.tokens[index] = ContextToken.fromRawText(token.searchModule.model, token.exampleInput, token.isPartial, trueInput.id); + tokens[index] = ContextToken.fromRawText(token.searchModule.model, token.exampleInput, token.isPartial, trueInput.id); } } + const lastTokenIndex = tokens.length - 1; + if(tokens[lastTokenIndex].isEmptyToken && tokens[lastTokenIndex-1]) { + tokens[lastTokenIndex].appliedTransitionId ??= tokens[lastTokenIndex-1].appliedTransitionId + } + return remadeTokenization; }); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/transition-helpers.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/transition-helpers.tests.ts index 882fc5a9d4..3e689365a2 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/transition-helpers.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/transition-helpers.tests.ts @@ -28,6 +28,7 @@ import { SearchQuotientCluster, TokenizationSubset, TokenizationTransitionEdits, + TransformUtils, TransitionEdge, transitionTokenizations } from '@keymanapp/lm-worker/test-index'; @@ -52,7 +53,8 @@ const plainModel = new TrieModel( function buildOutboundTransitionEdge ( baseTokenization: ContextTokenization, inputs: Distribution>, - tokenizedInputs: Distribution> + tokenizedInputs: Distribution>, + isBksp: boolean ): TransitionEdge { const primaryTokenizedInput = tokenizedInputs[0].sample; const relativeTailIndex = [...primaryTokenizedInput.keys()][0]; @@ -77,7 +79,8 @@ function buildOutboundTransitionEdge ( removedTokenCount: 0 }, inputs: tokenizedInputs, - inputSubsetId: generateSubsetId() + inputSubsetId: generateSubsetId(), + isBksp }; } @@ -132,14 +135,16 @@ function generateFixtureForTokenizationOutboundTransition ( inputPropBase: PathInputProperties ) { return dists.map((dist) => { + const isBksp = TransformUtils.isBackspace(dist.raw[0].sample); const primaryTokenizedInput = dist.tokenized[0].sample; - const tokenizationEdge = buildOutboundTransitionEdge(srcTokenization, dist.raw, dist.tokenized); + const tokenizationEdge = buildOutboundTransitionEdge(srcTokenization, dist.raw, dist.tokenized, isBksp); // Is only built for use in constructing the subset keys. We only need data // from one of the inputs here. const keyable: TokenizationTransitionEdits = { tokenizedTransform: primaryTokenizedInput, - alignment: tokenizationEdge.alignment + alignment: tokenizationEdge.alignment, + isBksp }; const key = precomputationSubsetKeyer(keyable);