diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts index 9426012af9..90cdcabb77 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts @@ -97,7 +97,7 @@ export class ContextToken { * @param model * @param rawText */ - static fromRawText(model: LexicalModel, rawText: string, isPartial?: boolean) { + static fromRawText(model: LexicalModel, rawText: string, isPartial?: boolean, transitionId?: number) { rawText ||= ''; // Supports the old pathway for: updateWithBackspace(tokenText: string, transitionId: number) @@ -108,7 +108,7 @@ export class ContextToken { let inputMetadata: PathInputProperties = { segment: { start: 0, - transitionId: undefined + transitionId: transitionId }, bestProbFromSet: BASE_PROBABILITY, subsetId: generateSubsetId() diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 61b06092ab..174f185c0e 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -154,6 +154,11 @@ export interface TokenizationTransitionEdits { * the end of the original context's tail token. */ tokenizedTransform: Map; + + /** + * Indicates that this tokenization-transition handles a backspace transform. + */ + isBksp?: boolean; } /** @@ -560,6 +565,7 @@ export class ContextTokenization { removedTokenCount }, tokenizedTransform: transformMap, + isBksp: transform.insert == '' && transform.deleteLeft == 1 }; } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts index ede7125a9c..32c59aa6eb 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-transition.ts @@ -161,23 +161,30 @@ export class ContextTransition { transformToApply: Transform, inputDistribution: Distribution ) => { + const appliesSuggestion = transformToApply == suggestion.transform; + const appliedDistribution = [{sample: transformToApply, p: 1}]; - const { subsets: applicationSubsets, keyMatchingUserContext } = precomputeTransitions( + const { subsets: transitionSubsets, keyMatchingUserContext } = precomputeTransitions( [rootTokenization], appliedDistribution ); // Filter out insert and delete edges here! ONLY the primary substitution // edge should be permitted! - const directSuggestionSubset: typeof applicationSubsets = new Map(); + let applicationSubsets: typeof transitionSubsets = new Map(); - // When applying suggestions, only consider the actual tokenization that would result. - directSuggestionSubset.set(keyMatchingUserContext, applicationSubsets.get(keyMatchingUserContext)); + const currentContextSubset = transitionSubsets.get(keyMatchingUserContext); + if(appliesSuggestion) { + // When applying suggestions, only consider the actual tokenization that would result. + applicationSubsets.set(keyMatchingUserContext, currentContextSubset); - // TODO: verify that 'insert' and 'delete' edit-spurs are ignored (once - // they're supported) + // TODO: verify that 'insert' and 'delete' edit-spurs are ignored (once + // they're supported) + } else { + applicationSubsets = transitionSubsets; + } const resultingTokenization = transitionTokenizations( - directSuggestionSubset, + applicationSubsets, appliedDistribution ).get(keyMatchingUserContext); diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-subsets.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-subsets.ts index a8617a7c0f..4ff4342187 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-subsets.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/tokenization-subsets.ts @@ -34,6 +34,11 @@ export interface TransitionEdge { */ inputs: Distribution> + /** + * Indicates that the modeled transition handles a raw BKSP. + */ + isBksp?: boolean; + /** * A unique identifier associated with this TransitionEdge and its * transforms within `SearchSpace`s. This ID assists with detecting when @@ -105,6 +110,10 @@ export function editKeyer(precomputation: TokenizationTransitionEdits): string[] }).join(',')); } + if(precomputation.isBksp) { + components.push('ISBKSP'); + } + return components; } @@ -243,7 +252,8 @@ export class TokenizationSubsetBuilder { const forTokenization: TransitionEdge = entry.transitionEdges.get(tokenization) ?? { alignment: precomputation.alignment, inputs: [], - inputSubsetId: generateSubsetId() + inputSubsetId: generateSubsetId(), + isBksp: precomputation.isBksp }; // Adds the incoming tokenized transform data for the pairing... diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/transition-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/transition-helpers.ts index ad520fb041..7d78f20609 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/transition-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/transition-helpers.ts @@ -46,13 +46,17 @@ export function precomputeTransitions( * context edited by the user. */ keyMatchingUserContext: string - } { +} { keyer ??= legacySubsetKeyer; let keyMatchingUserContext: string; const trueInput = transformDistribution[0].sample; const lexicalModel = startTokenizations[0]?.tail.searchModule.model; + if(trueInput.insert == '' && trueInput.deleteLeft == 0) { + transformDistribution = [transformDistribution[0]]; + } + const subsetBuilder = new TokenizationSubsetBuilder(keyer); for(let baseTokenization of startTokenizations) { @@ -140,6 +144,18 @@ export function transitionTokenizations( tokens[lastTokenIndex].appliedTransitionId ??= tokens[lastTokenIndex-1].appliedTransitionId } + if(precomp[1].isBksp) { + const appliedEdge = precomp[1]; + const affectedTokenCount = appliedEdge.inputs[0].sample.size; + + for(let i = 0; i < affectedTokenCount; i++) { + const index = tokens.length - affectedTokenCount + i; + const token = tokens[index]; + + remadeTokenization.tokens[index] = ContextToken.fromRawText(token.searchModule.model, token.exampleInput, token.isPartial, trueInput.id); + } + } + return remadeTokenization; });