diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts index e6064f4e95..a068a45ac5 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts @@ -257,7 +257,7 @@ export class ContextState { const tokens = resultTokenization.tokens; const lastIndex = tokens.length - 1; // Ignore a context-final empty '' token; the interesting one is what comes before. - const nonEmptyTail = tokens[lastIndex].sourceText != '' ? tokens[lastIndex] : tokens[lastIndex - 1]; + const nonEmptyTail = !tokens[lastIndex].isEmptyToken ? tokens[lastIndex] : tokens[lastIndex - 1]; const appliedSuggestionTransitionId = nonEmptyTail?.appliedTransitionId; // Used to construct and represent the part of the incoming transform that diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts index 7ed6be7503..0a67daf433 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts @@ -16,6 +16,14 @@ import Distribution = LexicalModelTypes.Distribution; import LexicalModel = LexicalModelTypes.LexicalModel; import Transform = LexicalModelTypes.Transform; +/** + * Notes critical properties of the inputs comprising each ContextToken. + */ +export interface TokenInputSource { + trueTransform: Transform; + inputStartIndex: number; +} + /** * Breaks apart a raw text string into individual, single-codepoint * transforms, all set with the specified transform ID. @@ -65,7 +73,7 @@ export class ContextToken { * applied to the actual context for the set of keystrokes contributing to * this token. */ - private _inputRange: Transform[]; + private _inputRange: TokenInputSource[]; /** * Constructs a new, empty instance for use with the specified LexicalModel. @@ -117,7 +125,10 @@ export class ContextToken { return [{sample: transform, p: 1.0}]; }); rawTransformDistributions.forEach((entry) => { - this._inputRange.push(entry[0].sample); + this._inputRange.push({ + trueTransform: entry[0].sample, + inputStartIndex: 0 + }); this.searchSpace.addInput(entry); }); } @@ -127,26 +138,52 @@ export class ContextToken { * Call this to record the original keystroke Transforms for the context range * corresponding to this token. */ - addSourceInput(transform: Transform) { - this._inputRange.push(transform); + addInput(inputSource: TokenInputSource, distribution: Distribution) { + this._inputRange.push(inputSource); + this.searchSpace.addInput(distribution); } /** * Denotes the original keystroke Transforms comprising the range corresponding * to this token. */ - get inputRange(): Readonly { + get inputRange(): Readonly { return this._inputRange; } /** - * Gets a simple, human-readable representation of `inputRange`. + * Indicates whether or not this ContextToken likely represents an empty token. + */ + get isEmptyToken(): boolean { + return this.exampleInput == ''; + } + + /** + * Gets a compact string-based representation of `inputRange` that + * maps compatible token source ranges to each other. + */ + get sourceRangeKey(): string { + const components: string[] = []; + + for(const source of this.inputRange) { + const i = source.inputStartIndex; + components.push(`T${source.trueTransform.id}${i != 0 ? '@' + i : ''}`); + } + + return components.join('+'); + } + + /** + * Gets a simple, compact string-based representation of `inputRange`. * - * Should not actually be used in code - its use is intended only for - * debugging. + * This should only ever be used for debugging purposes. */ get sourceText(): string { - const composite = this._inputRange.reduce((accum, current) => buildMergedTransform(accum, current), { insert: '', deleteLeft: 0 }); + const composite = this._inputRange.reduce((accum, current) => { + const alteredTransform = {...current.trueTransform}; + alteredTransform.insert = alteredTransform.insert.slice(current.inputStartIndex); + return buildMergedTransform(accum, current.trueTransform) + }, { insert: '', deleteLeft: 0 }); const prefix = '\u{2421}'.repeat(composite.deleteLeft); return prefix + composite.insert; } @@ -166,7 +203,7 @@ export class ContextToken { * most likely keystroke data afterward. */ const transforms = this.searchSpace.inputSequence.map((dist) => dist[0].sample) - const composite = transforms.reduce((accum, current) => buildMergedTransform(accum, current), { insert: '', deleteLeft: 0}); + const composite = transforms.reduce((accum, current) => buildMergedTransform(accum, current), {insert: '', deleteLeft: 0}); return composite.insert; } } \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 73eacf2f84..4fb071f14d 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -40,7 +40,7 @@ interface TokenMergeMap { interface TokenSplitMap { input: EditTokenMap, - matches: EditTokenMap[] + matches: (EditTokenMap & { textOffset: number })[] }; /** @@ -75,11 +75,11 @@ export class ContextTokenization { /** * Returns plain-text strings representing the most probable representation for all * tokens represented by this tokenization instance. + * + * Intended for debugging use only. */ get sourceText() { - return this.tokens - .filter(token => token.sourceText !== null) - .map(token => token.sourceText); + return this.tokens.map(token => token.sourceText); } /** @@ -87,10 +87,7 @@ export class ContextTokenization { * tokens represented by this tokenization instance. */ get exampleInput(): string[] { - return this.tokens - // Hide any tokens representing invisible wordbreaks. (Thinking ahead to phrase-level possibilities) - .filter(token => token.exampleInput !== null) - .map(token => token.exampleInput); + return this.tokens.map(token => token.exampleInput); } /** @@ -104,7 +101,7 @@ export class ContextTokenization { * the tokenization modeled by this instance. */ computeAlignment(incomingTokenization: string[], isSliding: boolean, noSubVerify?: boolean): ContextStateAlignment { - return computeAlignment(this.sourceText, incomingTokenization, isSliding, noSubVerify); + return computeAlignment(this.exampleInput, incomingTokenization, isSliding, noSubVerify); } /** @@ -368,6 +365,7 @@ export class ContextTokenization { // edited, those edits occur to the left as well - and further left of whatever // the new tail token is *if* tokens were removed. const firstTailEditIndex = Math.min((1 - tailEditLength), 0) + Math.min(tailTokenShift, 0); + let primaryInputAppliedLen = 0; for(let i = 0; i < tailEditLength; i++) { const tailIndex = firstTailEditIndex + i; @@ -393,11 +391,12 @@ export class ContextTokenization { // Erase any applied-suggestion transition ID; it is no longer valid. token.appliedTransitionId = undefined; const emptySample: ProbabilityMass = { sample: { insert: '', deleteLeft: 0 }, p: 1 }; - token.addSourceInput(primaryInput ?? emptySample.sample); - token.searchSpace.addInput(tokenDistribution.map((seq) => seq.get(tailIndex) ?? emptySample)); + const dist = tokenDistribution.map((seq) => seq.get(tailIndex) ?? emptySample); + token.addInput({trueTransform: primaryInput ?? emptySample.sample, inputStartIndex: primaryInputAppliedLen}, dist); } tokenization[incomingIndex] = token; + primaryInputAppliedLen += KMWString.length(primaryInput?.insert ?? ''); } if(tailTokenShift < 0) { @@ -454,11 +453,10 @@ export class ContextTokenization { // If there are no entries in our would-be distribution, there's no // reason to pass in what amounts to a no-op. if(transformDistribution) { - pushedToken.addSourceInput(primaryInput); // If we ever stop filtering tokenized transform distributions, it may // be worth adding an empty transform here with weight to balance // the distribution back to a cumulative prob sum of 1. - pushedToken.searchSpace.addInput(transformDistribution); + pushedToken.addInput({ trueTransform: primaryInput, inputStartIndex: primaryInputAppliedLen }, transformDistribution); } } else if(incomingToken.text) { // We have no transform data to match against an inserted token with text; abort! @@ -470,6 +468,7 @@ export class ContextTokenization { // Auto-replaces the search space to correspond with the new token. tokenization.push(pushedToken); + primaryInputAppliedLen += KMWString.length(primaryInput.insert); } } @@ -848,12 +847,12 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo text: preTokenization[input] }] }; - let currentMerge: string; + let currentMerge = preTokenization[input]; let inputLookahead = 1; // Look-ahead 1 - let nextMerge = preTokenization[input] + preTokenization[input + inputLookahead++]; + let nextMerge = currentMerge + preTokenization[input + inputLookahead++]; // Conditional validates if look-ahead 1 passes (which it should) - for(/* next line */; mergeTarget.indexOf(nextMerge) == 0; nextMerge = preTokenization[input + inputLookahead++]) { + for(/* next line */; mergeTarget.indexOf(nextMerge) == 0; nextMerge = currentMerge + preTokenization[input + inputLookahead++]) { merge.inputs.push({ index: input + inputLookahead - 1, text: preTokenization[input + inputLookahead-1] @@ -876,18 +875,21 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo }, matches: [ { index: match, - text: resultTokenization[match] + text: resultTokenization[match], + textOffset: 0 }], }; - let currentMerge: string; + let currentMerge = resultTokenization[match]; matchOffset = 1; // Look-ahead 1 - let nextMerge = resultTokenization[match] + resultTokenization[match + matchOffset++]; - for(/* next line */; splitTarget.indexOf(nextMerge) == 0; nextMerge = preTokenization[match + matchOffset++]) { + let nextMerge = currentMerge + resultTokenization[match + matchOffset++]; + for(/* next line */; splitTarget.indexOf(nextMerge) == 0; nextMerge = currentMerge + preTokenization[match + matchOffset++]) { + const textOffset = KMWString.length(currentMerge); currentMerge = nextMerge; split.matches.push({ index: match + matchOffset - 1, - text: resultTokenization[match + matchOffset-1] + text: resultTokenization[match + matchOffset-1], + textOffset }); // Each time we 'pass' the condition, we've successfully processed an associated edit. diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index 08774c218c..2252549b16 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -394,7 +394,7 @@ export function determineSuggestionAlignment( // Did the wordbreaker (or similar) append a blank token before the caret? If so, // preserve that by preventing corrections from triggering left-deletion. - if(transition.final.tokenization.tail.sourceText == '') { + if(transition.final.tokenization.tail.isEmptyToken) { deleteLeft = 0; } diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts index ed6ca9c976..b395ad2d41 100644 --- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts @@ -1062,7 +1062,7 @@ describe('ContextTokenization', function() { merges: [], splits: [ { input: { text: 'can\'', index: 7 }, - matches: [ { text: 'can', index: 7 }, { text: '\'', index: 8 }] + matches: [ { text: 'can', index: 7, textOffset: 0 }, { text: '\'', index: 8, textOffset: 3 }] } ], mergeOffset: 0, splitOffset: -1,