From 8f6c2d602d5a8134e7bbc48dc5429d38fb185ed6 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 19 Jul 2024 11:53:53 +0700 Subject: [PATCH 1/4] change(web): track whitespace-aware tokenization for context + correction-search caching --- .../src/main/correction/context-tracker.ts | 396 +++++++++--------- .../lm-worker/src/main/model-compositor.ts | 7 +- .../web/lm-worker/src/main/model-helpers.ts | 8 +- .../web/lm-worker/src/main/predict-helpers.ts | 19 +- .../cases/edit-distance/context-tracker.js | 127 ++++-- 5 files changed, 309 insertions(+), 248 deletions(-) diff --git a/common/web/lm-worker/src/main/correction/context-tracker.ts b/common/web/lm-worker/src/main/correction/context-tracker.ts index 9dde1d6f9c..a8c42e53a1 100644 --- a/common/web/lm-worker/src/main/correction/context-tracker.ts +++ b/common/web/lm-worker/src/main/correction/context-tracker.ts @@ -31,6 +31,7 @@ export class TrackedContextSuggestion { export class TrackedContextToken { raw: string; replacementText: string; + isWhitespace?: boolean; transformDistributions: Distribution[] = []; replacements: TrackedContextSuggestion[]; @@ -54,6 +55,30 @@ export class TrackedContextToken { revert() { delete this.activeReplacementId; } + + /** + * Used for 14.0's backspace workaround, which flattens all previous Distribution + * entries because of limitations with direct use of backspace transforms. + * @param tokenText + * @param transformId + */ + updateWithBackspace(tokenText: USVString, transformId: number) { + // It's a backspace transform; time for special handling! + // + // For now, with 14.0, we simply compress all remaining Transforms for the token into + // multiple single-char transforms. Probabalistically modeling BKSP is quite complex, + // so we simplify by assuming everything remaining after a BKSP is 'true' and 'intended' text. + // + // Note that we cannot just use a single, monolithic transform at this point b/c + // of our current edit-distance optimization strategy; diagonalization is currently... + // not very compatible with that. + let backspacedTokenContext: Distribution[] = textToCharTransforms(tokenText, transformId).map(function(transform) { + return [{sample: transform, p: 1.0}]; + }); + + this.raw = tokenText; + this.transformDistributions = backspacedTokenContext; + } } export class TrackedContextState { @@ -126,7 +151,7 @@ export class TrackedContextState { } popHead() { - this.tokens.splice(0, 2); + this.tokens.splice(0, 1); this.indexOffset -= 1; } @@ -144,59 +169,57 @@ export class TrackedContextState { } } - pushWhitespaceToTail(transformDistribution: Distribution = null) { - let whitespaceToken = new TrackedContextToken(); + // pushWhitespaceToTail(transformDistribution: Distribution = null) { + // let whitespaceToken = new TrackedContextToken(); - // Track the Transform that resulted in the whitespace 'token'. - // Will be needed for phrase-level correction/prediction. - whitespaceToken.transformDistributions = transformDistribution ? [transformDistribution] : []; + // // Track the Transform that resulted in the whitespace 'token'. + // // Will be needed for phrase-level correction/prediction. + // whitespaceToken.transformDistributions = transformDistribution ? [transformDistribution] : []; - whitespaceToken.raw = null; - this.tokens.push(whitespaceToken); - } + // whitespaceToken.raw = null; + // this.tokens.push(whitespaceToken); + // } - /** - * Used for 14.0's backspace workaround, which flattens all previous Distribution - * entries because of limitations with direct use of backspace transforms. - * @param tokenText - * @param transformId - */ - replaceTailForBackspace(tokenText: USVString, transformId: number) { - this.tokens.pop(); + // /** + // * Used for 14.0's backspace workaround, which flattens all previous Distribution + // * entries because of limitations with direct use of backspace transforms. + // * @param tokenText + // * @param transformId + // */ + // replaceTailForBackspace(tokenText: USVString, transformId: number) { + // this.tokens.pop(); - // It's a backspace transform; time for special handling! - // - // For now, with 14.0, we simply compress all remaining Transforms for the token into - // multiple single-char transforms. Probabalistically modeling BKSP is quite complex, - // so we simplify by assuming everything remaining after a BKSP is 'true' and 'intended' text. - // - // Note that we cannot just use a single, monolithic transform at this point b/c - // of our current edit-distance optimization strategy; diagonalization is currently... - // not very compatible with that. - let backspacedTokenContext: Distribution[] = textToCharTransforms(tokenText, transformId).map(function(transform) { - return [{sample: transform, p: 1.0}]; - }); + // // It's a backspace transform; time for special handling! + // // + // // For now, with 14.0, we simply compress all remaining Transforms for the token into + // // multiple single-char transforms. Probabalistically modeling BKSP is quite complex, + // // so we simplify by assuming everything remaining after a BKSP is 'true' and 'intended' text. + // // + // // Note that we cannot just use a single, monolithic transform at this point b/c + // // of our current edit-distance optimization strategy; diagonalization is currently... + // // not very compatible with that. + // let backspacedTokenContext: Distribution[] = textToCharTransforms(tokenText, transformId).map(function(transform) { + // return [{sample: transform, p: 1.0}]; + // }); - let compactedToken = new TrackedContextToken(); - compactedToken.raw = tokenText; - compactedToken.transformDistributions = backspacedTokenContext; - this.pushTail(compactedToken); - } - - updateTail(transformDistribution: Distribution, tokenText?: USVString) { - let editedToken = this.tail; + // let compactedToken = new TrackedContextToken(); + // compactedToken.raw = tokenText; + // compactedToken.transformDistributions = backspacedTokenContext; + // this.pushTail(compactedToken); + // } + updateToken(token: TrackedContextToken, transformDistribution: Distribution, tokenText?: USVString) { // Preserve existing text if new text isn't specified. - tokenText = tokenText || (tokenText === '' ? '' : editedToken.raw); + tokenText = tokenText || (tokenText === '' ? '' : token.raw); if(transformDistribution && transformDistribution.length > 0) { - editedToken.transformDistributions.push(transformDistribution); + token.transformDistributions.push(transformDistribution); if(this.searchSpace) { this.searchSpace.forEach(space => space.addInput(transformDistribution)); } } // Replace old token's raw-text with new token's raw-text. - editedToken.raw = tokenText; + token.raw = tokenText; } toRawTokenization() { @@ -295,7 +318,8 @@ class CircularArray { */ item(index: number) { if(index >= this.count) { - throw "Invalid array index"; + // JS arrays return `undefined` for invalid array indices. + return undefined; } let mappedIndex = (this.currentTail + index) % this.maxCount; @@ -304,187 +328,170 @@ class CircularArray { } export class ContextTracker extends CircularArray { - static attemptMatchContext(tokenizedContext: USVString[], - matchState: TrackedContextState, - transformDistribution?: Distribution): TrackedContextState { + static attemptMatchContext( + tokenizedContext: { text: USVString, isWhitespace?: boolean } [], + matchState: TrackedContextState, + transformDistribution?: Distribution + ): TrackedContextState { // Map the previous tokenized state to an edit-distance friendly version. let matchContext: USVString[] = matchState.toRawTokenization(); // Inverted order, since 'match' existed before our new context. - let mapping = ClassicalDistanceCalculation.computeDistance(matchContext.map(value => ({key: value})), - tokenizedContext.map(value => ({key: value})), - 1); + let mapping = ClassicalDistanceCalculation.computeDistance( + matchContext.map(value => ({key: value})), + tokenizedContext.map(value => ({key: value.text})), + // Must be at least 2, as adding a single whitespace after a token tends + // to add two tokens: one for whitespace, one for the empty token to + // follow it. + 3 + ); let editPath = mapping.editPath(); - let poppedHead = false; - let pushedTail = false; + // When the context has but two tokens, the path algorithm tends to invert + // 'insert' and 'substitute' from our preferred ordering for them. + // Logically, either order makes sense... but logic for other cases is + // far simpler if we have 'substitute' before 'insert'. + if(editPath.length == 2 && editPath[0] == 'insert' && editPath[1] == 'substitute') { + editPath[0] = 'substitute'; + editPath[1] = 'insert'; + } - // Matters greatly when starting from a nil context. - if(editPath.length > 1) { - // First entry: may not be an 'insert' or a 'transpose' op. - // 'insert' allowed if the next token is 'substitute', as this may occur with an edit path of length 2. - if((editPath[0] == 'insert' && !(editPath[1] == 'substitute' && editPath.length == 2)) || editPath[0].indexOf('transpose') >= 0) { - return null; - } else if(editPath[0] == 'delete') { - poppedHead = true; // a token from the previous state has been wholly removed. + const firstMatch = editPath.indexOf('match'); + const lastMatch = editPath.lastIndexOf('match'); + + // Assertion: for a long context, the bulk of the edit path should be a + // continuous block of 'match' entries. If there's anything else in + // the middle, we have a context mismatch. + if(firstMatch) { + for(let i = firstMatch+1; i < lastMatch; i++) { + if(editPath[i] != 'match') { + return null; + } } } - // Last entry: may not be a 'delete' or a 'transpose' op. - let tailIndex = editPath.length -1; - let ignorePenultimateMatch = false; - if(editPath[tailIndex] == 'delete' || editPath[0].indexOf('transpose') >= 0) { - return null; - } else if(editPath[tailIndex] == 'insert') { - pushedTail = true; - } else if(tailIndex > 0 && editPath[tailIndex-1] == 'insert' && editPath[tailIndex] == 'substitute') { - // Tends to happen when accepting suggestions. - pushedTail = true; - ignorePenultimateMatch = true; + // If we have a perfect match with a pre-existing context, no mutations have + // happened; just re-use the old context state. + if(firstMatch == 0 && lastMatch == editPath.length - 1) { + return matchState; } - // Can happen for the first text input after backspace deletes a wordbreaking character, - // thus the new input continues a previous word while dropping the empty word after - // that prior wordbreaking character. - // - // We can't handle it reliably from this match state, but a previous entry (without the empty token) - // should still be in the cache and will be reliable for this example case. - if(tailIndex > 0 && editPath[tailIndex-1] == 'delete' && editPath[tailIndex] == 'substitute') { - return null; - } + // If mutations HAVE happened, we have work to do. + let state = matchState; - // Now to check everything in-between: should be exclusively 'match'es. - for(let index = 1; index < editPath.length - (ignorePenultimateMatch ? 2 : 1); index++) { - if(editPath[index] != 'match') { - return null; + let priorEdit: typeof editPath[0]; + let poppedTokenCount = 0; + for(let i = 0; i < firstMatch; i++) { + switch(editPath[i]) { + case 'delete': + if(priorEdit && priorEdit != 'delete') { + return null; + } + if(state == matchState) { + state = new TrackedContextState(state); + } + state.popHead(); + poppedTokenCount++; + break; + case 'substitute': + // There's no major need to drop parts of a token being 'slid' out of the context window. + // We'll leave it intact. + break; + default: + // No 'insert' should exist on the leading edge of context when the + // context window slides. + // + // No 'transform' edits should exist within this section, either. + return null; } } - // If we've made it here... success! We have a context match! - let state: TrackedContextState; - - if(pushedTail) { - // On suggestion acceptance, we should update the previous final token. - // We do it first so that the acceptance is replicated in the new TrackedContextState - // as well. - if(ignorePenultimateMatch) { - // For this case, we were likely called by ModelCompositor.acceptSuggestion(), which - // would have marked the accepted suggestion. - matchState.tail.replacementText = tokenizedContext[tokenizedContext.length-2]; - } - - state = new TrackedContextState(matchState); - } else { - // We're continuing a previously-cached context; create a deep-copy of it. - // We can't just re-use the old instance, unfortunately; predictions break - // with multitaps otherwise - we should avoid tracking keystrokes that were - // rewound. - // - // If there are no incoming transforms, though... yeah, re-use is safe then. - state = !!transformDistribution ? new TrackedContextState(matchState) : matchState; - } - const hasDistribution = transformDistribution && Array.isArray(transformDistribution); let primaryInput = hasDistribution ? transformDistribution[0].sample : null; if(primaryInput && primaryInput.insert == "" && primaryInput.deleteLeft == 0 && !primaryInput.deleteRight) { primaryInput = null; } - const isWhitespace = primaryInput && TransformUtils.isWhitespace(primaryInput); + // TODO: "wordbreak" the `insert` section of the transform (if it exists). + // ... wait, might have to be done at a higher level... + // ... and will probably want its own unit test ... + const isBackspace = primaryInput && TransformUtils.isBackspace(primaryInput); - const finalToken = tokenizedContext[tokenizedContext.length-1]; - /* Assumption: This is an adequate check for its two sub-branches. - * - * Basis: - * - Assumption: one keystroke may only cause a single token to rotate out of context. - * - That is, no "reasonable" keystroke would emit enough code points to 'bump' two words simultaneously. - * - ... This one may need to be loosened a bit... but it should be enough for initial correction testing as-is. - * - Assumption: one keystroke may only cause a single token to be appended to the context - * - That is, no "reasonable" keystroke would emit a Transform adding two separate word tokens - * - For languages using whitespace to word-break, said keystroke would have to include said whitespace to break the assumption. - */ + // Reset priorEdit for the end-of-context updating loop. + priorEdit = undefined; - function maintainLastToken() { - if(isWhitespace && editPath[tailIndex] == 'match') { - /* - We can land here if there are multiple whitespaces in a row. - There's already an implied whitespace to the left, so we conceptually - merge the new whitespace with that one. - */ - return; - } else if(isBackspace) { - // Consider backspace entry for this case? - state.replaceTailForBackspace(finalToken, primaryInput.id); - } else { - state.updateTail(primaryInput ? transformDistribution : null, finalToken); - } - } + // Now to update the end of the context window. + for(let i = lastMatch+1; i < editPath.length; i++) { + const incomingToken = tokenizedContext[i - poppedTokenCount] + switch(editPath[i]) { + case 'substitute': + if(i == editPath.length - 1) { + state = new TrackedContextState(state); + } - // If there is/was more than one context token available... - if(editPath.length > 1) { - // We're removing a context token, but at least one remains. - if(poppedHead) { - state.popHead(); - } + if(isBackspace) { + state.tokens[i - poppedTokenCount].updateWithBackspace(incomingToken.text, primaryInput.id); + } else { + state.updateToken(state.tokens[i - poppedTokenCount], transformDistribution, incomingToken.text); + } - // We're adding an additional context token. - if(pushedTail) { - const tokenizedTail = tokenizedContext[tokenizedContext.length - 1]; - /* - * Common-case: most transforms that trigger this case are from pure-whitespace Transforms. MOST. - * - * Less-common, but noteworthy: some wordbreaks may occur without whitespace. Example: - * `"o` => ['"', 'o']. Make sure to double-check against `tokenizedContext`! - */ - let pushedToken = new TrackedContextToken(); - pushedToken.raw = tokenizedTail; + if(state != matchState) { + if(isBackspace) { + matchState.tokens[i].updateWithBackspace(incomingToken.text, primaryInput.id); + } else { + matchState.updateToken(state.tokens[i], transformDistribution, incomingToken.text); + } + } + break; + case 'insert': + if(priorEdit && priorEdit != 'substitute' && priorEdit != 'match') { + return null; + } - if(isWhitespace || !primaryInput) { - state.pushWhitespaceToTail(transformDistribution ?? []); - // Continuing the earlier assumption, that 'pure-whitespace Transform' does not emit any initial characters - // for the new word (token), so the input keystrokes do not correspond to the new text token. - pushedToken.transformDistributions = []; - } else { - state.pushWhitespaceToTail(); - // Assumption: Since we only allow one-transform-at-a-time changes between states, we shouldn't be missing - // any metadata used to construct the new context state token. + if(state == matchState) { + state = new TrackedContextState(state); + } + + let pushedToken = new TrackedContextToken(); + pushedToken.raw = incomingToken.text; + + // TODO: May need something more complicated if the keystroke's + // transform triggers a wordbreak _within_ its boundaries (rather than + // on an edge). (Probably some way to map the tokenization to the indices + // within `insert`.) pushedToken.transformDistributions = transformDistribution ? [transformDistribution] : []; - } + pushedToken.isWhitespace = incomingToken.isWhitespace; - state.pushTail(pushedToken); - } else { - // We're editing the final context token. - // TODO: Assumption: we didn't 'miss' any inputs somehow. - // As is, may be prone to fragility should the lm-layer's tracked context 'desync' from its host's. - maintainLastToken(); - } - // There is only one word in the context. - } else { - // TODO: Assumption: we didn't 'miss' any inputs somehow. - // As is, may be prone to fragility should the lm-layer's tracked context 'desync' from its host's. - - if(editPath[tailIndex] == 'insert') { - // Construct appropriate initial token. - let token = new TrackedContextToken(); - token.raw = tokenizedContext[0]; - token.transformDistributions = [transformDistribution]; - state.pushTail(token); - } else { - // Edit the lone context token. - maintainLastToken(); + state.pushTail(pushedToken); + break; + default: + // No 'delete' should exist on the trailing edge of context when the + // context window slides. While it can happen due to keystrokes with + // `deleteLeft`, we keep a cache of recent contexts - an older one will + // likely match sufficiently. + // - may see 'delete' followed by 'substitute' in such cases. + // + // No 'transform' edits should exist within this section, either. + return null; } } + return state; } - private static modelContextState(tokenizedContext: USVString[], - transformDistribution: Distribution, - lexicalModel: LexicalModel): TrackedContextState { + private static modelContextState( + tokenizedContext: {text: USVString, isWhitespace?: boolean}[], + lexicalModel: LexicalModel + ): TrackedContextState { let baseTokens = tokenizedContext.map(function(entry) { let token = new TrackedContextToken(); - token.raw = entry; + token.raw = entry.text; + if(entry.isWhitespace) { + token.isWhitespace = true; + } + if(token.raw) { token.transformDistributions = textToCharTransforms(token.raw).map(function(transform) { return [{sample: transform, p: 1.0}]; @@ -504,7 +511,7 @@ export class ContextTracker extends CircularArray { } while(baseTokens.length > 0) { - state.pushWhitespaceToTail(); + // state.pushWhitespaceToTail(); state.pushTail(baseTokens.splice(0, 1)[0]); } @@ -527,21 +534,32 @@ export class ContextTracker extends CircularArray { * @param context * @param transformDistribution */ - analyzeState(model: LexicalModel, - context: Context, - transformDistribution?: Distribution): TrackedContextState { + analyzeState( + model: LexicalModel, + context: Context, + transformDistribution?: Distribution + ): TrackedContextState { if(!model.traverseFromRoot) { // Assumption: LexicalModel provides a valid traverseFromRoot function. (Is technically optional) // Without it, no 'corrections' may be made; the model can only be used to predict, not correct. throw "This lexical model does not provide adequate data for correction algorithms and context reuse"; } + const inputTransform = transformDistribution?.[0]; + if(inputTransform) { + context = applyTransform(inputTransform.sample, context); + } + let tokenize = determineModelTokenizer(model); let tokenizedContext = tokenize(context); if(tokenizedContext.left.length > 0) { for(let i = this.count - 1; i >= 0; i--) { const priorMatchState = this.item(i); + + // Skip intermediate multitap-produced contexts. + // When multitapping, we skip all contexts from prior taps within the same interaction, + // but not any contexts from before the multitap started. const priorTaggedContext = priorMatchState.taggedContext; if(priorTaggedContext && transformDistribution && transformDistribution.length > 0) { // Using the potential `matchState` + the incoming transform, do the results line up for @@ -584,7 +602,7 @@ export class ContextTracker extends CircularArray { // // Assumption: as a caret needs to move to context before any actual transform distributions occur, // this state is only reached on caret moves; thus, transformDistribution is actually just a single null transform. - let state = ContextTracker.modelContextState(tokenizedContext.left, transformDistribution, model); + let state = ContextTracker.modelContextState(tokenizedContext.left, model); state.taggedContext = context; this.enqueue(state); return state; diff --git a/common/web/lm-worker/src/main/model-compositor.ts b/common/web/lm-worker/src/main/model-compositor.ts index 444de3c73b..73f973b382 100644 --- a/common/web/lm-worker/src/main/model-compositor.ts +++ b/common/web/lm-worker/src/main/model-compositor.ts @@ -216,11 +216,8 @@ export class ModelCompositor { let postContextTokenization = this.tokenize(postContext); if(postContextTokenization) { // Handles display string for reversions triggered by accepting a suggestion mid-token. - if(postContextTokenization.left.length > 0) { - revertedPrefix = postContextTokenization.left[postContextTokenization.left.length-1]; - } else { - revertedPrefix = ''; - } + const preCaretToken = postContextTokenization.left[postContextTokenization.left.length - 1]; + revertedPrefix = (preCaretToken && !preCaretToken.isWhitespace) ? preCaretToken.text : ''; revertedPrefix += postContextTokenization.caretSplitsToken ? postContextTokenization.right[0] : ''; } else { revertedPrefix = this.wordbreak(postContext); diff --git a/common/web/lm-worker/src/main/model-helpers.ts b/common/web/lm-worker/src/main/model-helpers.ts index afca8ecdc1..17ad212671 100644 --- a/common/web/lm-worker/src/main/model-helpers.ts +++ b/common/web/lm-worker/src/main/model-helpers.ts @@ -62,13 +62,7 @@ export function determineModelWordbreaker(model: LexicalModel): (context: Contex export function determineModelTokenizer(model: LexicalModel) { return (context: Context) => { if(model.wordbreaker) { - const fullTokenization = models.tokenize(model.wordbreaker, context); - - return { - left: fullTokenization.left .filter((entry) => !entry.isWhitespace).map((entry) => entry.text), - right: fullTokenization.right.filter((entry) => !entry.isWhitespace).map((entry) => entry.text), - caretSplitsToken: fullTokenization.caretSplitsToken - } + return models.tokenize(model.wordbreaker, context); } else { return null; } diff --git a/common/web/lm-worker/src/main/predict-helpers.ts b/common/web/lm-worker/src/main/predict-helpers.ts index 31b12e898b..e505245edc 100644 --- a/common/web/lm-worker/src/main/predict-helpers.ts +++ b/common/web/lm-worker/src/main/predict-helpers.ts @@ -162,14 +162,19 @@ export async function correctAndEnumerate( // facilitates a more thorough correction-search pattern. // Token replacement benefits greatly from knowledge of the prior context state. - let contextState = contextTracker.analyzeState(lexicalModel, context, null); + let contextState = contextTracker.analyzeState( + lexicalModel, + context, + null + ); // Corrections and predictions are based upon the post-context state, though. - postContextState = contextTracker.analyzeState( lexicalModel, - postContext, - !TransformUtils.isEmpty(inputTransform) - ? transformDistribution - : null - ); + postContextState = contextTracker.analyzeState( + lexicalModel, + context, + !TransformUtils.isEmpty(inputTransform) + ? transformDistribution + : null + ); // TODO: Should we filter backspaces & whitespaces out of the transform distribution? // Ideally, the answer (in the future) will be no, but leaving it in right now may pose an issue. diff --git a/common/web/lm-worker/src/test/mocha/cases/edit-distance/context-tracker.js b/common/web/lm-worker/src/test/mocha/cases/edit-distance/context-tracker.js index 5c9d4812b6..1ad31b3e8f 100644 --- a/common/web/lm-worker/src/test/mocha/cases/edit-distance/context-tracker.js +++ b/common/web/lm-worker/src/test/mocha/cases/edit-distance/context-tracker.js @@ -4,6 +4,9 @@ import { ContextTracker } from '#./correction/context-tracker.js'; import ModelCompositor from '#./model-compositor.js'; import * as models from '#./models/index.js'; +import { default as defaultBreaker } from '@keymanapp/models-wordbreakers'; +import { deepCopy } from '@keymanapp/web-utils'; + import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; describe('ContextTracker', function() { @@ -16,49 +19,76 @@ describe('ContextTracker', function() { describe('attemptMatchContext', function() { it("properly matches and aligns when lead token is removed", function() { - let existingContext = ["an", "apple", "a", "day", "keeps", "the", "doctor"]; + let existingContext = models.tokenize(defaultBreaker, { + left: "an apple a day keeps the doctor" + }); let transform = { insert: '', deleteLeft: 0 } - let newContext = existingContext.slice(0); - newContext.splice(0, 1); - let rawTokens = ["apple", null, "a", null, "day", null, "keeps", null, "the", null, "doctor"]; + let newContext = deepCopy(existingContext); + newContext.left.splice(0, 1); + let rawTokens = [" ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"]; - let existingState = ContextTracker.modelContextState(existingContext); - let state = ContextTracker.attemptMatchContext(newContext, existingState, null, toWrapperDistribution(transform)); + let existingState = ContextTracker.modelContextState(existingContext.left); + let state = ContextTracker.attemptMatchContext(newContext.left, existingState, null, toWrapperDistribution(transform)); + assert.isNotNull(state); + assert.deepEqual(state.tokens.map(token => token.raw), rawTokens); + }); + + it("properly matches and aligns when lead token + following whitespace are removed", function() { + let existingContext = models.tokenize(defaultBreaker, { + left: "an apple a day keeps the doctor" + }); + let transform = { + insert: '', + deleteLeft: 0 + } + let newContext = deepCopy(existingContext); + newContext.left.splice(0, 2); + let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"]; + + let existingState = ContextTracker.modelContextState(existingContext.left); + let state = ContextTracker.attemptMatchContext(newContext.left, existingState, null, toWrapperDistribution(transform)); assert.isNotNull(state); assert.deepEqual(state.tokens.map(token => token.raw), rawTokens); }); it("properly matches and aligns when final token is edited", function() { - let existingContext = ["an", "apple", "a", "day", "keeps", "the", "docto"]; + let existingContext = models.tokenize(defaultBreaker, { + left: "an apple a day keeps the docto" + }); let transform = { insert: 'r', deleteLeft: 0 } - let newContext = existingContext.slice(0); - newContext[newContext.length - 1] = 'doctor'; - let rawTokens = ["an", null, "apple", null, "a", null, "day", null, "keeps", null, "the", null, "doctor"]; + let newContext = models.tokenize(defaultBreaker, { + left: "an apple a day keeps the doctor" + }); + let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"]; - let existingState = ContextTracker.modelContextState(existingContext); - let state = ContextTracker.attemptMatchContext(newContext, existingState, null, toWrapperDistribution(transform)); + let existingState = ContextTracker.modelContextState(existingContext.left); + let state = ContextTracker.attemptMatchContext(newContext.left, existingState, null, toWrapperDistribution(transform)); assert.isNotNull(state); assert.deepEqual(state.tokens.map(token => token.raw), rawTokens); }); + // Needs improved context-state management (due to 2x tokens) it("properly matches and aligns when a 'wordbreak' is added", function() { - let existingContext = ["an", "apple", "a", "day", "keeps", "the", "doctor"]; + let existingContext = models.tokenize(defaultBreaker, { + left: "an apple a day keeps the doctor" + }); let transform = { insert: ' ', deleteLeft: 0 } - let newContext = existingContext.slice(0); - newContext.push(''); - let rawTokens = ["an", null, "apple", null, "a", null, "day", null, "keeps", null, "the", null, "doctor", null, ""]; + let newContext = models.tokenize(defaultBreaker, { + left: "an apple a day keeps the doctor " + }); + let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""]; - let existingState = ContextTracker.modelContextState(existingContext); - let state = ContextTracker.attemptMatchContext(newContext, existingState, toWrapperDistribution(transform)); + let existingState = ContextTracker.modelContextState(existingContext.left); + let state = ContextTracker.attemptMatchContext(newContext.left, existingState, toWrapperDistribution(transform)); assert.isNotNull(state); assert.deepEqual(state.tokens.map(token => token.raw), rawTokens); @@ -68,38 +98,44 @@ describe('ContextTracker', function() { }); it("properly matches and aligns when an implied 'wordbreak' occurs (as when following \"'\")", function() { - let existingContext = ["'"]; + let existingContext = models.tokenize(defaultBreaker, { + left: "'" + }); let transform = { insert: 'a', deleteLeft: 0 } - let newContext = existingContext.slice(0); - newContext.push('a'); // The incoming transform should produce a new token WITH TEXT. - let rawTokens = ["'", null, "a"]; + let newContext = models.tokenize(defaultBreaker, { + left: "'a" + }); + let rawTokens = ["'", "a"]; - let existingState = ContextTracker.modelContextState(existingContext); - let state = ContextTracker.attemptMatchContext(newContext, existingState, toWrapperDistribution(transform)); + let existingState = ContextTracker.modelContextState(existingContext.left); + let state = ContextTracker.attemptMatchContext(newContext.left, existingState, toWrapperDistribution(transform)); assert.isNotNull(state); assert.deepEqual(state.tokens.map(token => token.raw), rawTokens); // The 'wordbreak' transform - assert.isEmpty(state.tokens[state.tokens.length - 2].transformDistributions); + assert.isNotEmpty(state.tokens[state.tokens.length - 2].transformDistributions); assert.isNotEmpty(state.tokens[state.tokens.length - 1].transformDistributions); - }); + }) + // Needs improved context-state management (due to 2x tokens) it("properly matches and aligns when lead token is removed AND a 'wordbreak' is added'", function() { - let existingContext = ["an", "apple", "a", "day", "keeps", "the", "doctor"]; + let existingContext = models.tokenize(defaultBreaker, { + left: "an apple a day keeps the doctor" + }); let transform = { insert: ' ', deleteLeft: 0 } - let newContext = existingContext.slice(0); - newContext.splice(0, 1); - newContext.push(''); - let rawTokens = ["apple", null, "a", null, "day", null, "keeps", null, "the", null, "doctor", null, ""]; + let newContext = models.tokenize(defaultBreaker, { + left: "apple a day keeps the doctor " + }); + let rawTokens = ["apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""]; - let existingState = ContextTracker.modelContextState(existingContext); - let state = ContextTracker.attemptMatchContext(newContext, existingState, toWrapperDistribution(transform)); + let existingState = ContextTracker.modelContextState(existingContext.left); + let state = ContextTracker.attemptMatchContext(newContext.left, existingState, toWrapperDistribution(transform)); assert.isNotNull(state); assert.deepEqual(state.tokens.map(token => token.raw), rawTokens); @@ -111,18 +147,28 @@ describe('ContextTracker', function() { describe('modelContextState', function() { it('models without final wordbreak', function() { - let context = ["an", "apple", "a", "day", "keeps", "the", "doctor"]; - let rawTokens = ["an", null, "apple", null, "a", null, "day", null, "keeps", null, "the", null, "doctor"]; + let tokenized = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"].map((entry) => { + return { + text: entry, + isWhitespace: entry == " " + }; + }); + let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"]; - let state = ContextTracker.modelContextState(context); + let state = ContextTracker.modelContextState(tokenized); assert.deepEqual(state.tokens.map(token => token.raw), rawTokens); }); it('models with final wordbreak', function() { - let context = ["an", "apple", "a", "day", "keeps", "the", "doctor", ""]; - let rawTokens = ["an", null, "apple", null, "a", null, "day", null, "keeps", null, "the", null, "doctor", null, ""]; + let tokenized = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""].map((entry) => { + return { + text: entry, + isWhitespace: entry == " " + }; + }); + let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""]; - let state = ContextTracker.modelContextState(context); + let state = ContextTracker.modelContextState(tokenized); assert.deepEqual(state.tokens.map(token => token.raw), rawTokens); }); }); @@ -133,6 +179,7 @@ describe('ContextTracker', function() { insertAfterWord: ' ' }; + // Needs improved context-state management (due to 2x tokens) it('tracks an accepted suggestion', function() { let baseSuggestion = { transform: { @@ -180,7 +227,7 @@ describe('ContextTracker', function() { let postContextState = compositor.contextTracker.analyzeState(model, postContext); // Penultimate token corresponds to whitespace, which does not have a 'raw' representation. - assert.isNull(postContextState.tokens[postContextState.tokens.length - 2].raw); + assert.equal(postContextState.tokens[postContextState.tokens.length - 2].raw, ' '); // Final token is empty (follows a wordbreak) assert.equal(postContextState.tail.raw, ''); From ef4089d6dcb8472917830400bf6c2272e64e0714 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 25 Jul 2024 08:10:34 +0700 Subject: [PATCH 2/4] chore(web): pulls forward some changes from child branch --- .../src/main/correction/context-tracker.ts | 66 +++++-------------- 1 file changed, 18 insertions(+), 48 deletions(-) diff --git a/common/web/lm-worker/src/main/correction/context-tracker.ts b/common/web/lm-worker/src/main/correction/context-tracker.ts index a8c42e53a1..dd618efa20 100644 --- a/common/web/lm-worker/src/main/correction/context-tracker.ts +++ b/common/web/lm-worker/src/main/correction/context-tracker.ts @@ -169,45 +169,6 @@ export class TrackedContextState { } } - // pushWhitespaceToTail(transformDistribution: Distribution = null) { - // let whitespaceToken = new TrackedContextToken(); - - // // Track the Transform that resulted in the whitespace 'token'. - // // Will be needed for phrase-level correction/prediction. - // whitespaceToken.transformDistributions = transformDistribution ? [transformDistribution] : []; - - // whitespaceToken.raw = null; - // this.tokens.push(whitespaceToken); - // } - - // /** - // * Used for 14.0's backspace workaround, which flattens all previous Distribution - // * entries because of limitations with direct use of backspace transforms. - // * @param tokenText - // * @param transformId - // */ - // replaceTailForBackspace(tokenText: USVString, transformId: number) { - // this.tokens.pop(); - - // // It's a backspace transform; time for special handling! - // // - // // For now, with 14.0, we simply compress all remaining Transforms for the token into - // // multiple single-char transforms. Probabalistically modeling BKSP is quite complex, - // // so we simplify by assuming everything remaining after a BKSP is 'true' and 'intended' text. - // // - // // Note that we cannot just use a single, monolithic transform at this point b/c - // // of our current edit-distance optimization strategy; diagonalization is currently... - // // not very compatible with that. - // let backspacedTokenContext: Distribution[] = textToCharTransforms(tokenText, transformId).map(function(transform) { - // return [{sample: transform, p: 1.0}]; - // }); - - // let compactedToken = new TrackedContextToken(); - // compactedToken.raw = tokenText; - // compactedToken.transformDistributions = backspacedTokenContext; - // this.pushTail(compactedToken); - // } - updateToken(token: TrackedContextToken, transformDistribution: Distribution, tokenText?: USVString) { // Preserve existing text if new text isn't specified. tokenText = tokenText || (tokenText === '' ? '' : token.raw); @@ -424,25 +385,34 @@ export class ContextTracker extends CircularArray { // Now to update the end of the context window. for(let i = lastMatch+1; i < editPath.length; i++) { + const isLastToken = i == editPath.length - 1; + const incomingToken = tokenizedContext[i - poppedTokenCount] switch(editPath[i]) { case 'substitute': - if(i == editPath.length - 1) { + if(isLastToken) { state = new TrackedContextState(state); } + const token = state.tokens[i - poppedTokenCount]; + const matchToken = matchState.tokens[i]; + if(isBackspace) { - state.tokens[i - poppedTokenCount].updateWithBackspace(incomingToken.text, primaryInput.id); + token.updateWithBackspace(incomingToken.text, primaryInput.id); } else { - state.updateToken(state.tokens[i - poppedTokenCount], transformDistribution, incomingToken.text); + state.updateToken(token, transformDistribution, incomingToken.text); } - if(state != matchState) { - if(isBackspace) { - matchState.tokens[i].updateWithBackspace(incomingToken.text, primaryInput.id); - } else { - matchState.updateToken(state.tokens[i], transformDistribution, incomingToken.text); - } + // For this case, we were _likely_ called by + // ModelCompositor.acceptSuggestion(), which would have marked the + // accepted suggestion. + // + // Upon inspection, this doesn't seem entirely ideal. It works for + // the common case, but not for specially crafted keystroke + // transforms. That said, it's also very low impact. Best as I can + // see, this is only really used for debugging info? + if(state != matchState && !isLastToken) { + matchToken.replacementText = incomingToken.text; } break; case 'insert': From e6cf85baf5f499d6151eddf7d0ba016ee2bed14b Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 1 Aug 2024 08:28:57 +0700 Subject: [PATCH 3/4] chore(common/models): defines Token type per review suggestion --- common/models/templates/src/index.ts | 2 +- common/models/templates/src/tokenization.ts | 15 +++++++-------- .../src/main/correction/context-tracker.ts | 4 ++-- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/common/models/templates/src/index.ts b/common/models/templates/src/index.ts index 8563004fef..0966fb4494 100644 --- a/common/models/templates/src/index.ts +++ b/common/models/templates/src/index.ts @@ -3,5 +3,5 @@ export { transformToSuggestion, defaultApplyCasing } from "./common.js"; export { default as QuoteBehavior } from "./quote-behavior.js"; -export { Tokenization, tokenize, getLastPreCaretToken, wordbreak } from "./tokenization.js"; +export { getLastPreCaretToken, Token, Tokenization, tokenize, wordbreak } from "./tokenization.js"; export { default as TrieModel, TrieModelOptions } from "./trie-model.js"; \ No newline at end of file diff --git a/common/models/templates/src/tokenization.ts b/common/models/templates/src/tokenization.ts index 0be102daf0..46290a455d 100644 --- a/common/models/templates/src/tokenization.ts +++ b/common/models/templates/src/tokenization.ts @@ -1,24 +1,23 @@ // While we _could_ define this within @keymanapp/models-wordbreakers instead, it's probably // better to leave that package as _just_ the wordbreakers. +export interface Token { + text: string, + isWhitespace?: boolean +} + export interface Tokenization { /** * An array of tokens to the left of the caret. If the caret is in the middle of a token, * only the part to the left of the caret is included. */ - left: { - text: USVString, - isWhitespace?: boolean - }[], + left: Token[], /** * An array of tokens to the right of the caret. If the caret is in the middle of a token, * only the part to the right of the caret is included. */ - right: { - text: USVString, - isWhitespace?: boolean - }[], + right: Token[], /** * A flag indicating whether or not the caret's position in the context caused a token diff --git a/common/web/lm-worker/src/main/correction/context-tracker.ts b/common/web/lm-worker/src/main/correction/context-tracker.ts index 44ae267acf..dc13ed6126 100644 --- a/common/web/lm-worker/src/main/correction/context-tracker.ts +++ b/common/web/lm-worker/src/main/correction/context-tracker.ts @@ -1,4 +1,4 @@ -import { applyTransform } from '@keymanapp/models-templates'; +import { Token, applyTransform } from '@keymanapp/models-templates'; import { ClassicalDistanceCalculation } from './classical-calculation.js'; import { SearchSpace } from './distance-modeler.js'; @@ -290,7 +290,7 @@ class CircularArray { export class ContextTracker extends CircularArray { static attemptMatchContext( - tokenizedContext: { text: USVString, isWhitespace?: boolean } [], + tokenizedContext: Token[], matchState: TrackedContextState, transformDistribution?: Distribution ): TrackedContextState { From d09d90c41a546c5ca8087ffa788d5f5827c6e6ef Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 1 Aug 2024 08:29:17 +0700 Subject: [PATCH 4/4] chore(web): minor cleanup for dropped code per review --- common/web/lm-worker/src/main/correction/context-tracker.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/common/web/lm-worker/src/main/correction/context-tracker.ts b/common/web/lm-worker/src/main/correction/context-tracker.ts index dc13ed6126..edc6cac9ba 100644 --- a/common/web/lm-worker/src/main/correction/context-tracker.ts +++ b/common/web/lm-worker/src/main/correction/context-tracker.ts @@ -481,7 +481,6 @@ export class ContextTracker extends CircularArray { } while(baseTokens.length > 0) { - // state.pushWhitespaceToTail(); state.pushTail(baseTokens.splice(0, 1)[0]); }