diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts index 942773fa57..4d4247a086 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-token.ts @@ -10,7 +10,7 @@ import { LexicalModelTypes } from '@keymanapp/common-types'; import { SearchQuotientNode, PathInputProperties } from "./search-quotient-node.js"; -import { TokenSplitMap } from "./context-tokenization.js"; +import { TokenSplitMapping } from "./context-tokenization.js"; import { LegacyQuotientSpur } from "./legacy-quotient-spur.js"; import { LegacyQuotientRoot } from "./legacy-quotient-root.js"; import { generateSubsetId } from './tokenization-subsets.js'; @@ -199,7 +199,7 @@ export class ContextToken { * @param lexicalModel * @returns */ - split(split: TokenSplitMap): ContextToken[] { + split(split: TokenSplitMapping): ContextToken[] { // Split from tail to head - leave as much 'head' intact as possible at each // step, rather than needing to reconstruct the tail multiple times. const splitSpecs = split.matches.slice(); diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index dcd7b0a295..9e130f6caa 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -31,7 +31,7 @@ const MIN_CHARS_TO_RECONSIDER_FOR_TOKENIZATION = 8; * This type is used to indicate properties of tokens affected by merges and * splits during a context transition. */ -interface EditTokenMap { +interface EditTokenMappingHalf { /** * The index of the affected token. */ @@ -42,10 +42,22 @@ interface EditTokenMap { text: string }; +/** + * This type is used to indicate properties of tokens affected by splits during + * a context transition. + */ +interface SplitTokenMappingHalf extends EditTokenMappingHalf { + /** + * The codepoint index within the original token at which the split-off token + * begins. + */ + textOffset: number +}; + /** * This type represents mappings for tokens affected by merge edit operations. */ -interface TokenMergeMap { +interface TokenMergeMapping { /** * Entries here represent source-context tokens that are combined as part of * the merge edit operation. @@ -53,7 +65,7 @@ interface TokenMergeMap { * Entries will appear in the same ordering as the codepoints they represent * appear in the underlying context. */ - inputs: EditTokenMap[], + inputs: EditTokenMappingHalf[], /** * This entry represents the post-transition context token produced from the @@ -62,14 +74,14 @@ interface TokenMergeMap { * Note that it is possible for extra codepoints to exist here that were not * represented in the original source-context tokens. */ - match: EditTokenMap + match: EditTokenMappingHalf }; /** * This type represents mappings for tokens affected by split edit operations. */ -export interface TokenSplitMap { +export interface TokenSplitMapping { /** * This entry represents the source-context token that was split as part of * the split edit operation. @@ -77,7 +89,7 @@ export interface TokenSplitMap { * Note that it is possible for codepoints to exist here but not be * represented in the post-transition context tokens resulting from the split. */ - input: EditTokenMap, + input: EditTokenMappingHalf, /** * Entries here represent post-transition tokens that represent pieces of the @@ -86,7 +98,7 @@ export interface TokenSplitMap { * Entries will appear in the same ordering as the codepoints they represent * appear in the underlying context. */ - matches: (EditTokenMap & { textOffset: number })[] + matches: SplitTokenMappingHalf[] }; /** @@ -99,11 +111,11 @@ export interface TransitionEdgeAlignment { /** * Denotes any token merge edits needed after applying the Transform. */ - merges: TokenMergeMap[]; + merges: TokenMergeMapping[]; /** * Denotes any token split edits needed after applying the Transform. */ - splits: TokenSplitMap[]; + splits: TokenSplitMapping[]; /** * Denotes any further token edits needed that cannot be attributed to * 'merge's, 'split's, or edits from the input `Transform`. @@ -1036,13 +1048,13 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo * generality, if two separate groups of tokens are merged, two groups will be * defined - one for each token resulting from a merge. */ - merges: TokenMergeMap[], + merges: TokenMergeMapping[], /** * Indicates groupings of directly related splits. Without loss of * generality, if two separate tokens are split, two groups will be defined - * one for each source token split. */ - splits: TokenSplitMap[] + splits: TokenSplitMapping[] } { // We've found the root token to which changes may apply. // We've found the last post-application token to which transform changes contributed. @@ -1080,8 +1092,8 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo const mappedPath: EditTuple[] = []; let mergeOffset = 0; let splitOffset = 0; - const merges: TokenMergeMap[] = []; - const splits: TokenSplitMap[] = []; + const merges: TokenMergeMapping[] = []; + const splits: TokenSplitMapping[] = []; while(queueIndex < editPath.length) { const edit = editPath[queueIndex]; const { input, match } = edit; @@ -1091,7 +1103,7 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo let matchOffset: number = 0; if(op == 'merge') { const mergeTarget = resultTokenization[match]; - const merge: TokenMergeMap = { + const merge: TokenMergeMapping = { match: { index: match, text: mergeTarget @@ -1122,7 +1134,7 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo merges.push(merge); } else if(op == 'split') { const splitTarget = preTokenization[input]; - const split: TokenSplitMap = { + const split: TokenSplitMapping = { input: { index: input, text: splitTarget