From e6cf85baf5f499d6151eddf7d0ba016ee2bed14b Mon Sep 17 00:00:00 2001
From: "Joshua A. Horton" <joshua_horton@sil.org>
Date: Thu, 1 Aug 2024 08:28:57 +0700
Subject: [PATCH] chore(common/models): defines Token type per review
 suggestion

---
 common/models/templates/src/index.ts              |  2 +-
 common/models/templates/src/tokenization.ts       | 15 +++++++--------
 .../src/main/correction/context-tracker.ts        |  4 ++--
 3 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/common/models/templates/src/index.ts b/common/models/templates/src/index.ts
index 8563004fef..0966fb4494 100644
--- a/common/models/templates/src/index.ts
+++ b/common/models/templates/src/index.ts
@@ -3,5 +3,5 @@ export {
   transformToSuggestion, defaultApplyCasing
 } from "./common.js";
 export { default as QuoteBehavior } from "./quote-behavior.js";
-export { Tokenization, tokenize, getLastPreCaretToken, wordbreak } from "./tokenization.js";
+export { getLastPreCaretToken, Token, Tokenization, tokenize, wordbreak } from "./tokenization.js";
 export { default as TrieModel, TrieModelOptions } from "./trie-model.js";
\ No newline at end of file
diff --git a/common/models/templates/src/tokenization.ts b/common/models/templates/src/tokenization.ts
index 0be102daf0..46290a455d 100644
--- a/common/models/templates/src/tokenization.ts
+++ b/common/models/templates/src/tokenization.ts
@@ -1,24 +1,23 @@
 // While we _could_ define this within @keymanapp/models-wordbreakers instead, it's probably
 // better to leave that package as _just_ the wordbreakers.
 
+export interface Token {
+  text: string,
+  isWhitespace?: boolean
+}
+
 export interface Tokenization {
   /**
    * An array of tokens to the left of the caret.  If the caret is in the middle of a token,
    * only the part to the left of the caret is included.
    */
-  left: {
-    text: USVString,
-    isWhitespace?: boolean
-  }[],
+  left: Token[],
 
   /**
    * An array of tokens to the right of the caret.  If the caret is in the middle of a token,
    * only the part to the right of the caret is included.
    */
-  right: {
-    text: USVString,
-    isWhitespace?: boolean
-  }[],
+  right: Token[],
 
   /**
    * A flag indicating whether or not the caret's position in the context caused a token
diff --git a/common/web/lm-worker/src/main/correction/context-tracker.ts b/common/web/lm-worker/src/main/correction/context-tracker.ts
index 44ae267acf..dc13ed6126 100644
--- a/common/web/lm-worker/src/main/correction/context-tracker.ts
+++ b/common/web/lm-worker/src/main/correction/context-tracker.ts
@@ -1,4 +1,4 @@
-import { applyTransform } from '@keymanapp/models-templates';
+import { Token, applyTransform } from '@keymanapp/models-templates';
 
 import { ClassicalDistanceCalculation } from './classical-calculation.js';
 import { SearchSpace } from './distance-modeler.js';
@@ -290,7 +290,7 @@ class CircularArray<Item> {
 
 export class ContextTracker extends CircularArray<TrackedContextState> {
   static attemptMatchContext(
-    tokenizedContext: { text: USVString, isWhitespace?: boolean } [],
+    tokenizedContext: Token[],
     matchState: TrackedContextState,
     transformDistribution?: Distribution<Transform>
   ): TrackedContextState {