From b1a9e07ef3245265c0376b22cabcebc485dada0f Mon Sep 17 00:00:00 2001
From: Joshua Horton <joshua_horton@sil.org>
Date: Thu, 25 Sep 2025 15:18:30 -0500
Subject: [PATCH] feat(web): provide startOffset for detected splits based on
 start index within original token

---
 .../src/main/correction/context-tokenization.ts          | 9 ++++++---
 .../worker-thread/context/context-tokenization.tests.ts  | 2 +-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
index 9b20ec3e5e..c00d600746 100644
--- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
+++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
@@ -40,7 +40,7 @@ interface TokenMergeMap {
 
 interface TokenSplitMap {
   input: EditTokenMap,
-  matches: EditTokenMap[]
+  matches: (EditTokenMap & { textOffset: number })[]
 };
 
 /**
@@ -875,7 +875,8 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo
         },
         matches: [ {
           index: match,
-          text: resultTokenization[match]
+          text: resultTokenization[match],
+          textOffset: 0
         }],
       };
       let currentMerge = resultTokenization[match];
@@ -883,10 +884,12 @@ export function analyzePathMergesAndSplits(priorTokenization: string[], resultTo
       // Look-ahead 1
       let nextMerge = currentMerge + resultTokenization[match + matchOffset++];
       for(/* next line */; splitTarget.indexOf(nextMerge) == 0; nextMerge = currentMerge + preTokenization[match + matchOffset++]) {
+        const textOffset = KMWString.length(currentMerge);
         currentMerge = nextMerge;
         split.matches.push({
           index: match + matchOffset - 1,
-          text: resultTokenization[match + matchOffset-1]
+          text: resultTokenization[match + matchOffset-1],
+          textOffset
         });
         // Each time we 'pass' the condition, we've successfully processed an associated edit.
 
diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts
index 2395e48d72..35940b46b4 100644
--- a/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts
+++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts
@@ -1062,7 +1062,7 @@ describe('ContextTokenization', function() {
         merges: [],
         splits: [ {
           input: { text: 'can\'', index: 7 },
-          matches: [ { text: 'can', index: 7 }, { text: '\'', index: 8 }]
+          matches: [ { text: 'can', index: 7, textOffset: 0 }, { text: '\'', index: 8, textOffset: 3 }]
         } ],
         mergeOffset: 0,
         splitOffset: -1,