From c3e5c66a6ea8adc3d7753febb6876603c55aebbd Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 7 Oct 2025 11:19:15 -0500 Subject: [PATCH] docs(web): thoughts toward upcoming changes for preservation transforms --- .../src/main/correction/context-state.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts index 6a1f3a3558..7af06ea917 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts @@ -294,6 +294,21 @@ export class ContextState { transformKeys.pop(); for(let i of transformKeys) { + /* + * Thinking ahead to multitokenization: + * + * If what we have is not on the "true" tokenization, then... we need to + * do multitoken effects, right? We're basing new suggestions based on a + * state that does not currently exist! We'd need to enforce THAT state, + * *then* do the suggestion! + * - Which gets fun if we auto-apply such a case, as the new "true" tokenization + * no longer results directly from the true input. + * + * If we give tokens unique IDs on first creation, we could backtrace to + * find the most recent common ancestor. + * - simple cases (same 'token', but different input transform lengths/effects) + * will have the same prior token ID + */ const primaryInput = bestResultAnalysis.inputs[0].sample.get(i); if(!preservationTransform) { preservationTransform = primaryInput;