erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-11-05 18:44:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Merge pull request #16072 from keymanapp/change/web/remove-preservation-transform
Some checks failed
Keyman Build Summary / Summarize build status checks (push) Has been cancelled
Details

Browse source 
change(web): remove .preservationTransform, .taillessTrueKeystroke 🔪
This commit is contained in:
Joshua Horton 2026-06-30 22:41:49 +07:00 • committed by GitHub
commit 8689e207d5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 18 additions and 182 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

84
web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
View file

@ -108,26 +108,11 @@ export class ContextTokenization {
*/
readonly tokens: ContextToken[];
/**
* The portion of edits from the true input keystroke that are not part of the
* final entry in `token`. If `null`, all edits are considered part of the
* final token's contents.
*
* If the final token is new due to a newly-introduced wordboundary traversed
* by the keystroke, this will generally be set to an empty transform that
* 'finalizes' the previous tail token.
*
* (Refer to #12494 for an example case.)
*/
readonly taillessTrueKeystroke: Transform;
constructor(priorToClone: ContextTokenization);
constructor(tokens: ContextToken[]);
constructor(tokens: ContextToken[], alignment: TransitionEdge, taillessTrueKeystroke: Transform);
constructor(tokens: ContextToken[], alignment: TransitionEdge);
constructor(
param1: ContextToken[] | ContextTokenization,
tokenizationPath?: TransitionEdge,
taillessTrueKeystroke?: Transform
param1: ContextToken[] | ContextTokenization
) {
if(!(param1 instanceof ContextTokenization)) {
const tokens = param1;
@ -135,11 +120,9 @@ export class ContextTokenization {
throw new Error("ContextTokenization requires at least one existing ContextToken");
}
this.tokens = [].concat(tokens);
this.taillessTrueKeystroke = taillessTrueKeystroke;
} else {
const priorToClone = param1;
this.tokens = priorToClone.tokens.map((entry) => new ContextToken(entry));
this.taillessTrueKeystroke = priorToClone.taillessTrueKeystroke;
}
}
@ -391,7 +374,7 @@ export class ContextTokenization {
tokenization.push(token);
}
return new ContextTokenization(this.tokens.slice(0, sliceIndex).concat(tokenization), null, this.taillessTrueKeystroke);
return new ContextTokenization(this.tokens.slice(0, sliceIndex).concat(tokenization));
}
/**
@ -486,11 +469,7 @@ export class ContextTokenization {
affectedToken = null;
}
return new ContextTokenization(
this.tokens.slice(0, sliceIndex).concat(tailTokenization),
null,
determineTaillessTrueKeystroke(transitionEdge)
);
return new ContextTokenization(this.tokens.slice(0, sliceIndex).concat(tailTokenization));
}
}
@ -1175,59 +1154,4 @@ export function assembleTransforms(stackedInserts: string[], stackedDeletes: num
}
return transformMap;
}
/**
* Used to construct and represent the part of the incoming transform that does
* not land as part of the final token in the resulting context. This component
* should be preserved by any suggestions that get applied.
* @param tokenizationAnalysis
* @returns
*/
export function determineTaillessTrueKeystroke(tokenizationAnalysis: TransitionEdge) {
// undefined by default; we haven't yet determined if we're still affecting
// the same token that was the tail in the previous tokenization state.
let taillessTrueKeystroke: Transform;
// If tokens were inserted, emit an empty transform; this prevents
// suggestions from replacing the "current" token.
const bestTokenizedInput = tokenizationAnalysis.inputs[0].sample;
if(bestTokenizedInput.has(1)) {
// Sets a default transform that will be returned even if the main
// transform body lies entirely within a new token.
taillessTrueKeystroke = { insert: '', deleteLeft: 0 };
// While the .size() > 1 case could also land here, it is ALSO covered
// by the loop that follows, without fail.
}
const transformKeys = [...tokenizationAnalysis.inputs[0].sample.keys()];
transformKeys.pop();
for(let i of transformKeys) {
/*
* Thinking ahead to multitokenization:
*
* If what we have is not on the "true" tokenization, then... we need to
* do multitoken effects, right? We're basing new suggestions based on a
* state that does not currently exist! We'd need to enforce THAT state,
* *then* do the suggestion!
* - Which gets fun if we auto-apply such a case, as the new "true" tokenization
* no longer results directly from the true input.
*
* If we give tokens unique IDs on first creation, we could backtrace to
* find the most recent common ancestor.
* - simple cases (same 'token', but different input transform lengths/effects)
* will have the same prior token ID
*/
const primaryInput = tokenizationAnalysis.inputs[0].sample.get(i);
if(!taillessTrueKeystroke) {
taillessTrueKeystroke = {...primaryInput};
} else {
taillessTrueKeystroke.insert += primaryInput.insert;
taillessTrueKeystroke.deleteLeft += primaryInput.deleteLeft;
}
}
return taillessTrueKeystroke;
}

2
web/src/engine/predictive-text/worker-thread/src/main/correction/transition-helpers.ts
View file

@ -130,7 +130,7 @@ export function transitionTokenizations(
// Following call: is actually designed to build SubstitutionQuotientSpurs.
const transitionedTokenization = rootTokenization.evaluateTransition(precomp[1], trueInput.id, bestProb);
const remadeTokenization = new ContextTokenization(transitionedTokenization.tokens, subset.transitionEdges.get(rootTokenization), transitionedTokenization.taillessTrueKeystroke);
const remadeTokenization = new ContextTokenization(transitionedTokenization.tokens, subset.transitionEdges.get(rootTokenization));
// If the last token is empty and has no flag for a revertable transition,
// attempt to copy the previous token's revertable transition flag.

49
web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts
View file

@ -6,7 +6,7 @@ import { searchForProperty, WordBreakProperty } from '@keymanapp/models-wordbrea
import { TransformUtils } from './transformUtils.js';
import { detectCurrentCasing, determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js';
import { ContextTokenLike } from './correction/context-token.js';
import { ContextTokenization, mapWhitespacedTokenization } from './correction/context-tokenization.js';
import { ContextTokenization } from './correction/context-tokenization.js';
import { ContextTracker } from './correction/context-tracker.js';
import { ContextState, determineContextSlideTransform } from './correction/context-state.js';
import { ContextTransition } from './correction/context-transition.js';
@ -180,12 +180,6 @@ export interface PredictionMetadata {
* available upon initial construction of this type.
*/
matchLevel?: SuggestionSimilarity;
/**
* Text from the triggering input that should _not_ be affected by the
* prediction.
*/
preservationTransform?: Transform;
}
export interface TokenizedIntermediatePrediction {
@ -311,21 +305,9 @@ export function determineTraversallessCorrectionSequences(
suggestionParams.tokens.forEach((token) => token.correction.sample.id = transformId);
}
const tokenizationMapping = mapWhitespacedTokenization(tokenization.left.map((t) => { return {exampleInput: t.text, codepointLength: KMWString.length(t.text)} }), lexicalModel, correction.sample);
const tokenizedCorrection = tokenizationMapping.tokenizedTransform;
const tokenizedCorrectionEntries = [...tokenizedCorrection.values()];
// IF: array has multiple entries, then build the preservation-transform as below, including the deleteLeft.
// If not, don't make one!
const preservationTransform = tokenizedCorrectionEntries.slice(0, -1).reduce((accum, curr) => {
return { insert: accum.insert + curr.insert, deleteLeft: accum.deleteLeft + curr.deleteLeft };
}, { insert: '', deleteLeft: 0, id: correction.sample.id});
returnedPredictionData.push({
...suggestionParams,
applyInPost: (p) => {
p.metadata.preservationTransform = preservationTransform;
}
applyInPost: (p) => {}
})
}
@ -615,16 +597,9 @@ export function determineTokenizedCorrectionSequence(
suggestionParams.tokens.forEach((t) => t.correction.sample.id = transition.transitionId);
}
const { deleteLeft } = transitionParams;
return {
...suggestionParams,
applyInPost: (entry: TokenizedIntermediatePrediction) => {
entry.metadata.preservationTransform = tokenization.taillessTrueKeystroke;
// // Will need an extra lookup layer if the suggestion is generated from within a cluster.
// entry.baseTokenization = transition.final.tokenizationSourceMap.get(tokenization);
entry.components[0].prediction.transform.deleteLeft = deleteLeft;
}
applyInPost: (entry) => {}
};
}
@ -1322,24 +1297,6 @@ export function finalizeSuggestions(
const suggestions = deduplicatedSuggestionTuples.map((tuple) => {
const prediction = tuple.components.prediction;
// If this is a suggestion after any form of wordbreak input, make sure we preserve any components
// from prior tokens!
//
// Note: may need adjustment if/when supporting phrase-level correction.
if(tuple.metadata.preservationTransform) {
const mergedTransform = {
...models.buildMergedTransform(tuple.metadata.preservationTransform, {...prediction.transform, deleteLeft: 0}),
deleteLeft: prediction.transform.deleteLeft
};
// Temporarily and locally drops 'readonly' semantics so that we can reassign the transform.
// See https://www.typescriptlang.org/docs/handbook/release-notes/typescript-2-8.html#improved-control-over-mapped-type-modifiers
let mutableSuggestion = prediction as {-readonly [transform in keyof Suggestion]: Suggestion[transform]};
// Assignment via by-reference behavior, as suggestion is an object
mutableSuggestion.transform = mergedTransform;
}
// Is sometimes not set during unit tests.
if(prediction.transformId !== undefined) {
prediction.transform.id = prediction.transformId;

12
web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-state.tests.ts
View file

@ -247,9 +247,6 @@ describe('ContextState', () => {
let newContextMatch = baseState.analyzeTransition(existingContext, toWrapperDistribution(transform));
assert.isNotNull(newContextMatch?.final);
assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens);
// We want to preserve the added whitespace when predicting a token that follows after it.
assert.deepEqual(newContextMatch.final.displayTokenization.taillessTrueKeystroke, { insert: ' ', deleteLeft: 0 });
// The 'wordbreak' transform
let state = newContextMatch?.final;
@ -275,8 +272,6 @@ describe('ContextState', () => {
let newContextMatch = baseState.analyzeTransition(existingContext, toWrapperDistribution(transform));
assert.isNotNull(newContextMatch?.final);
assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens);
// We want to preserve the added whitespace when predicting a token that follows after it.
assert.deepEqual(newContextMatch.final.displayTokenization.taillessTrueKeystroke, { insert: ' ', deleteLeft: 0 });
// The 'wordbreak' transform
let state = newContextMatch?.final;
@ -319,7 +314,6 @@ describe('ContextState', () => {
let newContextMatch = baseState.analyzeTransition(existingContext, toWrapperDistribution(transform));
assert.isNotNull(newContextMatch?.final);
assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens);
assert.deepEqual(newContextMatch.final.displayTokenization.taillessTrueKeystroke, { insert: '', deleteLeft: 0 });
// The 'wordbreak' transform
let state = newContextMatch.final;
@ -345,8 +339,6 @@ describe('ContextState', () => {
let newContextMatch = baseState.analyzeTransition(newContext, toWrapperDistribution(transform));
assert.isNotNull(newContextMatch?.final);
assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens);
// We want to preserve the added whitespace when predicting a token that follows after it.
assert.deepEqual(newContextMatch.final.displayTokenization.taillessTrueKeystroke, { insert: ' ', deleteLeft: 0 });
// The 'wordbreak' transform
let state = newContextMatch.final;
@ -376,8 +368,6 @@ describe('ContextState', () => {
let newContextMatch = baseState.analyzeTransition(existingContext, [{sample: transform, p: 1}]);
assert.isNotNull(newContextMatch?.final);
assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens);
// We want to preserve all text preceding the new token when applying a suggestion.
assert.deepEqual(newContextMatch.final.displayTokenization.taillessTrueKeystroke, { insert: 'd ', deleteLeft: 0});
// The 'wordbreak' transform
let state = newContextMatch.final;
@ -401,8 +391,6 @@ describe('ContextState', () => {
let newContextMatch = baseState.analyzeTransition(existingContext, [{sample: transform, p: 1}]);
assert.isNotNull(newContextMatch?.final);
assert.deepEqual(newContextMatch.final.displayTokenization.tokens.map(token => token.exampleInput), rawTokens);
// We want to preserve all text preceding the new token when applying a suggestion.
assert.deepEqual(newContextMatch.final.displayTokenization.taillessTrueKeystroke, { insert: 'tor ', deleteLeft: 0 });
// The 'wordbreak' transform
let state = newContextMatch.final;

4
web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts
View file

@ -107,7 +107,7 @@ describe('ContextTokenization', function() {
const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
const tokens = rawTextTokens.map((text => toTransformToken(text)));
let tokenization = new ContextTokenization(tokens, null, null /* dummy val */);
let tokenization = new ContextTokenization(tokens);
assert.deepEqual(tokenization.tokens.map((entry) => entry.exampleInput), rawTextTokens);
assert.deepEqual(tokenization.tokens.map((entry) => entry.isWhitespace), rawTextTokens.map((entry) => entry == ' '));
@ -118,7 +118,7 @@ describe('ContextTokenization', function() {
it('clones', () => {
const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
const tokens = rawTextTokens.map((text => toTransformToken(text)));
let baseTokenization = new ContextTokenization(tokens, null, null /* dummy val */);
let baseTokenization = new ContextTokenization(tokens);
let cloned = new ContextTokenization(baseTokenization);
assert.sameOrderedMembers(

6
web/src/test/auto/headless/engine/predictive-text/worker-thread/context/transition-helpers.tests.ts
View file

@ -175,11 +175,7 @@ function generateFixtureForTokenizationOutboundTransition (
// CURRENTLY NOT DONE: adding new or replacement tokens for text to be placed after 'quotientNodeToExtend'.
const transitionedTokenization = new ContextTokenization(
srcTokenization.tokens.slice(0, srcTokenization.tokens.length - 1 + relativeTailIndex).concat(token),
tokenizationEdge,
null
);
const transitionedTokenization = new ContextTokenization(srcTokenization.tokens.slice(0, srcTokenization.tokens.length - 1 + relativeTailIndex).concat(token));
return {
/**

2
web/src/test/auto/headless/engine/predictive-text/worker-thread/correction-search/tokenization-corrector.tests.ts
View file

@ -359,7 +359,7 @@ describe('TokenizationCorrector', () => {
p: 1
}
const therefxyz = new ContextToken(new SubstitutionQuotientSpur(therefxy, [zInput], zInput));
const therefxyzTokenization = new ContextTokenization([therefxyz], null, null);
const therefxyzTokenization = new ContextTokenization([therefxyz]);
const instance = new TokenizationCorrector(
therefxyzTokenization,

1
web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-context-transition.tests.ts
View file

@ -107,7 +107,6 @@ describe('determineContextTransition', () => {
assert.equal(transition.final.context.left, targetContext.left);
assert.equal(transition.final.context.right ?? "", targetContext.right ?? "");
assert.sameDeepOrderedMembers(transition.inputDistribution, inputDistribution);
assert.isNotOk(transition.final.displayTokenization.taillessTrueKeystroke);
assert.equal(transition.transitionId, 1);
} finally {
warningEmitterSpy.restore();

28
web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-suggestion-range.tests.ts
View file

@ -47,8 +47,6 @@ function buildQuickBrownFixture() {
baseTokenization.tokens.slice(0, baseTokenCount-1).concat(
new ContextToken(new LegacyQuotientSpur(baseTokenization.tail.searchModule, plainInsertDistrib, plainInsertDistrib[0]))
),
null,
null
);
const newTokenInsertDistrib: Distribution<Transform> = [
@ -58,8 +56,6 @@ function buildQuickBrownFixture() {
baseTokenization.tokens.slice(0, baseTokenCount).concat(
new ContextToken(new LegacyQuotientSpur(new LegacyQuotientRoot(plainModel), newTokenInsertDistrib, newTokenInsertDistrib[0]))
),
null,
null
);
const charReplaceDistrib: Distribution<Transform> = [
@ -69,9 +65,7 @@ function buildQuickBrownFixture() {
const charReplaceTokenization = new ContextTokenization(
baseTokenization.tokens.slice(0, baseTokenCount - 1).concat(
new ContextToken(new LegacyQuotientSpur(baseTokenization.tail.searchModule, charReplaceDistrib, charReplaceDistrib[0]))
),
null,
null
)
);
const eraseTokenDistrib: Distribution<Transform> = [
@ -80,9 +74,7 @@ function buildQuickBrownFixture() {
const eraseTokenTokenization = new ContextTokenization(
baseTokenization.tokens.slice(0, baseTokenCount - 1).concat(
new ContextToken(new LegacyQuotientRoot(plainModel))
),
null,
null
)
);
const del5Insert5Distrib: Distribution<Transform> = [
@ -91,9 +83,7 @@ function buildQuickBrownFixture() {
const del5Insert5Tokenization = new ContextTokenization(
baseTokenization.tokens.slice(0, baseTokenCount - 3).concat(
new ContextToken(new LegacyQuotientSpur(baseTokenization.tokens[baseTokenCount-2].searchModule, del5Insert5Distrib, del5Insert5Distrib[0]))
),
null,
null
)
);
const deleteToBoundDistrib: Distribution<Transform> = [
@ -102,9 +92,7 @@ function buildQuickBrownFixture() {
const deleteToBoundTokenization = new ContextTokenization(
baseTokenization.tokens.slice(0, baseTokenCount - 3).concat(
new ContextToken(new LegacyQuotientSpur(baseTokenization.tokens[baseTokenCount-2].searchModule, deleteToBoundDistrib, deleteToBoundDistrib[0]))
),
null,
null
)
);
const deleteLeftCalc = (tokens: ContextToken[]) => tokens.reduce((accum, curr) => accum + curr.codepointLength, 0);
@ -269,9 +257,7 @@ describe('determineSuggestionRange', () => {
const tokensToAppend = rawText.map((t) => ContextToken.fromRawText(plainModel, t, false));
const foxVsAlligatorTokenization = new ContextTokenization(
originalQuickBrownTokenization.tokens.slice(0, transitionSliceIndex).concat(tokensToAppend),
null,
null
originalQuickBrownTokenization.tokens.slice(0, transitionSliceIndex).concat(tokensToAppend)
)
const analysis = determineSuggestionRange(originalQuickBrownTokenization.tokens, foxVsAlligatorTokenization.tokens, tokenEquality);
@ -295,9 +281,7 @@ describe('determineSuggestionRange', () => {
const tokensToAppend = rawText.map((t) => ContextToken.fromRawText(plainModel, t, false));
const dogsAndCatTokenization = new ContextTokenization(
originalQuickBrownTokenization.tokens.slice(0, originalTokenCount - 1).concat(tokensToAppend),
null,
null
originalQuickBrownTokenization.tokens.slice(0, originalTokenCount - 1).concat(tokensToAppend)
)
const analysis = determineSuggestionRange(originalQuickBrownTokenization.tokens, dogsAndCatTokenization.tokens, tokenEquality);

6
web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-tokenized-correction-sequence.tests.ts
View file

@ -13,7 +13,6 @@ import { assert } from 'chai';
import { LexicalModelTypes } from "@keymanapp/common-types";
import * as wordBreakers from '@keymanapp/models-wordbreakers';
import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs';
import { KMWString } from 'keyman/common/web-utils';
import {
determineTokenizedCorrectionSequence,
@ -403,10 +402,5 @@ describe('determineTokenizedCorrectionSequence', () => {
};
results.applyInPost(dummiedTuple);
assert.deepEqual(dummiedTuple.metadata.preservationTransform, {
insert: trueInput.sample.insert.substring(0, KMWString.length(trueInput.sample.insert) - 1), // remove the 'd'.
deleteLeft: trueInput.sample.deleteLeft - 1
});
});
});

6
web/src/test/auto/headless/engine/predictive-text/worker-thread/prediction-helpers/determine-traversalless-correction-sequences.tests.ts
View file

@ -11,7 +11,6 @@ import { assert } from 'chai';
import { LexicalModelTypes } from "@keymanapp/common-types";
import * as wordBreakers from '@keymanapp/models-wordbreakers';
import { KMWString } from 'keyman/common/web-utils';
import { determineTraversallessCorrectionSequences, TokenizedIntermediatePrediction, ModelCompositor, models } from "@keymanapp/lm-worker/test-index";
@ -453,10 +452,5 @@ describe('determineTraversallessCorrectionSequences', () => {
};
entry.applyInPost(dummiedTuple);
assert.deepEqual(dummiedTuple.metadata.preservationTransform, {
insert: trueInput.sample.insert.substring(0, KMWString.length(trueInput.sample.insert) - 1), // remove the 'd'.
deleteLeft: trueInput.sample.deleteLeft - 1
});
});
});