erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-11-05 18:44:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Merge branch 'change/web/multi-token-prediction-intermediates' into change/web/adjust-tokenization-corrector-spec

Browse source
This commit is contained in:
Joshua Horton 2026-06-22 10:48:52 -05:00
commit cda2d40ac1
2 changed files with 17 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts
View file

@ -1,7 +1,7 @@
import * as models from '@keymanapp/models-templates';
import { LexicalModelTypes } from '@keymanapp/common-types';
import { applySuggestionCasing, compositeIntermediatePredictions, correctAndEnumerate, createDefaultKeep, dedupeSuggestions, finalizeSuggestions, predictionAutoSelect, processSimilarity, toAnnotatedSuggestion, tupleDisplayOrderSort } from './predict-helpers.js';
import { applySuggestionCasing, composeIntermediatePredictions, correctAndEnumerate, createDefaultKeep, dedupeSuggestions, finalizeSuggestions, predictionAutoSelect, processSimilarity, toAnnotatedSuggestion, tupleDisplayOrderSort } from './predict-helpers.js';
import { determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js';
import { ContextTracker } from './correction/context-tracker.js';
@ -151,7 +151,7 @@ export class ModelCompositor {
// We want to dedupe before trimming the list so that we can present a full set
// of viable distinct suggestions if available.
const deduplicatedSuggestionTuples = dedupeSuggestions(this.lexicalModel, compositeIntermediatePredictions(rawPredictions), context);
const deduplicatedSuggestionTuples = dedupeSuggestions(this.lexicalModel, composeIntermediatePredictions(rawPredictions), context);
// Needs "casing" to be applied first.
const postContext = postContextState?.context ?? models.applyTransform(inputTransform, context);

34
web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts
View file

@ -17,7 +17,6 @@ import { TokenResult } from './correction/tokenization-corrector.js';
import { TokenizationCorrector } from './correction/tokenization-corrector.js';
import { TokenizationResultMapping } from './correction/tokenization-result-mapping.js';
import CasingForm = LexicalModelTypes.CasingForm;
import Context = LexicalModelTypes.Context;
import Distribution = LexicalModelTypes.Distribution;
import Keep = LexicalModelTypes.Keep;
@ -795,12 +794,7 @@ export function shouldStopSearchingEarly(
* "rank" and select the best predictions once the search is complete. This is
* performed at later stages.
* @param lexicalModel
* @param corrections Each `correction` should insert a full token's text to be
* appended to the context resulting from all preceding corrections.
* @param rootContext This context should represent all portions of the
* post-context not represented by the entries of the `corrections` array.
* @param transitionId Indicates the unique ID of the transition that triggered
* prediction generation.
* @param predictionPrep
* @returns
*/
export function predictFromCorrectionSequence(
@ -930,33 +924,28 @@ export function predictFromCorrectionSequence(
/**
* Applies the specified casing-form to generated suggestions, leveraging the model's
* defined casing behaviors to do so.
* @param suggestion
* @param baseWord
* @param predictionToken A tuple representing a single context token's prediction and base correction
* @param lexicalModel
* @param casingForm
*/
export function applySuggestionCasing(predictionToken: TokenizedPredictionData, lexicalModel: LexicalModel) {
const suggestion = predictionToken.prediction;
// Step 0: our pattern for generating predictions and corrections already
// enforces them to encompass the whole word.
// Step 1: detect the original token's casing
let casingForm: CasingForm;
// enforces that they encompass the whole word.
const suggestion = predictionToken.prediction;
// If we are using the context-tracking engine (when traversals are enabled),
// we just leverage the context token's exampleInput to determine casing.
//
// If it's not available, the correction entry reflects a word-broken piece of
// the original context, with its original casing - so we use that instead.
let casingRoot = predictionToken.casingRoot ? predictionToken.casingRoot : predictionToken.correction;
const casingRoot = predictionToken.casingRoot ? predictionToken.casingRoot : predictionToken.correction;
if(!casingRoot) {
// There's no text in place to verify casing expectations; just leave it
// unchanged.
return;
}
casingForm = detectCurrentCasing(lexicalModel, {
// Step 1: detect the original token's casing
const casingForm = detectCurrentCasing(lexicalModel, {
left: casingRoot,
startOfBuffer: true,
endOfBuffer: true
@ -969,7 +958,14 @@ export function applySuggestionCasing(predictionToken: TokenizedPredictionData,
}
}
export function compositeIntermediatePredictions(predictions: IntermediateTokenizedPrediction[]): IntermediateCompositedPrediction[] {
/**
* Composes a set of `IntermediateTokenizedPrediction`s, merging the tokenized
* data into corresponding `IntermediateCompositedPrediction`s representing the
* full range of affected context.
* @param predictions
* @returns
*/
export function composeIntermediatePredictions(predictions: IntermediateTokenizedPrediction[]): IntermediateCompositedPrediction[] {
return predictions.map((predictionData) => {
const components = predictionData.components;