mirror of
https://github.com/keymanapp/keyman.git
synced 2026-10-15 10:27:31 +00:00
788 lines
35 KiB
TypeScript
788 lines
35 KiB
TypeScript
/// <reference path="correction/context-tracker.ts" />
|
|
|
|
class ModelCompositor {
|
|
private lexicalModel: LexicalModel;
|
|
private contextTracker?: correction.ContextTracker;
|
|
private static readonly MAX_SUGGESTIONS = 12;
|
|
readonly punctuation: LexicalModelPunctuation;
|
|
|
|
/**
|
|
* Controls the strength of anti-corrective measures for single-character scenarios.
|
|
* The base key probability will be raised to this power for this specific case.
|
|
*
|
|
* Current selection's motivation: (0.5 / 0.4) ^ 16 ~= 35.5.
|
|
* - if the most likely has p=0.5 and second-most has p=0.4 - a highly-inaccurate key
|
|
* stroke - the net effect will apply a factor of 35.5 to the lexical probability of
|
|
* the best key's prediction roots, favoring it in this manner.
|
|
* - less extreme edge cases will have a significantly stronger factor, acting as a
|
|
* "soft threshold".
|
|
* - truly ambiguous, "coin flip" cases will have a lower factor and thus favor the
|
|
* more likely words from the pair.
|
|
* - Our OSK key-element borders aren't visible to the user, so the 'spot' where
|
|
* behavior changes might feel arbitrary to users if we used a hard threshold instead.
|
|
*/
|
|
private static readonly SINGLE_CHAR_KEY_PROB_EXPONENT = 16;
|
|
|
|
private SUGGESTION_ID_SEED = 0;
|
|
|
|
private testMode: boolean = false
|
|
|
|
constructor(lexicalModel: LexicalModel, testMode?: boolean) {
|
|
this.lexicalModel = lexicalModel;
|
|
if(lexicalModel.traverseFromRoot) {
|
|
this.contextTracker = new correction.ContextTracker();
|
|
}
|
|
this.punctuation = ModelCompositor.determinePunctuationFromModel(lexicalModel);
|
|
this.testMode = !!testMode;
|
|
}
|
|
|
|
private predictFromCorrections(corrections: ProbabilityMass<Transform>[], context: Context): Distribution<Suggestion> {
|
|
let returnedPredictions: Distribution<Suggestion> = [];
|
|
|
|
for(let correction of corrections) {
|
|
let predictions = this.lexicalModel.predict(correction.sample, context);
|
|
|
|
let predictionSet = predictions.map(function(pair: ProbabilityMass<Suggestion>) {
|
|
let transform = correction.sample;
|
|
let inputProb = correction.p;
|
|
// Let's not rely on the model to copy transform IDs.
|
|
// Only bother is there IS an ID to copy.
|
|
if(transform.id !== undefined) {
|
|
pair.sample.transformId = transform.id;
|
|
}
|
|
|
|
let prediction = {sample: pair.sample, p: pair.p * inputProb};
|
|
return prediction;
|
|
}, this);
|
|
|
|
returnedPredictions = returnedPredictions.concat(predictionSet);
|
|
}
|
|
|
|
return returnedPredictions;
|
|
}
|
|
|
|
predict(transformDistribution: Transform | Distribution<Transform>, context: Context): Suggestion[] {
|
|
let suggestionDistribution: Distribution<Suggestion> = [];
|
|
let lexicalModel = this.lexicalModel;
|
|
let punctuation = this.punctuation;
|
|
|
|
if(!(transformDistribution instanceof Array)) {
|
|
transformDistribution = [ {sample: transformDistribution, p: 1.0} ];
|
|
} else if(transformDistribution.length == 0) {
|
|
/*
|
|
Robust stop-gap: if our other filters somehow fail, this fixes the
|
|
zero-length array by making it match the form of the array that
|
|
would result if it were instead called with the other legal
|
|
parameter type - a single Transform.
|
|
|
|
Unfortunately, the method will lack all data about even
|
|
the original keystroke that resulted in the call... but this way,
|
|
we can at least get some predictions rather than shortcutting
|
|
and producing none whatsoever.
|
|
*/
|
|
transformDistribution.push({
|
|
sample: {
|
|
insert: '',
|
|
deleteLeft: 0
|
|
},
|
|
p: 1.0
|
|
})
|
|
}
|
|
|
|
let inputTransform = transformDistribution.sort(function(a, b) {
|
|
return b.p - a.p;
|
|
})[0].sample;
|
|
|
|
// Only allow new-word suggestions if space was the most likely keypress.
|
|
let allowSpace = TransformUtils.isWhitespace(inputTransform);
|
|
let allowBksp = TransformUtils.isBackspace(inputTransform);
|
|
|
|
let postContext = models.applyTransform(inputTransform, context);
|
|
let keepOptionText = this.wordbreak(postContext);
|
|
let keepOption: Outcome<Keep> = null;
|
|
|
|
let rawPredictions: Distribution<Suggestion> = [];
|
|
|
|
// Used to restore whitespaces if operations would remove them.
|
|
let prefixTransform: Transform;
|
|
let postContextState: correction.TrackedContextState = null;
|
|
|
|
// Section 1: determining 'prediction roots'.
|
|
if(!this.contextTracker) {
|
|
let predictionRoots: ProbabilityMass<Transform>[];
|
|
|
|
// Generates raw prediction distributions for each valid input. Can only 'correct'
|
|
// against the final input.
|
|
//
|
|
// This is the old, 12.0-13.0 'correction' style.
|
|
if(allowSpace) {
|
|
// Detect start of new word; prevent whitespace loss here.
|
|
predictionRoots = [{sample: inputTransform, p: 1.0}];
|
|
prefixTransform = inputTransform;
|
|
} else {
|
|
predictionRoots = transformDistribution.map((alt) => {
|
|
let transform = alt.sample;
|
|
|
|
// Filter out special keys unless they're expected.
|
|
if(TransformUtils.isWhitespace(transform) && !allowSpace) {
|
|
return null;
|
|
} else if(TransformUtils.isBackspace(transform) && !allowBksp) {
|
|
return null;
|
|
}
|
|
|
|
return alt;
|
|
});
|
|
}
|
|
|
|
// Remove `null` entries.
|
|
predictionRoots = predictionRoots.filter(tuple => !!tuple);
|
|
|
|
// Running in bulk over all suggestions, duplicate entries may be possible.
|
|
rawPredictions = this.predictFromCorrections(predictionRoots, context);
|
|
} else {
|
|
// Token replacement benefits greatly from knowledge of the prior context state.
|
|
let contextState = this.contextTracker.analyzeState(this.lexicalModel, context, null);
|
|
// Corrections and predictions are based upon the post-context state, though.
|
|
postContextState = this.contextTracker.analyzeState(this.lexicalModel,
|
|
postContext,
|
|
!TransformUtils.isEmpty(inputTransform) ?
|
|
transformDistribution:
|
|
null
|
|
);
|
|
|
|
// TODO: Should we filter backspaces & whitespaces out of the transform distribution?
|
|
// Ideally, the answer (in the future) will be no, but leaving it in right now may pose an issue.
|
|
|
|
// Rather than go "full hog" and make a priority queue out of the eventual, future competing search spaces...
|
|
// let's just note that right now, there will only ever be one.
|
|
//
|
|
// The 'eventual' logic will be significantly more complex, though still manageable.
|
|
let searchSpace = postContextState.searchSpace[0];
|
|
|
|
// No matter the prediction, once we know the root of the prediction, we'll always 'replace' the
|
|
// same amount of text. We can handle this before the big 'prediction root' loop.
|
|
let deleteLeft = 0;
|
|
|
|
// The amount of text to 'replace' depends upon whatever sort of context change occurs
|
|
// from the received input.
|
|
const postContextTokens = postContextState.tokens;
|
|
let postContextLength = postContextTokens.length;
|
|
let contextLengthDelta = postContextTokens.length - contextState.tokens.length;
|
|
// If the context now has more tokens, the token we'll be 'predicting' didn't originally exist.
|
|
if(postContextLength == 0 || contextLengthDelta > 0) {
|
|
// As the word/token being corrected/predicted didn't originally exist, there's no
|
|
// part of it to 'replace'.
|
|
deleteLeft = 0;
|
|
|
|
// If the new token is due to whitespace or due to a different input type that would
|
|
// likely imply a tokenization boundary...
|
|
if(TransformUtils.isWhitespace(inputTransform)) {
|
|
/* TODO: consider/implement: the second half of the comment above.
|
|
* For example: on input of a `'`, predict new words instead of replacing the `'`.
|
|
* (since after a letter, the `'` will be ignored, anyway)
|
|
*
|
|
* Idea: if the model's most likely prediction (with no root) would make a new
|
|
* token if appended to the current token, that's probably a good case.
|
|
* Keeps the check simple & quick.
|
|
*
|
|
* Might need a mixed mode, though: ';' is close enough that `l` is a reasonable
|
|
* fat-finger guess. So yeah, we're not addressing this idea right now.
|
|
* - so... consider multiple context behavior angles when building prediction roots?
|
|
*
|
|
* May need something similar to help handle contractions during their construction,
|
|
* but that'd be within `ContextTracker`.
|
|
* can' => [`can`, `'`]
|
|
* can't => [`can't`] (WB6, 7 of https://unicode.org/reports/tr29/#Word_Boundary_Rules)
|
|
*
|
|
* (Would also helps WB7b+c for Hebrew text)
|
|
*/
|
|
|
|
// Infer 'new word' mode, even if we received new text when reaching
|
|
// this position. That new text didn't exist before, so still - nothing
|
|
// to 'replace'.
|
|
prefixTransform = inputTransform;
|
|
context = postContext; // As far as predictions are concerned, the post-context state
|
|
// should not be replaced. Predictions are to be rooted on
|
|
// text "up for correction" - so we want a null root for this
|
|
// branch.
|
|
contextState = postContextState;
|
|
}
|
|
// If the tokenized context length is shorter... sounds like a backspace (or similar).
|
|
} else if (contextLengthDelta < 0) {
|
|
/* Ooh, we've dropped context here. Almost certainly from a backspace.
|
|
* Even if we drop multiple tokens... well, we know exactly how many chars
|
|
* were actually deleted - `inputTransform.deleteLeft`.
|
|
* Since we replace a word being corrected/predicted, we take length of the remaining
|
|
* context's tail token in addition to however far was deleted to reach that state.
|
|
*/
|
|
deleteLeft = this.wordbreak(postContext).kmwLength() + inputTransform.deleteLeft;
|
|
} else {
|
|
// Suggestions are applied to the pre-input context, so get the token's original length.
|
|
// We're on the same token, so just delete its text for the replacement op.
|
|
deleteLeft = this.wordbreak(context).kmwLength();
|
|
}
|
|
|
|
// Is the token under construction newly-constructed / is there no pre-existing root?
|
|
// If so, we want to strongly avoid overcorrection, even for 'nearby' keys.
|
|
// (Strong lexical frequency differences can easily cause overcorrection when only
|
|
// one key's available.)
|
|
//
|
|
// NOTE: we only want this applied word-initially, when any corrections 'correct'
|
|
// 100% of the word. Things are generally fine once it's not "all or nothing."
|
|
let tailToken = postContextTokens[postContextTokens.length - 1];
|
|
const isTokenStart = tailToken.transformDistributions.length <= 1;
|
|
|
|
// TODO: whitespace, backspace filtering. Do it here.
|
|
// Whitespace is probably fine, actually. Less sure about backspace.
|
|
|
|
let bestCorrectionCost: number;
|
|
const SEARCH_TIMEOUT = this.testMode ? 0 : correction.SearchSpace.DEFAULT_ALLOTTED_CORRECTION_TIME_INTERVAL;
|
|
for(let matches of searchSpace.getBestMatches(SEARCH_TIMEOUT)) {
|
|
// Corrections obtained: now to predict from them!
|
|
let predictionRoots = matches.map(function(match) {
|
|
let correction = match.matchString;
|
|
|
|
// Worth considering: extend Traversal to allow direct prediction lookups?
|
|
// let traversal = match.finalTraversal;
|
|
|
|
// Find a proper Transform ID to map the correction to.
|
|
// Without it, we can't apply the suggestion.
|
|
let finalInput: Transform;
|
|
if(match.inputSequence.length > 0) {
|
|
finalInput = match.inputSequence[match.inputSequence.length - 1].sample;
|
|
} else {
|
|
finalInput = inputTransform; // A fallback measure. Greatly matters for empty contexts.
|
|
}
|
|
|
|
// Replace the existing context with the correction.
|
|
let correctionTransform: Transform = {
|
|
insert: correction, // insert correction string
|
|
deleteLeft: deleteLeft,
|
|
id: inputTransform.id // The correction should always be based on the most recent external transform/transcription ID.
|
|
}
|
|
|
|
let rootCost = match.totalCost;
|
|
|
|
/* If we're dealing with the FIRST keystroke of a new sequence, we'll **dramatically** boost
|
|
* the exponent to ensure only VERY nearby corrections have a chance of winning, and only if
|
|
* there are significantly more likely words. We only need this to allow very minor fat-finger
|
|
* adjustments for 100% keystroke-sequence corrections in order to prevent finickiness on
|
|
* key borders.
|
|
*
|
|
* Technically, the probabilities this produces won't be normalized as-is... but there's no
|
|
* true NEED to do so for it, even if it'd be 'nice to have'. Consistently tracking when
|
|
* to apply it could become tricky, so it's simpler to leave out.
|
|
*
|
|
* Worst-case, it's possible to temporarily add normalization if a code deep-dive
|
|
* is needed in the future.
|
|
*/
|
|
if(isTokenStart) {
|
|
/* Suppose a key distribution: most likely with p=0.5, second-most with 0.4 - a pretty
|
|
* ambiguous case that would only arise very near the center of the boundary between two keys.
|
|
* Raising (0.5/0.4)^16 ~= 35.53. (At time of writing, SINGLE_CHAR_KEY_PROB_EXPONENT = 16.)
|
|
* That seems 'within reason' for correction very near boundaries.
|
|
*
|
|
* So, with the second-most-likely key being that close in probability, its best suggestion
|
|
* must be ~ 35.5x more likely than that of the truly-most-likely key to "win". So, it's not
|
|
* a HARD cutoff, but more of a 'soft' one. Keeping the principles in mind documented above,
|
|
* it's possible to tweak this to a more harsh or lenient setting if desired, rather than
|
|
* being totally "all or nothing" on which key is taken for highly-ambiguous keypresses.
|
|
*/
|
|
rootCost *= ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT; // note the `Math.exp` below.
|
|
}
|
|
|
|
return {
|
|
sample: correctionTransform,
|
|
p: Math.exp(-rootCost)
|
|
};
|
|
}, this);
|
|
|
|
// Running in bulk over all suggestions, duplicate entries may be possible.
|
|
let predictions = this.predictFromCorrections(predictionRoots, context);
|
|
|
|
// Only set 'best correction' cost when a correction ACTUALLY YIELDS predictions.
|
|
if(predictions.length > 0 && bestCorrectionCost === undefined) {
|
|
bestCorrectionCost = -Math.log(predictionRoots[0].p);
|
|
}
|
|
|
|
rawPredictions = rawPredictions.concat(predictions);
|
|
// TODO: We don't currently de-duplicate predictions at this point quite yet, so
|
|
// it's technically possible that we return too few.
|
|
|
|
let correctionCost = matches[0].totalCost;
|
|
// Searching a bit longer is permitted when no predictions have been found.
|
|
if(correctionCost >= bestCorrectionCost + 8) {
|
|
break;
|
|
// If enough have been found, we're safe to terminate earlier.
|
|
} else if(rawPredictions.length >= ModelCompositor.MAX_SUGGESTIONS) {
|
|
if(correctionCost >= bestCorrectionCost + 4) { // e^-4 = 0.0183156388. Allows "80%" of an extra edit.
|
|
// Very useful for stopping 'sooner' when words reach a sufficient length.
|
|
break;
|
|
} else {
|
|
// Sort the prediction list; we need them in descending order for the next check.
|
|
rawPredictions.sort(function(a, b) {
|
|
return b.p - a.p;
|
|
});
|
|
|
|
// If the best suggestion from the search's current tier fails to beat the worst
|
|
// pending suggestion from previous tiers, assume all further corrections will
|
|
// similarly fail to win; terminate the search-loop.
|
|
if(rawPredictions[ModelCompositor.MAX_SUGGESTIONS-1].p > Math.exp(-correctionCost)) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Section 2 - post-analysis for our generated predictions, managing 'keep'.
|
|
// Assumption: Duplicated 'displayAs' properties indicate duplicated Suggestions.
|
|
// When true, we can use an 'associative array' to de-duplicate everything.
|
|
let suggestionDistribMap: {[key: string]: ProbabilityMass<Suggestion>} = {};
|
|
let currentCasing: CasingForm = null;
|
|
if(lexicalModel.languageUsesCasing) {
|
|
currentCasing = this.detectCurrentCasing(postContext);
|
|
}
|
|
|
|
let baseWord = this.wordbreak(context);
|
|
|
|
// Deduplicator + annotator of 'keep' suggestions.
|
|
for(let prediction of rawPredictions) {
|
|
// Combine duplicate samples.
|
|
let displayText = prediction.sample.displayAs;
|
|
let preserveAsKeep = displayText == keepOptionText;
|
|
|
|
// De-duplication should be case-insensitive, but NOT
|
|
// diacritic-insensitive.
|
|
if(this.lexicalModel.languageUsesCasing) {
|
|
preserveAsKeep = preserveAsKeep || displayText == this.lexicalModel.applyCasing('lower', keepOptionText);
|
|
}
|
|
|
|
if(preserveAsKeep) {
|
|
// Preserve the original, pre-keyed version of the text.
|
|
if(!keepOption) {
|
|
let baseTransform = prediction.sample.transform;
|
|
|
|
let keepTransform = {
|
|
insert: keepOptionText,
|
|
deleteLeft: baseTransform.deleteLeft,
|
|
deleteRight: baseTransform.deleteRight,
|
|
id: baseTransform.id
|
|
}
|
|
|
|
let intermediateKeep = models.transformToSuggestion(keepTransform, prediction.p);
|
|
keepOption = this.toAnnotatedSuggestion(intermediateKeep, 'keep', models.QuoteBehavior.noQuotes);
|
|
keepOption.matchesModel = true;
|
|
|
|
// Since we replaced the original Suggestion with a keep-annotated one,
|
|
// we must manually preserve the transform ID.
|
|
keepOption.transformId = prediction.sample.transformId;
|
|
} else if(keepOption.p && prediction.p) {
|
|
keepOption.p += prediction.p;
|
|
}
|
|
} else {
|
|
// Apply capitalization rules now; facilitates de-duplication of suggestions
|
|
// that may be caused as a result.
|
|
//
|
|
// Example: "apple" and "Apple" are separate when 'lower', but identical for 'initial' and 'upper'.
|
|
if(currentCasing && currentCasing != 'lower') {
|
|
this.applySuggestionCasing(prediction.sample, baseWord, currentCasing);
|
|
// update the mapping string, too.
|
|
displayText = prediction.sample.displayAs;
|
|
}
|
|
|
|
let existingSuggestion = suggestionDistribMap[displayText];
|
|
if(existingSuggestion) {
|
|
existingSuggestion.p += prediction.p;
|
|
} else {
|
|
suggestionDistribMap[displayText] = prediction;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Generate a default 'keep' option if one was not otherwise produced.
|
|
if(!keepOption && keepOptionText != '') {
|
|
// IMPORTANT: duplicate the original transform. Causes nasty side-effects
|
|
// for context-tracking otherwise!
|
|
let keepTransform: Transform = { ...inputTransform };
|
|
|
|
// 1 is a filler value; goes unused b/c is for a 'keep'.
|
|
let keepSuggestion = models.transformToSuggestion(keepTransform, 1);
|
|
// This is the one case where the transform doesn't insert the full word; we need to override the displayAs param.
|
|
keepSuggestion.displayAs = keepOptionText;
|
|
|
|
keepOption = this.toAnnotatedSuggestion(keepSuggestion, 'keep');
|
|
keepOption.matchesModel = false;
|
|
}
|
|
|
|
// Section 3: Finalize suggestions, truncate list to the N (MAX_SUGGESTIONS) most optimal, return.
|
|
|
|
// Now that we've calculated a unique set of probability masses, time to make them into a proper
|
|
// distribution and prep for return.
|
|
for(let key in suggestionDistribMap) {
|
|
let pair = suggestionDistribMap[key];
|
|
suggestionDistribution.push(pair);
|
|
}
|
|
|
|
suggestionDistribution = suggestionDistribution.sort(function(a, b) {
|
|
return b.p - a.p; // Use descending order - we want the largest probabilty suggestions first!
|
|
});
|
|
|
|
let suggestions = suggestionDistribution.splice(0, ModelCompositor.MAX_SUGGESTIONS).map(function(value) {
|
|
if(value.sample['p']) {
|
|
// For analysis / debugging
|
|
value.sample['lexical-p'] = value.sample['p'];
|
|
value.sample['correction-p'] = value.p / value.sample['p'];
|
|
// Use of the Trie model always exposed the lexical model's probability for a word to KMW.
|
|
// It's useful for debugging right now, so may as well repurpose it as the posterior.
|
|
//
|
|
// We still condition on 'p' existing so that test cases aren't broken.
|
|
value.sample['p'] = value.p;
|
|
}
|
|
//
|
|
return value.sample;
|
|
});
|
|
|
|
if(keepOption) {
|
|
suggestions = [ keepOption as Suggestion ].concat(suggestions);
|
|
}
|
|
|
|
// Apply 'after word' punctuation and casing (when applicable). Also, set suggestion IDs.
|
|
// We delay until now so that utility functions relying on the unmodified Transform may execute properly.
|
|
|
|
let compositor = this;
|
|
suggestions.forEach(function(suggestion) {
|
|
// Valid 'keep' suggestions may have zero length; we still need to evaluate the following code
|
|
// for such cases.
|
|
|
|
// Do we need to manipulate the suggestion's transform based on the current state of the context?
|
|
if(!context.right) {
|
|
suggestion.transform.insert += punctuation.insertAfterWord;
|
|
} else {
|
|
// If we're mid-word, delete its original post-caret text.
|
|
const tokenization = compositor.tokenize(context);
|
|
if(tokenization && tokenization.caretSplitsToken) {
|
|
// While we wait on the ability to provide a more 'ideal' solution, let's at least
|
|
// go with a more stable, if slightly less ideal, solution for now.
|
|
//
|
|
// A predictive text default (on iOS, at least) - immediately wordbreak
|
|
// on suggestions accepted mid-word.
|
|
suggestion.transform.insert += punctuation.insertAfterWord;
|
|
}
|
|
}
|
|
|
|
// If this is a suggestion after wordbreak input, make sure we preserve the wordbreak transform!
|
|
if(prefixTransform) {
|
|
let mergedTransform = models.buildMergedTransform(prefixTransform, suggestion.transform);
|
|
mergedTransform.id = suggestion.transformId;
|
|
|
|
// Temporarily and locally drops 'readonly' semantics so that we can reassign the transform.
|
|
// See https://www.typescriptlang.org/docs/handbook/release-notes/typescript-2-8.html#improved-control-over-mapped-type-modifiers
|
|
let mutableSuggestion = suggestion as {-readonly [transform in keyof Suggestion]: Suggestion[transform]};
|
|
|
|
// Assignment via by-reference behavior, as suggestion is an object
|
|
mutableSuggestion.transform = mergedTransform;
|
|
}
|
|
|
|
suggestion.id = compositor.SUGGESTION_ID_SEED;
|
|
compositor.SUGGESTION_ID_SEED++;
|
|
});
|
|
|
|
// Store the suggestions on the final token of the current context state (if it exists).
|
|
// Or, once phrase-level suggestions are possible, on whichever token serves as each prediction's root.
|
|
if(postContextState) {
|
|
postContextState.tail.replacements = suggestions.map(function(suggestion) {
|
|
return {
|
|
suggestion: suggestion,
|
|
tokenWidth: 1
|
|
}
|
|
});
|
|
}
|
|
|
|
return suggestions;
|
|
}
|
|
|
|
// Responsible for applying casing rules to suggestions.
|
|
private applySuggestionCasing(suggestion: Suggestion, baseWord: USVString, casingForm: CasingForm) {
|
|
// Step 1: does the suggestion replace the whole word? If not, we should extend the suggestion to do so.
|
|
let unchangedLength = baseWord.kmwLength() - suggestion.transform.deleteLeft;
|
|
|
|
if(unchangedLength > 0) {
|
|
suggestion.transform.deleteLeft += unchangedLength;
|
|
suggestion.transform.insert = baseWord.kmwSubstr(0, unchangedLength) + suggestion.transform.insert;
|
|
}
|
|
|
|
// Step 2: Now that the transform affects the whole word, we may safely apply casing rules.
|
|
suggestion.transform.insert = this.lexicalModel.applyCasing(casingForm, suggestion.transform.insert);
|
|
suggestion.displayAs = this.lexicalModel.applyCasing(casingForm, suggestion.displayAs);
|
|
}
|
|
|
|
private toAnnotatedSuggestion(suggestion: Outcome<Suggestion>,
|
|
annotationType: SuggestionTag,
|
|
quoteBehavior?: models.QuoteBehavior): Outcome<Suggestion>;
|
|
private toAnnotatedSuggestion(suggestion: Outcome<Suggestion>,
|
|
annotationType: 'keep',
|
|
quoteBehavior?: models.QuoteBehavior): Outcome<Keep>;
|
|
private toAnnotatedSuggestion(suggestion: Outcome<Suggestion>,
|
|
annotationType: 'revert',
|
|
quoteBehavior?: models.QuoteBehavior): Outcome<Reversion>;
|
|
private toAnnotatedSuggestion(suggestion: Outcome<Suggestion>,
|
|
annotationType: SuggestionTag,
|
|
quoteBehavior: models.QuoteBehavior = models.QuoteBehavior.default): Outcome<Suggestion> {
|
|
// A method-internal 'import' of the enum.
|
|
let QuoteBehavior = models.QuoteBehavior;
|
|
|
|
let defaultQuoteBehavior = QuoteBehavior.noQuotes;
|
|
if(annotationType == 'keep' || annotationType == 'revert') {
|
|
defaultQuoteBehavior = QuoteBehavior.useQuotes;
|
|
}
|
|
|
|
return {
|
|
transform: suggestion.transform,
|
|
transformId: suggestion.transformId,
|
|
displayAs: QuoteBehavior.apply(quoteBehavior, suggestion.displayAs, this.punctuation, defaultQuoteBehavior),
|
|
tag: annotationType,
|
|
p: suggestion.p
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Returns the punctuation used for this model, filling out unspecified fields
|
|
*/
|
|
private static determinePunctuationFromModel(model: LexicalModel): LexicalModelPunctuation {
|
|
let defaults = DEFAULT_PUNCTUATION;
|
|
|
|
// Use the defaults of the model does not provide any punctuation at all.
|
|
if (!model.punctuation)
|
|
return defaults;
|
|
|
|
let specifiedPunctuation = model.punctuation;
|
|
let insertAfterWord = specifiedPunctuation.insertAfterWord;
|
|
if (insertAfterWord !== '' && !insertAfterWord) {
|
|
insertAfterWord = defaults.insertAfterWord;
|
|
}
|
|
|
|
let quotesForKeepSuggestion = specifiedPunctuation.quotesForKeepSuggestion;
|
|
if (!quotesForKeepSuggestion) {
|
|
quotesForKeepSuggestion = defaults.quotesForKeepSuggestion;
|
|
}
|
|
|
|
let isRTL = specifiedPunctuation.isRTL;
|
|
// Default: false / undefined, so no need to directly specify it.
|
|
|
|
return {
|
|
insertAfterWord, quotesForKeepSuggestion, isRTL
|
|
}
|
|
}
|
|
|
|
acceptSuggestion(suggestion: Suggestion, context: Context, postTransform?: Transform): Reversion {
|
|
// Step 1: generate and save the reversion's Transform.
|
|
let sourceTransform = suggestion.transform;
|
|
let deletedLeftChars = context.left.kmwSubstr(-sourceTransform.deleteLeft, sourceTransform.deleteLeft);
|
|
let insertedLength = sourceTransform.insert.kmwLength();
|
|
|
|
let reversionTransform: Transform = {
|
|
insert: deletedLeftChars,
|
|
deleteLeft: insertedLength
|
|
};
|
|
|
|
// Step 2: building the proper 'displayAs' string for the Reversion
|
|
let postContext = context;
|
|
if(postTransform) {
|
|
// The code above restores the state to the context at the time the `Suggestion` was created.
|
|
// `postTransform` handles any missing context that came later.
|
|
reversionTransform = models.buildMergedTransform(reversionTransform, postTransform);
|
|
|
|
// Now that we've built the reversion based upon the Suggestion's original context,
|
|
// we manipulate it in order to get a proper 'displayAs' string.
|
|
postContext = models.applyTransform(postTransform, postContext);
|
|
}
|
|
|
|
let revertedPrefix: string;
|
|
let postContextTokenization = this.tokenize(postContext);
|
|
if(postContextTokenization) {
|
|
// Handles display string for reversions triggered by accepting a suggestion mid-token.
|
|
if(postContextTokenization.left.length > 0) {
|
|
revertedPrefix = postContextTokenization.left[postContextTokenization.left.length-1];
|
|
} else {
|
|
revertedPrefix = '';
|
|
}
|
|
revertedPrefix += postContextTokenization.caretSplitsToken ? postContextTokenization.right[0] : '';
|
|
} else {
|
|
revertedPrefix = this.wordbreak(postContext);
|
|
}
|
|
|
|
let firstConversion = models.transformToSuggestion(reversionTransform);
|
|
firstConversion.displayAs = revertedPrefix;
|
|
|
|
// Build the actual Reversion, which is technically an annotated Suggestion.
|
|
// Since we're outside of the standard `predict` control path, we'll need to
|
|
// set the Reversion's ID directly.
|
|
let reversion = this.toAnnotatedSuggestion(firstConversion, 'revert');
|
|
if(suggestion.transformId != null) {
|
|
reversion.transformId = -suggestion.transformId;
|
|
}
|
|
if(suggestion.id != null) {
|
|
// Since a reversion inverts its source suggestion, we set its ID to be the
|
|
// additive inverse of the source suggestion's ID. Makes easy mapping /
|
|
// verification later.
|
|
reversion.id = -suggestion.id;
|
|
} else {
|
|
reversion.id = -this.SUGGESTION_ID_SEED;
|
|
this.SUGGESTION_ID_SEED++;
|
|
}
|
|
|
|
// Step 3: if we track Contexts, update the tracking data as appropriate.
|
|
if(this.contextTracker) {
|
|
let contextState = this.contextTracker.newest;
|
|
if(!contextState) {
|
|
contextState = this.contextTracker.analyzeState(this.lexicalModel, context);
|
|
}
|
|
|
|
contextState.tail.activeReplacementId = suggestion.id;
|
|
let acceptedContext = models.applyTransform(suggestion.transform, context);
|
|
this.contextTracker.analyzeState(this.lexicalModel, acceptedContext);
|
|
}
|
|
|
|
return reversion;
|
|
}
|
|
|
|
applyReversion(reversion: Reversion, context: Context): Suggestion[] {
|
|
// If we are unable to track context (because the model does not support LexiconTraversal),
|
|
// we need a "fallback" strategy.
|
|
let compositor = this;
|
|
let fallbackSuggestions = function() {
|
|
let revertedContext = models.applyTransform(reversion.transform, context);
|
|
let suggestions = compositor.predict({insert: '', deleteLeft: 0}, revertedContext);
|
|
suggestions.forEach(function(suggestion) {
|
|
// A reversion's transform ID is the additive inverse of its original suggestion;
|
|
// we revert to the state of said original suggestion.
|
|
suggestion.transformId = -reversion.transformId;
|
|
});
|
|
return suggestions;
|
|
}
|
|
|
|
if(!this.contextTracker) {
|
|
return fallbackSuggestions();
|
|
}
|
|
|
|
// When the context is tracked, we prefer the tracked information.
|
|
let contextMatchFound = false;
|
|
for(let c = this.contextTracker.count - 1; c >= 0; c--) {
|
|
let contextState = this.contextTracker.item(c);
|
|
|
|
if(contextState.tail.activeReplacementId == -reversion.id) {
|
|
contextMatchFound = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if(!contextMatchFound) {
|
|
return fallbackSuggestions();
|
|
}
|
|
|
|
// Remove all contexts more recent than the one we're reverting to.
|
|
while(this.contextTracker.newest.tail.activeReplacementId != -reversion.id) {
|
|
this.contextTracker.popNewest();
|
|
}
|
|
|
|
this.contextTracker.newest.tail.revert();
|
|
|
|
// Will need to be modified a bit if/when phrase-level suggestions are implemented.
|
|
// Those will be tracked on the first token of the phrase, which won't be the tail
|
|
// if they cover multiple tokens.
|
|
let suggestions = this.contextTracker.newest.tail.replacements.map(function(trackedSuggestion) {
|
|
return trackedSuggestion.suggestion;
|
|
});
|
|
|
|
suggestions.forEach(function(suggestion) {
|
|
// A reversion's transform ID is the additive inverse of its original suggestion;
|
|
// we revert to the state of said original suggestion.
|
|
suggestion.transformId = -reversion.transformId;
|
|
});
|
|
return suggestions;
|
|
}
|
|
|
|
private wordbreak(context: Context): string {
|
|
let model = this.lexicalModel;
|
|
|
|
if(model.wordbreaker || !model.wordbreak) {
|
|
// We don't need a 12.0 / 13.0 compatibility mode here.
|
|
// We're either relying on defaults or on the 14.0+ wordbreaker spec.
|
|
let wordbreaker = model.wordbreaker || wordBreakers.default;
|
|
|
|
return models.wordbreak(wordbreaker, context);
|
|
} else {
|
|
// 1. This model does not provide a model following the 14.0+ wordbreaking spec
|
|
// 2. This model DOES define a custom wordbreaker following the 12.0-13.0 spec.
|
|
|
|
// Since the model relies on custom wordbreaking behavior, we need to use the
|
|
// old, deprecated wordbreaking pattern.
|
|
return model.wordbreak(context);
|
|
}
|
|
}
|
|
|
|
private tokenize(context: Context): models.Tokenization {
|
|
let model = this.lexicalModel;
|
|
|
|
if(model.wordbreaker) {
|
|
return models.tokenize(model.wordbreaker, context);
|
|
} else {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
public resetContext(context: Context) {
|
|
// Force-resets the context, throwing out any previous fat-finger data, etc.
|
|
// Designed for use when the caret has been directly moved and/or the context sourced from a different control
|
|
// than before.
|
|
if(this.contextTracker) {
|
|
let tokenizedContext = models.tokenize(this.lexicalModel.wordbreaker || wordBreakers.default, context);
|
|
let contextState = correction.ContextTracker.modelContextState(tokenizedContext.left, null, this.lexicalModel);
|
|
this.contextTracker.enqueue(contextState);
|
|
}
|
|
}
|
|
|
|
private detectCurrentCasing(context: Context): CasingForm {
|
|
let model = this.lexicalModel;
|
|
|
|
let text = this.wordbreak(context);
|
|
|
|
if(!model.languageUsesCasing) {
|
|
throw "Invalid attempt to detect casing: languageUsesCasing is set to false";
|
|
}
|
|
|
|
if(!model.applyCasing) {
|
|
// The worker should automatically 'sub in' default behavior during the model's load if that
|
|
// function isn't defined explicitly as part of the model.
|
|
throw "Invalid LMLayer state: languageUsesCasing is set to true, but no applyCasing function exists";
|
|
}
|
|
|
|
// If the user has selected Shift or Caps layer, that overrides our
|
|
// text analysis
|
|
if(context.casingForm == 'upper' || context.casingForm == 'initial') {
|
|
return context.casingForm;
|
|
}
|
|
if(model.applyCasing('lower', text) == text) {
|
|
return 'lower';
|
|
} else if(model.applyCasing('upper', text) == text) {
|
|
// If only a single character has been input, assume we're in 'initial' mode.
|
|
return text.kmwLength() > 1 ? 'upper' : 'initial';
|
|
} else if(model.applyCasing('initial', text) == text) {
|
|
// We check 'initial' last, as upper-case input is indistinguishable.
|
|
return 'initial';
|
|
} else {
|
|
// If we do not have a casing form given to us by the keyboard, then
|
|
// 'null' is returned when no casing pattern matches the input.
|
|
return context.casingForm ?? null;
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* The default punctuation and spacing produced by the model.
|
|
*/
|
|
const DEFAULT_PUNCTUATION: LexicalModelPunctuation = {
|
|
quotesForKeepSuggestion: { open: `“`, close: `”`},
|
|
insertAfterWord: " " ,
|
|
};
|