mirror of
https://github.com/keymanapp/keyman.git
synced 2026-10-20 12:57:49 +00:00
change(web): address MD PR review concerns
This commit is contained in:
parent
4434c22edd
commit
f61cdfb1de
2 changed files with 4 additions and 3 deletions
|
|
@ -124,6 +124,7 @@ export class ContextToken {
|
|||
rawText ||= '';
|
||||
|
||||
// Supports the old pathway for: updateWithBackspace(tokenText: string, transformId: number)
|
||||
// Build a token that represents the current text with no ambiguity - probability at max (1.0)
|
||||
const rawTransformDistributions: Distribution<Transform>[] = textToCharTransforms(rawText).map(function(transform) {
|
||||
return [{sample: transform, p: 1.0}];
|
||||
});
|
||||
|
|
@ -131,9 +132,9 @@ export class ContextToken {
|
|||
this._inputRange.push({
|
||||
trueTransform: entry[0].sample,
|
||||
inputStartIndex: 0,
|
||||
bestProbFromSet: 1
|
||||
bestProbFromSet: entry[0].p
|
||||
});
|
||||
this.searchSpace.addInput(entry, 1);
|
||||
this.searchSpace.addInput(entry, entry[0].p);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -586,7 +586,7 @@ export class ContextTokenization {
|
|||
if(affectedToken.inputRange.length == 0 && distribution[0].sample.deleteLeft != 0) {
|
||||
distribution = distribution.map((mass) => ({sample: { ...mass.sample, deleteLeft: 0 }, p: mass.p }));
|
||||
}
|
||||
affectedToken.addInput({trueTransform: sourceInput, inputStartIndex: appliedLength, bestProbFromSet: bestProbFromSet}, distribution);
|
||||
affectedToken.addInput({trueTransform: sourceInput, inputStartIndex: appliedLength, bestProbFromSet}, distribution);
|
||||
appliedLength += KMWString.length(distribution[0].sample.insert);
|
||||
|
||||
const tokenize = determineModelTokenizer(lexicalModel);
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue