change(web): address MD PR review concerns

This commit is contained in:
Joshua Horton 2025-11-13 08:47:18 -06:00
parent 4434c22edd
commit f61cdfb1de
2 changed files with 4 additions and 3 deletions

View file

@ -124,6 +124,7 @@ export class ContextToken {
rawText ||= '';
// Supports the old pathway for: updateWithBackspace(tokenText: string, transformId: number)
// Build a token that represents the current text with no ambiguity - probability at max (1.0)
const rawTransformDistributions: Distribution<Transform>[] = textToCharTransforms(rawText).map(function(transform) {
return [{sample: transform, p: 1.0}];
});
@ -131,9 +132,9 @@ export class ContextToken {
this._inputRange.push({
trueTransform: entry[0].sample,
inputStartIndex: 0,
bestProbFromSet: 1
bestProbFromSet: entry[0].p
});
this.searchSpace.addInput(entry, 1);
this.searchSpace.addInput(entry, entry[0].p);
});
}
}

View file

@ -586,7 +586,7 @@ export class ContextTokenization {
if(affectedToken.inputRange.length == 0 && distribution[0].sample.deleteLeft != 0) {
distribution = distribution.map((mass) => ({sample: { ...mass.sample, deleteLeft: 0 }, p: mass.p }));
}
affectedToken.addInput({trueTransform: sourceInput, inputStartIndex: appliedLength, bestProbFromSet: bestProbFromSet}, distribution);
affectedToken.addInput({trueTransform: sourceInput, inputStartIndex: appliedLength, bestProbFromSet}, distribution);
appliedLength += KMWString.length(distribution[0].sample.insert);
const tokenize = determineModelTokenizer(lexicalModel);