mirror of
https://github.com/keymanapp/keyman.git
synced 2026-08-16 13:49:23 +00:00
Merge pull request #14428 from keymanapp/refactor/web/pred-context-tokenization
refactor(web): refactor tracked-context tokenization and usage pattern 🚂
This commit is contained in:
Joshua Horton
GitHub
commit
3c0f5a8908
9 changed files with 207 additions and 101 deletions
|
|
@ -0,0 +1,52 @@
|
|||
/*
|
||||
* Keyman is copyright (C) SIL Global. MIT License.
|
||||
*
|
||||
* Created by jahorton on 2025-07-30
|
||||
*
|
||||
* Represents cached data about one potential tokenization of contents of
|
||||
* the sliding context window for one specific instance of context state.
|
||||
*/
|
||||
|
||||
import { ContextToken } from './context-token.js';
|
||||
import { TrackedContextStateAlignment } from './context-tracker.js';
|
||||
|
||||
/**
|
||||
* This class represents the sequence of tokens (words and whitespace blocks)
|
||||
* held within the active sliding context-window at a single point in time.
|
||||
*/
|
||||
export class ContextTokenization {
|
||||
readonly tokens: ContextToken[];
|
||||
readonly alignment?: TrackedContextStateAlignment;
|
||||
|
||||
constructor(priorToClone: ContextTokenization);
|
||||
constructor(tokens: ContextToken[], alignment?: TrackedContextStateAlignment);
|
||||
constructor(param1: ContextToken[] | ContextTokenization, alignment?: TrackedContextStateAlignment) {
|
||||
if(!(param1 instanceof ContextTokenization)) {
|
||||
const tokens = param1;
|
||||
this.tokens = [].concat(tokens);
|
||||
this.alignment = alignment;
|
||||
} else {
|
||||
const priorToClone = param1;
|
||||
this.tokens = priorToClone.tokens.map((entry) => new ContextToken(entry));
|
||||
this.alignment = {...priorToClone.alignment};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the token adjacent to the text insertion point.
|
||||
*/
|
||||
get tail(): ContextToken {
|
||||
return this.tokens[this.tokens.length - 1];
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a plain-text string representing the most probable representation for all
|
||||
* tokens represented by this tokenization instance.
|
||||
*/
|
||||
get exampleInput(): string[] {
|
||||
return this.tokens
|
||||
// Hide any tokens representing invisible wordbreaks. (Thinking ahead to phrase-level possibilities)
|
||||
.filter(token => token.exampleInput !== null)
|
||||
.map(token => token.exampleInput);
|
||||
}
|
||||
}
|
||||
|
|
@ -11,6 +11,7 @@ import LexicalModel = LexicalModelTypes.LexicalModel;
|
|||
import Suggestion = LexicalModelTypes.Suggestion;
|
||||
import Transform = LexicalModelTypes.Transform;
|
||||
import { ContextToken } from './context-token.js';
|
||||
import { ContextTokenization } from './context-tokenization.js';
|
||||
|
||||
/**
|
||||
* Determines the proper 'last match' index for a tokenized sequence based on its edit path.
|
||||
|
|
@ -50,7 +51,8 @@ export class TrackedContextState {
|
|||
taggedContext: Context;
|
||||
model: LexicalModel;
|
||||
|
||||
tokens: ContextToken[];
|
||||
tokenization: ContextTokenization;
|
||||
|
||||
/**
|
||||
* How many tokens were removed from the start of the best-matching ancestor.
|
||||
* Useful for restoring older states, e.g., when the user moves the caret backwards, we can recover the context at that position.
|
||||
|
|
@ -63,52 +65,18 @@ export class TrackedContextState {
|
|||
if(obj instanceof TrackedContextState) {
|
||||
let source = obj;
|
||||
// Be sure to deep-copy the tokens! Pointer-aliasing is bad here.
|
||||
this.tokens = source.tokens.map((token) => new ContextToken(token));
|
||||
this.tokenization = new ContextTokenization(source.tokenization.tokens.map((token) => new ContextToken(token)));
|
||||
|
||||
this.indexOffset = 0;
|
||||
this.model = obj.model;
|
||||
this.taggedContext = obj.taggedContext;
|
||||
} else {
|
||||
let lexicalModel = obj;
|
||||
this.tokens = [];
|
||||
this.tokenization = null;
|
||||
this.indexOffset = Number.MIN_SAFE_INTEGER;
|
||||
this.model = lexicalModel;
|
||||
}
|
||||
}
|
||||
|
||||
get head(): ContextToken {
|
||||
return this.tokens[0];
|
||||
}
|
||||
|
||||
get tail(): ContextToken {
|
||||
return this.tokens[this.tokens.length - 1];
|
||||
}
|
||||
|
||||
set tail(token: ContextToken) {
|
||||
this.tokens[this.tokens.length - 1] = token;
|
||||
}
|
||||
|
||||
popHead() {
|
||||
this.tokens.splice(0, 1);
|
||||
this.indexOffset -= 1;
|
||||
}
|
||||
|
||||
pushTail(token: ContextToken) {
|
||||
this.tokens.push(token);
|
||||
}
|
||||
|
||||
toRawTokenization() {
|
||||
let sequence: string[] = [];
|
||||
|
||||
for(let token of this.tokens) {
|
||||
// Hide any tokens representing wordbreaks. (Thinking ahead to phrase-level possibilities)
|
||||
if(token.exampleInput !== null) {
|
||||
sequence.push(token.exampleInput);
|
||||
}
|
||||
}
|
||||
|
||||
return sequence;
|
||||
}
|
||||
}
|
||||
|
||||
class CircularArray<Item> {
|
||||
|
|
@ -237,7 +205,7 @@ interface ContextMatchResult {
|
|||
* Represents token-count values resulting from an alignment attempt between two
|
||||
* different modeled context states.
|
||||
*/
|
||||
type TrackedContextStateAlignment = {
|
||||
export type TrackedContextStateAlignment = {
|
||||
/**
|
||||
* Denotes whether or not alignment is possible between two contexts.
|
||||
*/
|
||||
|
|
@ -570,7 +538,7 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
|
|||
transformSequenceDistribution?: Distribution<Transform[]>
|
||||
): ContextMatchResult {
|
||||
// Map the previous tokenized state to an edit-distance friendly version.
|
||||
let matchContext: string[] = matchState.toRawTokenization();
|
||||
let matchContext: string[] = matchState.tokenization.exampleInput;
|
||||
|
||||
const alignmentResults = this.attemptTokenizedAlignment(tokenizedContext.map((token) => token.text), matchContext);
|
||||
|
||||
|
|
@ -601,13 +569,10 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
|
|||
}
|
||||
|
||||
// If mutations HAVE happened, we have work to do.
|
||||
let state = matchState;
|
||||
const tokenization = matchState.tokenization.tokens.map((token) => new ContextToken(token));
|
||||
|
||||
if(leadTokenShift < 0) {
|
||||
state = new TrackedContextState(state);
|
||||
for(let i = 0; i > leadTokenShift; i--) {
|
||||
state.popHead();
|
||||
}
|
||||
tokenization.splice(0, -leadTokenShift);
|
||||
} else if(leadTokenShift > 0) {
|
||||
// TODO: insert token(s) at the start to match the text that's back within the
|
||||
// sliding context window.
|
||||
|
|
@ -618,6 +583,8 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
|
|||
|
||||
// If no TAIL mutations have happened, we're safe to return now.
|
||||
if(tailEditLength == 0 && tailTokenShift == 0) {
|
||||
const state = new TrackedContextState(matchState);
|
||||
state.tokenization = new ContextTokenization(tokenization, alignmentResults);
|
||||
return {
|
||||
state: state,
|
||||
baseState: matchState,
|
||||
|
|
@ -665,7 +632,7 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
|
|||
const matchingIndex = i + matchingTailUpdateIndex;
|
||||
|
||||
const incomingToken = tokenizedContext[incomingIndex];
|
||||
const matchedToken = matchState.tokens[matchingIndex];
|
||||
const matchedToken = matchState.tokenization.tokens[matchingIndex];
|
||||
|
||||
let primaryInput = hasDistribution ? tokenizedPrimaryInput[i] : null;
|
||||
const isBackspace = primaryInput && TransformUtils.isBackspace(primaryInput);
|
||||
|
|
@ -673,7 +640,6 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
|
|||
const isLastToken = incomingIndex == tokenizedContext.length - 1;
|
||||
|
||||
if(isLastToken) {
|
||||
state = new TrackedContextState(state);
|
||||
// If this token's transform component is not part of the final token,
|
||||
// it's something we'll want to preserve even when applying suggestions
|
||||
// for the final token.
|
||||
|
|
@ -689,19 +655,17 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
|
|||
token = new ContextToken(matchState.model, incomingToken.text);
|
||||
token.searchSpace.inputSequence.forEach((entry) => entry[0].sample.id = primaryInput.id);
|
||||
} else {
|
||||
// Assumption: there have been no intervening keystrokes since the last well-aligned context.
|
||||
// (May not be valid with epic/dict-breaker or with complex, word-boundary crossing transforms)
|
||||
token = new ContextToken(matchedToken);
|
||||
token.searchSpace.addInput(tokenDistribution.map((seq) => seq[tailIndex]));
|
||||
}
|
||||
|
||||
state.tokens[incomingIndex] = token;
|
||||
tokenization[incomingIndex] = token;
|
||||
tailIndex++;
|
||||
}
|
||||
|
||||
if(tailTokenShift < 0) {
|
||||
if(state == matchState) {
|
||||
state = new TrackedContextState(state);
|
||||
}
|
||||
|
||||
// delete tail tokens
|
||||
for(let i = 0; i > tailTokenShift; i--) {
|
||||
// If ALL that remains are deletes, we're good to go.
|
||||
|
|
@ -709,13 +673,9 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
|
|||
// This may not be the token at the index, but since all that remains are deletes,
|
||||
// we'll have deleted the correct total number from the end once all iterations
|
||||
// are done.
|
||||
state.tokens.pop();
|
||||
tokenization.pop();
|
||||
}
|
||||
} else {
|
||||
if(state == matchState) {
|
||||
state = new TrackedContextState(state);
|
||||
}
|
||||
|
||||
for(let i = tailEditLength; i < tailEditLength + tailTokenShift; i++) {
|
||||
// create tail tokens
|
||||
const incomingIndex = i + incomingTailUpdateIndex;
|
||||
|
|
@ -738,11 +698,7 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
|
|||
preservationTransform = preservationTransform && primaryInput ? buildMergedTransform(preservationTransform, primaryInput) : (primaryInput ?? preservationTransform);
|
||||
}
|
||||
|
||||
if(state == matchState) {
|
||||
state = new TrackedContextState(state);
|
||||
}
|
||||
|
||||
let pushedToken = new ContextToken(state.model);
|
||||
let pushedToken = new ContextToken(matchState.model);
|
||||
|
||||
// TODO: assumes that there was no shift in wordbreaking from the
|
||||
// prior context to the current one. This may actually be a major
|
||||
|
|
@ -778,12 +734,15 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
|
|||
pushedToken.isWhitespace = incomingToken.isWhitespace;
|
||||
|
||||
// Auto-replaces the search space to correspond with the new token.
|
||||
state.pushTail(pushedToken);
|
||||
tokenization.push(pushedToken);
|
||||
|
||||
tailIndex++;
|
||||
}
|
||||
}
|
||||
|
||||
const state = new TrackedContextState(matchState);
|
||||
state.tokenization = new ContextTokenization(tokenization, alignmentResults);
|
||||
|
||||
return {
|
||||
state,
|
||||
baseState: matchState,
|
||||
|
|
@ -809,16 +768,18 @@ export class ContextTracker extends CircularArray<TrackedContextState> {
|
|||
|
||||
// And now build the final context state object, which includes whitespace 'tokens'.
|
||||
let state = new TrackedContextState(lexicalModel);
|
||||
const tokenization: ContextToken[] = [];
|
||||
|
||||
while(baseTokens.length > 0) {
|
||||
state.pushTail(baseTokens.splice(0, 1)[0]);
|
||||
tokenization.push(baseTokens.splice(0, 1)[0]);
|
||||
}
|
||||
|
||||
if(state.tokens.length == 0) {
|
||||
if(tokenization.length == 0) {
|
||||
let token = new ContextToken(lexicalModel);
|
||||
state.pushTail(token);
|
||||
tokenization.push(token);
|
||||
}
|
||||
|
||||
state.tokenization = new ContextTokenization(tokenization);
|
||||
return state;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -191,7 +191,7 @@ export class ModelCompositor {
|
|||
// Store the suggestions on the final token of the current context state (if it exists).
|
||||
// Or, once phrase-level suggestions are possible, on whichever token serves as each prediction's root.
|
||||
if(postContextState) {
|
||||
postContextState.tail.suggestions = suggestions;
|
||||
postContextState.tokenization.tail.suggestions = suggestions;
|
||||
}
|
||||
|
||||
return suggestions;
|
||||
|
|
@ -258,7 +258,7 @@ export class ModelCompositor {
|
|||
contextState = this.contextTracker.analyzeState(this.lexicalModel, context).state;
|
||||
}
|
||||
|
||||
contextState.tail.appliedSuggestionId = suggestion.id;
|
||||
contextState.tokenization.tail.appliedSuggestionId = suggestion.id;
|
||||
let acceptedContext = models.applyTransform(suggestion.transform, context);
|
||||
if(suggestion.appendedTransform) {
|
||||
acceptedContext = models.applyTransform(suggestion.appendedTransform, acceptedContext);
|
||||
|
|
@ -297,7 +297,7 @@ export class ModelCompositor {
|
|||
for(let c = this.contextTracker.count - 1; c >= 0; c--) {
|
||||
let contextState = this.contextTracker.item(c);
|
||||
|
||||
if(contextState.tail.appliedSuggestionId == -reversion.id) {
|
||||
if(contextState.tokenization.tail.appliedSuggestionId == -reversion.id) {
|
||||
contextMatchFound = true;
|
||||
break;
|
||||
}
|
||||
|
|
@ -308,16 +308,16 @@ export class ModelCompositor {
|
|||
}
|
||||
|
||||
// Remove all contexts more recent than the one we're reverting to.
|
||||
while(this.contextTracker.newest.tail.appliedSuggestionId != -reversion.id) {
|
||||
while(this.contextTracker.newest.tokenization.tail.appliedSuggestionId != -reversion.id) {
|
||||
this.contextTracker.popNewest();
|
||||
}
|
||||
|
||||
this.contextTracker.newest.tail.appliedSuggestionId = undefined;
|
||||
this.contextTracker.newest.tokenization.tail.appliedSuggestionId = undefined;
|
||||
|
||||
// Will need to be modified a bit if/when phrase-level suggestions are implemented.
|
||||
// Those will be tracked on the first token of the phrase, which won't be the tail
|
||||
// if they cover multiple tokens.
|
||||
let suggestions = this.contextTracker.newest.tail.suggestions;
|
||||
let suggestions = this.contextTracker.newest.tokenization.tail.suggestions;
|
||||
|
||||
suggestions.forEach(function(suggestion) {
|
||||
// A reversion's transform ID is the additive inverse of its original suggestion;
|
||||
|
|
|
|||
|
|
@ -263,7 +263,7 @@ export async function correctAndEnumerate(
|
|||
// let's just note that right now, there will only ever be one.
|
||||
//
|
||||
// The 'eventual' logic will be significantly more complex, though still manageable.
|
||||
const searchSpace = postContextState.tail.searchSpace;
|
||||
const searchSpace = postContextState.tokenization.tail.searchSpace;
|
||||
|
||||
// No matter the prediction, once we know the root of the prediction, we'll always 'replace' the
|
||||
// same amount of text. We can handle this before the big 'prediction root' loop.
|
||||
|
|
@ -271,9 +271,9 @@ export async function correctAndEnumerate(
|
|||
|
||||
// The amount of text to 'replace' depends upon whatever sort of context change occurs
|
||||
// from the received input.
|
||||
const postContextTokens = postContextState.tokens;
|
||||
const postContextTokens = postContextState.tokenization.tokens;
|
||||
// Only use of `contextState`.
|
||||
let contextLengthDelta = postContextTokens.length - contextState.tokens.length;
|
||||
let contextLengthDelta = postContextTokens.length - contextState.tokenization.tokens.length;
|
||||
// If the context now has more tokens, the token we'll be 'predicting' didn't originally exist.
|
||||
if(contextChangeAnalysis.preservationTransform) {
|
||||
// As the word/token being corrected/predicted didn't originally exist, there's no
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
export { ClassicalDistanceCalculation } from './correction/classical-calculation.js';
|
||||
export { ContextToken } from './correction/context-token.js';
|
||||
export { ContextTokenization } from './correction/context-tokenization.js';
|
||||
export { ContextTracker } from './correction/context-tracker.js';
|
||||
export * as correction from './correction/index.js';
|
||||
export * from './model-helpers.js';
|
||||
|
|
|
|||
|
|
@ -0,0 +1,92 @@
|
|||
/*
|
||||
* Keyman is copyright (C) SIL Global. MIT License.
|
||||
*
|
||||
* Created by jahorton on 2025-07-30
|
||||
*
|
||||
* This file contains low-level tests designed to validate the behavior of the
|
||||
* of the ContextTokenization class and its integration with the lower-level
|
||||
* classes that it utilizes.
|
||||
*/
|
||||
|
||||
import { assert } from 'chai';
|
||||
|
||||
import { default as defaultBreaker } from '@keymanapp/models-wordbreakers';
|
||||
import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs';
|
||||
|
||||
import { ContextToken, ContextTokenization, models } from '@keymanapp/lm-worker/test-index';
|
||||
|
||||
import TrieModel = models.TrieModel;
|
||||
|
||||
var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'),
|
||||
{wordBreaker: defaultBreaker});
|
||||
|
||||
function toToken(text: string) {
|
||||
let isWhitespace = text == ' ';
|
||||
let token = new ContextToken(plainModel, text);
|
||||
token.isWhitespace = isWhitespace;
|
||||
return token;
|
||||
}
|
||||
|
||||
describe('ContextTokenization', function() {
|
||||
describe("<constructor>", () => {
|
||||
it("constructs from just a token array", () => {
|
||||
const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
|
||||
let tokenization = new ContextTokenization(rawTextTokens.map((text => toToken(text))));
|
||||
|
||||
assert.deepEqual(tokenization.tokens.map((entry) => entry.exampleInput), rawTextTokens);
|
||||
assert.deepEqual(tokenization.tokens.map((entry) => entry.isWhitespace), rawTextTokens.map((entry) => entry == ' '));
|
||||
assert.isNotOk(tokenization.alignment);
|
||||
assert.equal(tokenization.tail.exampleInput, 'day');
|
||||
assert.isFalse(tokenization.tail.isWhitespace);
|
||||
assert.isUndefined(tokenization.tail.appliedSuggestionId);
|
||||
});
|
||||
|
||||
it("constructs from a token array + alignment data", () => {
|
||||
const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
|
||||
let alignment = {
|
||||
canAlign: true,
|
||||
leadTokenShift: 0,
|
||||
matchLength: 6,
|
||||
tailEditLength: 1,
|
||||
tailTokenShift: 0
|
||||
};
|
||||
|
||||
let tokenization = new ContextTokenization(rawTextTokens.map((text => toToken(text))), alignment);
|
||||
|
||||
assert.deepEqual(tokenization.tokens.map((entry) => entry.exampleInput), rawTextTokens);
|
||||
assert.deepEqual(tokenization.tokens.map((entry) => entry.isWhitespace), rawTextTokens.map((entry) => entry == ' '));
|
||||
assert.isOk(tokenization.alignment);
|
||||
assert.deepEqual(tokenization.alignment, alignment);
|
||||
assert.equal(tokenization.tail.exampleInput, 'day');
|
||||
assert.isFalse(tokenization.tail.isWhitespace);
|
||||
assert.isUndefined(tokenization.tail.appliedSuggestionId);
|
||||
});
|
||||
|
||||
it('clones', () => {
|
||||
const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
|
||||
|
||||
let baseTokenization = new ContextTokenization(rawTextTokens.map((text => toToken(text))), {
|
||||
canAlign: true,
|
||||
leadTokenShift: 0,
|
||||
matchLength: 6,
|
||||
tailEditLength: 1,
|
||||
tailTokenShift: 0
|
||||
});
|
||||
|
||||
let cloned = new ContextTokenization(baseTokenization);
|
||||
|
||||
assert.notDeepEqual(cloned, baseTokenization);
|
||||
assert.notDeepEqual(cloned.tokens, baseTokenization.tokens);
|
||||
assert.deepEqual(cloned.tokens.map((token) => token.searchSpace.inputSequence),
|
||||
baseTokenization.tokens.map((token) => token.searchSpace.inputSequence));
|
||||
assert.deepEqual(cloned.alignment, baseTokenization.alignment);
|
||||
});
|
||||
});
|
||||
|
||||
it('exampleInput', () => {
|
||||
const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
|
||||
let tokenization = new ContextTokenization(rawTextTokens.map((text => toToken(text))));
|
||||
|
||||
assert.deepEqual(tokenization.exampleInput, rawTextTokens);
|
||||
});
|
||||
});
|
||||
|
|
@ -427,7 +427,7 @@ describe('ContextTracker', function() {
|
|||
let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
|
||||
let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
|
||||
assert.isNotNull(newContextMatch?.state);
|
||||
assert.deepEqual(newContextMatch.state.tokens.map(token => token.exampleInput), rawTokens);
|
||||
assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
assert.equal(newContextMatch.headTokensRemoved, 1);
|
||||
assert.equal(newContextMatch.tailTokensAdded, 0);
|
||||
});
|
||||
|
|
@ -447,7 +447,7 @@ describe('ContextTracker', function() {
|
|||
let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
|
||||
let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
|
||||
assert.isNotNull(newContextMatch?.state);
|
||||
assert.deepEqual(newContextMatch.state.tokens.map(token => token.exampleInput), rawTokens);
|
||||
assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
assert.equal(newContextMatch.headTokensRemoved, 2);
|
||||
assert.equal(newContextMatch.tailTokensAdded, 0);
|
||||
});
|
||||
|
|
@ -468,7 +468,7 @@ describe('ContextTracker', function() {
|
|||
let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
|
||||
let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
|
||||
assert.isNotNull(newContextMatch?.state);
|
||||
assert.deepEqual(newContextMatch.state.tokens.map(token => token.exampleInput), rawTokens);
|
||||
assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
assert.equal(newContextMatch.headTokensRemoved, 0);
|
||||
assert.equal(newContextMatch.tailTokensAdded, 0);
|
||||
});
|
||||
|
|
@ -490,14 +490,14 @@ describe('ContextTracker', function() {
|
|||
let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
|
||||
let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
|
||||
assert.isNotNull(newContextMatch?.state);
|
||||
assert.deepEqual(newContextMatch.state.tokens.map(token => token.exampleInput), rawTokens);
|
||||
assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
// We want to preserve the added whitespace when predicting a token that follows after it.
|
||||
assert.deepEqual(newContextMatch.preservationTransform, { insert: ' ', deleteLeft: 0 });
|
||||
|
||||
// The 'wordbreak' transform
|
||||
let state = newContextMatch?.state;
|
||||
assert.isNotEmpty(state.tokens[state.tokens.length - 2].searchSpace.inputSequence);
|
||||
assert.isEmpty(state.tokens[state.tokens.length - 1].searchSpace.inputSequence);
|
||||
assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence);
|
||||
assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence);
|
||||
assert.equal(newContextMatch.headTokensRemoved, 0);
|
||||
assert.equal(newContextMatch.tailTokensAdded, 2);
|
||||
});
|
||||
|
|
@ -518,7 +518,7 @@ describe('ContextTracker', function() {
|
|||
let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
|
||||
let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
|
||||
assert.isOk(newContextMatch?.state);
|
||||
assert.deepEqual(newContextMatch?.state.tokens.map(token => token.exampleInput), rawTokens);
|
||||
assert.deepEqual(newContextMatch?.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
|
||||
// The 'wordbreak' transform
|
||||
assert.equal(newContextMatch.headTokensRemoved, 0);
|
||||
|
|
@ -541,13 +541,13 @@ describe('ContextTracker', function() {
|
|||
let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
|
||||
let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
|
||||
assert.isNotNull(newContextMatch?.state);
|
||||
assert.deepEqual(newContextMatch.state.tokens.map(token => token.exampleInput), rawTokens);
|
||||
assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
assert.deepEqual(newContextMatch.preservationTransform, { insert: '', deleteLeft: 0 });
|
||||
|
||||
// The 'wordbreak' transform
|
||||
let state = newContextMatch.state;
|
||||
assert.isNotEmpty(state.tokens[state.tokens.length - 2].searchSpace.inputSequence);
|
||||
assert.isNotEmpty(state.tokens[state.tokens.length - 1].searchSpace.inputSequence);
|
||||
assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence);
|
||||
assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence);
|
||||
|
||||
assert.equal(newContextMatch.headTokensRemoved, 0);
|
||||
assert.equal(newContextMatch.tailTokensAdded, 1);
|
||||
|
|
@ -570,14 +570,14 @@ describe('ContextTracker', function() {
|
|||
let baseContextMatch = ContextTracker.modelContextState(existingContext.left, plainModel);
|
||||
let newContextMatch = ContextTracker.attemptMatchContext(newContext.left, baseContextMatch, toWrapperDistribution(transform));
|
||||
assert.isNotNull(newContextMatch?.state);
|
||||
assert.deepEqual(newContextMatch.state.tokens.map(token => token.exampleInput), rawTokens);
|
||||
assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
// We want to preserve the added whitespace when predicting a token that follows after it.
|
||||
assert.deepEqual(newContextMatch.preservationTransform, { insert: ' ', deleteLeft: 0 });
|
||||
|
||||
// The 'wordbreak' transform
|
||||
let state = newContextMatch.state;
|
||||
assert.isNotEmpty(state.tokens[state.tokens.length - 2].searchSpace.inputSequence);
|
||||
assert.isEmpty(state.tokens[state.tokens.length - 1].searchSpace.inputSequence);
|
||||
assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence);
|
||||
assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence);
|
||||
|
||||
assert.equal(newContextMatch.headTokensRemoved, 2);
|
||||
assert.equal(newContextMatch.tailTokensAdded, 2);
|
||||
|
|
@ -603,14 +603,14 @@ describe('ContextTracker', function() {
|
|||
tokenizeTransformDistribution(tokenizer, {left: "an", startOfBuffer: true, endOfBuffer: true}, [{sample: transform, p: 1}])
|
||||
);
|
||||
assert.isNotNull(newContextMatch?.state);
|
||||
assert.deepEqual(newContextMatch.state.tokens.map(token => token.exampleInput), rawTokens);
|
||||
assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
// We want to preserve all text preceding the new token when applying a suggestion.
|
||||
assert.deepEqual(newContextMatch.preservationTransform, { insert: 'd ', deleteLeft: 0});
|
||||
|
||||
// The 'wordbreak' transform
|
||||
let state = newContextMatch.state;
|
||||
assert.isNotEmpty(state.tokens[state.tokens.length - 2].searchSpace.inputSequence);
|
||||
assert.isEmpty(state.tokens[state.tokens.length - 1].searchSpace.inputSequence);
|
||||
assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence);
|
||||
assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence);
|
||||
|
||||
assert.equal(newContextMatch.headTokensRemoved, 0);
|
||||
assert.equal(newContextMatch.tailTokensAdded, 2);
|
||||
|
|
@ -636,14 +636,14 @@ describe('ContextTracker', function() {
|
|||
tokenizeTransformDistribution(tokenizer, {left: "apple a day keeps the doc", startOfBuffer: true, endOfBuffer: true}, [{sample: transform, p: 1}])
|
||||
);
|
||||
assert.isNotNull(newContextMatch?.state);
|
||||
assert.deepEqual(newContextMatch.state.tokens.map(token => token.exampleInput), rawTokens);
|
||||
assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
// We want to preserve all text preceding the new token when applying a suggestion.
|
||||
assert.deepEqual(newContextMatch.preservationTransform, { insert: 'tor ', deleteLeft: 0 });
|
||||
|
||||
// The 'wordbreak' transform
|
||||
let state = newContextMatch.state;
|
||||
assert.isNotEmpty(state.tokens[state.tokens.length - 2].searchSpace.inputSequence);
|
||||
assert.isEmpty(state.tokens[state.tokens.length - 1].searchSpace.inputSequence);
|
||||
assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence);
|
||||
assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence);
|
||||
|
||||
assert.equal(newContextMatch.headTokensRemoved, 0);
|
||||
assert.equal(newContextMatch.tailTokensAdded, 2);
|
||||
|
|
@ -688,7 +688,7 @@ describe('ContextTracker', function() {
|
|||
let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor"];
|
||||
|
||||
let state = ContextTracker.modelContextState(tokenized, plainModel);
|
||||
assert.deepEqual(state.tokens.map(token => token.exampleInput), rawTokens);
|
||||
assert.deepEqual(state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
});
|
||||
|
||||
it('models with final wordbreak', function() {
|
||||
|
|
@ -701,7 +701,7 @@ describe('ContextTracker', function() {
|
|||
let rawTokens = ["an", " ", "apple", " ", "a", " ", "day", " ", "keeps", " ", "the", " ", "doctor", " ", ""];
|
||||
|
||||
let state = ContextTracker.modelContextState(tokenized, plainModel);
|
||||
assert.deepEqual(state.tokens.map(token => token.exampleInput), rawTokens);
|
||||
assert.deepEqual(state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -743,12 +743,12 @@ describe('ContextTracker', function() {
|
|||
let compositor = new ModelCompositor(model);
|
||||
let baseContextMatch = compositor.contextTracker.analyzeState(model, baseContext);
|
||||
|
||||
baseContextMatch.state.tail.suggestions = [ baseSuggestion ];
|
||||
baseContextMatch.state.tokenization.tail.suggestions = [ baseSuggestion ];
|
||||
|
||||
let reversion = compositor.acceptSuggestion(baseSuggestion, baseContext, postTransform);
|
||||
|
||||
// Actual test assertion - was the replacement tracked?
|
||||
assert.equal(baseContextMatch.state.tail.appliedSuggestionId, baseSuggestion.id);
|
||||
assert.equal(baseContextMatch.state.tokenization.tail.appliedSuggestionId, baseSuggestion.id);
|
||||
assert.equal(reversion.id, -baseSuggestion.id);
|
||||
|
||||
// Next step - on the followup context, is the replacement still active?
|
||||
|
|
@ -756,10 +756,10 @@ describe('ContextTracker', function() {
|
|||
let postContextMatch = compositor.contextTracker.analyzeState(model, postContext);
|
||||
|
||||
// Penultimate token corresponds to whitespace, which does not have a 'raw' representation.
|
||||
assert.equal(postContextMatch.state.tokens[postContextMatch.state.tokens.length - 2].exampleInput, ' ');
|
||||
assert.equal(postContextMatch.state.tokenization.tokens[postContextMatch.state.tokenization.tokens.length - 2].exampleInput, ' ');
|
||||
|
||||
// Final token is empty (follows a wordbreak)
|
||||
assert.equal(postContextMatch.state.tail.exampleInput, '');
|
||||
assert.equal(postContextMatch.state.tokenization.tail.exampleInput, '');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -898,7 +898,7 @@ describe('ModelCompositor', function() {
|
|||
assert.equal(compositor.contextTracker.count, 3);
|
||||
|
||||
// The replacement should be marked on the context-tracking token.
|
||||
assert.isAtLeast(suggestionContextState.tail.appliedSuggestionId, 0);
|
||||
assert.isAtLeast(suggestionContextState.tokenization.tail.appliedSuggestionId, 0);
|
||||
|
||||
let appliedContext = models.applyTransform(baseSuggestion.transform, baseContext);
|
||||
compositor.applyReversion(reversion, appliedContext);
|
||||
|
|
@ -908,7 +908,7 @@ describe('ModelCompositor', function() {
|
|||
assert.equal(compositor.contextTracker.item(1), suggestionContextState);
|
||||
|
||||
// The replacement should no longer be marked for the context-tracking token.
|
||||
assert.isNotOk(suggestionContextState.tail.appliedSuggestionId);
|
||||
assert.isNotOk(suggestionContextState.tokenization.tail.appliedSuggestionId);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue