mirror of
https://github.com/keymanapp/keyman.git
synced 2026-08-07 01:15:33 +00:00
Merge pull request #14440 from keymanapp/refactor/web/pred-context-transition
refactor(web): refactor tracked-context state-transition return object 🚂
This commit is contained in:
Joshua Horton
GitHub
commit
e4edb6f78b
5 changed files with 224 additions and 103 deletions
|
|
@ -11,6 +11,7 @@ import Transform = LexicalModelTypes.Transform;
|
|||
import { ContextToken } from './context-token.js';
|
||||
import { ContextTokenization } from './context-tokenization.js';
|
||||
import { ContextState } from './context-state.js';
|
||||
import { ContextTransition } from './context-transition.js';
|
||||
|
||||
class CircularArray<Item> {
|
||||
static readonly DEFAULT_ARRAY_SIZE = 5;
|
||||
|
|
@ -103,37 +104,6 @@ class CircularArray<Item> {
|
|||
}
|
||||
}
|
||||
|
||||
interface ContextMatchResult {
|
||||
/**
|
||||
* Represents the current state of the context after applying incoming keystroke data.
|
||||
*/
|
||||
state: ContextState;
|
||||
|
||||
/**
|
||||
* Represents the previously-cached context state that best matches `state` if available.
|
||||
* May be `null` if no such state could be found within the context-state cache.
|
||||
*/
|
||||
baseState: ContextState;
|
||||
|
||||
/**
|
||||
* Indicates the portion of the incoming keystroke data, if any, that applies to
|
||||
* tokens before the last pre-caret token and thus should not be replaced by predictions
|
||||
* based upon `state`. If the provided context state + the incoming transform do not
|
||||
* adequately match the current context, the match attempt will fail with a `null` result.
|
||||
*
|
||||
* Should generally be non-null if the token before the caret did not previously exist.
|
||||
*
|
||||
* The result may be null if it does not match the prior context state or if bookkeeping
|
||||
* based upon it is problematic - say, if wordbreaking effects shift due to new input,
|
||||
* causing a mismatch with the prior state's tokenization.
|
||||
* (Refer to #12494 for an example case.)
|
||||
*/
|
||||
preservationTransform?: Transform;
|
||||
|
||||
headTokensRemoved: number;
|
||||
tailTokensAdded: number;
|
||||
}
|
||||
|
||||
export class ContextTracker extends CircularArray<ContextState> {
|
||||
// Aim: relocate to ContextTransition in some form?
|
||||
// Or can we split it up in some manner across the different types?
|
||||
|
|
@ -143,7 +113,8 @@ export class ContextTracker extends CircularArray<ContextState> {
|
|||
matchState: ContextState,
|
||||
// the distribution should be tokenized already.
|
||||
transformDistribution?: Distribution<Transform> // transform distribution is needed here.
|
||||
): ContextMatchResult {
|
||||
): ContextTransition {
|
||||
const baseTransition = new ContextTransition(matchState, matchState.appliedInput?.id);
|
||||
const transformSequenceDistribution = tokenizeAndFilterDistribution(context, lexicalModel, transformDistribution);
|
||||
|
||||
if(transformDistribution?.[0]) {
|
||||
|
|
@ -168,7 +139,8 @@ export class ContextTracker extends CircularArray<ContextState> {
|
|||
// If we have a perfect match with a pre-existing context, no mutations have
|
||||
// happened; just re-use the old context state.
|
||||
if(tailEditLength == 0 && leadTokenShift == 0 && tailTokenShift == 0) {
|
||||
return { state: matchState, baseState: matchState, headTokensRemoved: 0, tailTokensAdded: 0 };
|
||||
baseTransition.finalize(matchState, transformDistribution);
|
||||
return baseTransition;
|
||||
} else {
|
||||
// If we didn't get any input, we really should perfectly match
|
||||
// a previous context state. If such a state is out of our cache,
|
||||
|
|
@ -195,12 +167,8 @@ export class ContextTracker extends CircularArray<ContextState> {
|
|||
if(tailEditLength == 0 && tailTokenShift == 0) {
|
||||
const state = new ContextState(context, lexicalModel);
|
||||
state.tokenization = new ContextTokenization(tokenization, alignmentResults);
|
||||
return {
|
||||
state: state,
|
||||
baseState: matchState,
|
||||
headTokensRemoved: -leadTokenShift,
|
||||
tailTokensAdded: tailTokenShift
|
||||
}
|
||||
baseTransition.finalize(state, transformDistribution);
|
||||
return baseTransition;
|
||||
}
|
||||
|
||||
// ***
|
||||
|
|
@ -352,14 +320,8 @@ export class ContextTracker extends CircularArray<ContextState> {
|
|||
|
||||
const state = new ContextState(context, lexicalModel);
|
||||
state.tokenization = new ContextTokenization(tokenization, alignmentResults);
|
||||
|
||||
return {
|
||||
state,
|
||||
baseState: matchState,
|
||||
preservationTransform,
|
||||
headTokensRemoved: alignmentResults.leadTokenShift < 0 ? -alignmentResults.leadTokenShift : 0,
|
||||
tailTokensAdded: alignmentResults.tailTokenShift
|
||||
};
|
||||
baseTransition.finalize(state, transformDistribution, preservationTransform);
|
||||
return baseTransition;
|
||||
}
|
||||
|
||||
// Aim: relocate to ContextState in some form... or ContextTransition?
|
||||
|
|
@ -378,7 +340,7 @@ export class ContextTracker extends CircularArray<ContextState> {
|
|||
context: Context,
|
||||
transformDistribution?: Distribution<Transform>,
|
||||
preserveMatchState?: boolean
|
||||
): ContextMatchResult {
|
||||
): ContextTransition {
|
||||
if(!model.traverseFromRoot) {
|
||||
// Assumption: LexicalModel provides a valid traverseFromRoot function. (Is technically optional)
|
||||
// Without it, no 'corrections' may be made; the model can only be used to predict, not correct.
|
||||
|
|
@ -388,6 +350,7 @@ export class ContextTracker extends CircularArray<ContextState> {
|
|||
if(transformDistribution?.length == 0) {
|
||||
transformDistribution = null;
|
||||
}
|
||||
|
||||
const inputTransform = transformDistribution?.[0];
|
||||
const postContext = inputTransform ? applyTransform(inputTransform.sample, context) : context;
|
||||
|
||||
|
|
@ -420,17 +383,17 @@ export class ContextTracker extends CircularArray<ContextState> {
|
|||
|
||||
let result = ContextTracker.attemptMatchContext(context, model, this.item(i), transformDistribution);
|
||||
|
||||
if(result?.state) {
|
||||
if(result?.final) {
|
||||
// Keep it reasonably current! And it's probably fine to have it more than once
|
||||
// in the history. However, if it's the most current already, there's no need
|
||||
// to refresh it.
|
||||
if(this.newest != result.state && this.newest != priorMatchState) {
|
||||
if(this.newest != result.final && this.newest != priorMatchState) {
|
||||
// Already has a taggedContext.
|
||||
this.enqueue(priorMatchState);
|
||||
}
|
||||
|
||||
if(result.state != this.item(i)) {
|
||||
this.enqueue(result.state);
|
||||
if(result.final != this.item(i)) {
|
||||
this.enqueue(result.final);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
|
@ -444,7 +407,11 @@ export class ContextTracker extends CircularArray<ContextState> {
|
|||
// this state is only reached on caret moves; thus, transformDistribution is actually just a single null transform.
|
||||
let state = new ContextState(context, model);
|
||||
this.enqueue(state);
|
||||
return { state, baseState: null, headTokensRemoved: 0, tailTokensAdded: 0 };
|
||||
const transition = new ContextTransition(state, /* TODO: we need a clear value here in the future! */ null);
|
||||
// Hacky, but holds the course for now. This should only really happen from context resets, which can
|
||||
// then use a different path.
|
||||
transition.finalize(state, []);
|
||||
return transition;
|
||||
}
|
||||
|
||||
clearCache() {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,115 @@
|
|||
/*
|
||||
* Keyman is copyright (C) SIL Global. MIT License.
|
||||
*
|
||||
* Created by jahorton on 2025-07-30
|
||||
*
|
||||
* Represents cached data about a single context transition event, as well
|
||||
* as the state of the context both before and after the transition.
|
||||
*/
|
||||
|
||||
import { LexicalModelTypes } from '@keymanapp/common-types';
|
||||
|
||||
import { ContextState } from './context-state.js';
|
||||
|
||||
import Distribution = LexicalModelTypes.Distribution;
|
||||
import Transform = LexicalModelTypes.Transform;
|
||||
|
||||
/**
|
||||
* Represents the transition between two context states as triggered
|
||||
* by input keystrokes or applied suggestions.
|
||||
*/
|
||||
export class ContextTransition {
|
||||
/**
|
||||
* Represents the state of the context before the transition event occurred.
|
||||
*/
|
||||
readonly base: ContextState;
|
||||
private _final: ContextState;
|
||||
|
||||
/**
|
||||
* Indicates the fat-finger distribution for the incoming keystroke related to
|
||||
* the context transition event.
|
||||
*/
|
||||
inputDistribution?: Distribution<Transform>;
|
||||
|
||||
// The transform ID in play.
|
||||
private _transitionId?: number;
|
||||
|
||||
/**
|
||||
* Indicates the portion of the incoming keystroke data, if any, that applies to
|
||||
* tokens before the last pre-caret token and thus should not be replaced by predictions
|
||||
* based upon `state`. If the provided context state + the incoming transform do not
|
||||
* adequately match the current context, the match attempt will fail with a `null` result.
|
||||
*
|
||||
* Should generally be non-null if the token before the caret did not previously exist.
|
||||
*
|
||||
* The result may be null if it does not match the prior context state or if bookkeeping
|
||||
* based upon it is problematic - say, if wordbreaking effects shift due to new input,
|
||||
* causing a mismatch with the prior state's tokenization.
|
||||
* (Refer to #12494 for an example case.)
|
||||
*/
|
||||
preservationTransform?: Transform;
|
||||
|
||||
/**
|
||||
* Constructs a partial context transition object for use during the process
|
||||
* of analyzing context transitions or for representing the base state of a
|
||||
* reset context.
|
||||
* @param context The base state for the represented context transition
|
||||
* @param transitionId The unique ID corresponding to the transition event
|
||||
* or context state.
|
||||
*/
|
||||
constructor(context: ContextState, transitionId: number);
|
||||
/**
|
||||
* Deep-copies a ContextTransition instance.
|
||||
* @param baseTransition
|
||||
*/
|
||||
constructor(baseTransition: ContextTransition);
|
||||
constructor(param: ContextState | ContextTransition, transitionId?: number) {
|
||||
if(!(param instanceof ContextTransition)) {
|
||||
const contextState = param;
|
||||
// We're initializing a ContextTransition from a blank or reset context.
|
||||
this.base = contextState;
|
||||
this._final = null;
|
||||
this._transitionId = transitionId;
|
||||
} else {
|
||||
const baseTransition = param;
|
||||
Object.assign(this, baseTransition);
|
||||
|
||||
// These need to be deep-copied.
|
||||
this.base = new ContextState(baseTransition.base);
|
||||
this._final = new ContextState(baseTransition._final);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the context state resulting from the context transition event,
|
||||
* including any generated suggestions and data regarding potential
|
||||
* application thereof.
|
||||
*/
|
||||
get final(): ContextState {
|
||||
return this._final;
|
||||
}
|
||||
|
||||
/**
|
||||
* The unique ID corresponding to the transition event or context state.
|
||||
*/
|
||||
get transitionId(): number {
|
||||
return this._transitionId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Records the context state resulting from the context transition generated
|
||||
* by a keystroke.
|
||||
* @param state The context state to record as the result of the transition
|
||||
* @param inputDistribution Fat-finger data corresponding to the triggering keystroke
|
||||
* @param preservationTransform Portions of the most likely input that do not contribute to the final token
|
||||
* in the final context's tokenization.
|
||||
*/
|
||||
finalize(state: ContextState, inputDistribution: Distribution<Transform>, preservationTransform?: Transform) {
|
||||
this._final = state;
|
||||
this.inputDistribution = inputDistribution;
|
||||
// Long-term, this should never be null... but we need to allow it at this point
|
||||
// in the refactoring process.
|
||||
this._transitionId = inputDistribution?.find((entry) => entry.sample.id !== undefined)?.sample.id;
|
||||
this.preservationTransform = preservationTransform;
|
||||
}
|
||||
}
|
||||
|
|
@ -255,7 +255,7 @@ export class ModelCompositor {
|
|||
if(this.contextTracker) {
|
||||
let contextState = this.contextTracker.newest;
|
||||
if(!contextState) {
|
||||
contextState = this.contextTracker.analyzeState(this.lexicalModel, context).state;
|
||||
contextState = this.contextTracker.analyzeState(this.lexicalModel, context).final;
|
||||
}
|
||||
|
||||
contextState.tokenization.tail.appliedSuggestionId = suggestion.id;
|
||||
|
|
|
|||
|
|
@ -241,11 +241,11 @@ export async function correctAndEnumerate(
|
|||
// facilitates a more thorough correction-search pattern.
|
||||
|
||||
// Token replacement benefits greatly from knowledge of the prior context state.
|
||||
let { state: contextState } = contextTracker.analyzeState(
|
||||
let contextState = contextTracker.analyzeState(
|
||||
lexicalModel,
|
||||
context,
|
||||
null
|
||||
);
|
||||
).final;
|
||||
|
||||
// Corrections and predictions are based upon the post-context state, though.
|
||||
const contextChangeAnalysis = contextTracker.analyzeState(
|
||||
|
|
@ -255,7 +255,7 @@ export async function correctAndEnumerate(
|
|||
? transformDistribution
|
||||
: null
|
||||
);
|
||||
const postContextState = contextChangeAnalysis.state;
|
||||
const postContextState = contextChangeAnalysis.final;
|
||||
|
||||
// TODO: Should we filter backspaces & whitespaces out of the transform distribution?
|
||||
// Ideally, the answer (in the future) will be no, but leaving it in right now may pose an issue.
|
||||
|
|
|
|||
|
|
@ -51,10 +51,15 @@ describe('ContextTracker', function() {
|
|||
|
||||
let baseState = new ContextState(existingContext, plainModel);
|
||||
let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
|
||||
assert.isNotNull(newContextMatch?.state);
|
||||
assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
assert.equal(newContextMatch.headTokensRemoved, 1);
|
||||
assert.equal(newContextMatch.tailTokensAdded, 0);
|
||||
assert.isNotNull(newContextMatch?.final);
|
||||
assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
|
||||
if(!newContextMatch.final.tokenization.alignment.canAlign) {
|
||||
// Done this way b/c TS can infer types correctly afterward.
|
||||
assert.fail("context alignment failed");
|
||||
}
|
||||
assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, -1);
|
||||
assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0);
|
||||
});
|
||||
|
||||
it("properly matches and aligns when lead token + following whitespace are removed", function() {
|
||||
|
|
@ -76,10 +81,15 @@ describe('ContextTracker', function() {
|
|||
|
||||
let baseState = new ContextState(existingContext, plainModel);
|
||||
let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
|
||||
assert.isNotNull(newContextMatch?.state);
|
||||
assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
assert.equal(newContextMatch.headTokensRemoved, 2);
|
||||
assert.equal(newContextMatch.tailTokensAdded, 0);
|
||||
assert.isNotNull(newContextMatch?.final);
|
||||
assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
|
||||
if(!newContextMatch.final.tokenization.alignment.canAlign) {
|
||||
// Done this way b/c TS can infer types correctly afterward.
|
||||
assert.fail("context alignment failed");
|
||||
}
|
||||
assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, -2);
|
||||
assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0);
|
||||
});
|
||||
|
||||
it("properly matches and aligns when final token is edited", function() {
|
||||
|
|
@ -96,10 +106,15 @@ describe('ContextTracker', function() {
|
|||
|
||||
let baseState = new ContextState(existingContext, plainModel);
|
||||
let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform));
|
||||
assert.isNotNull(newContextMatch?.state);
|
||||
assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
assert.equal(newContextMatch.headTokensRemoved, 0);
|
||||
assert.equal(newContextMatch.tailTokensAdded, 0);
|
||||
assert.isNotNull(newContextMatch?.final);
|
||||
assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
|
||||
if(!newContextMatch.final.tokenization.alignment.canAlign) {
|
||||
// Done this way b/c TS can infer types correctly afterward.
|
||||
assert.fail("context alignment failed");
|
||||
}
|
||||
assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0);
|
||||
assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 0);
|
||||
});
|
||||
|
||||
// Needs improved context-state management (due to 2x tokens)
|
||||
|
|
@ -117,17 +132,22 @@ describe('ContextTracker', function() {
|
|||
|
||||
let baseState = new ContextState(existingContext, plainModel);
|
||||
let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform));
|
||||
assert.isNotNull(newContextMatch?.state);
|
||||
assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
assert.isNotNull(newContextMatch?.final);
|
||||
assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
// We want to preserve the added whitespace when predicting a token that follows after it.
|
||||
assert.deepEqual(newContextMatch.preservationTransform, { insert: ' ', deleteLeft: 0 });
|
||||
|
||||
// The 'wordbreak' transform
|
||||
let state = newContextMatch?.state;
|
||||
let state = newContextMatch?.final;
|
||||
assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence);
|
||||
assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence);
|
||||
assert.equal(newContextMatch.headTokensRemoved, 0);
|
||||
assert.equal(newContextMatch.tailTokensAdded, 2);
|
||||
|
||||
if(!newContextMatch.final.tokenization.alignment.canAlign) {
|
||||
// Done this way b/c TS can infer types correctly afterward.
|
||||
assert.fail("context alignment failed");
|
||||
}
|
||||
assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0);
|
||||
assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 2);
|
||||
});
|
||||
|
||||
it("properly matches and aligns when a 'wordbreak' is removed via backspace", function() {
|
||||
|
|
@ -144,12 +164,15 @@ describe('ContextTracker', function() {
|
|||
|
||||
let baseState = new ContextState(existingContext, plainModel);
|
||||
let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform));
|
||||
assert.isOk(newContextMatch?.state);
|
||||
assert.deepEqual(newContextMatch?.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
assert.isOk(newContextMatch?.final);
|
||||
assert.deepEqual(newContextMatch?.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
|
||||
// The 'wordbreak' transform
|
||||
assert.equal(newContextMatch.headTokensRemoved, 0);
|
||||
assert.equal(newContextMatch.tailTokensAdded, -2);
|
||||
if(!newContextMatch.final.tokenization.alignment.canAlign) {
|
||||
// Done this way b/c TS can infer types correctly afterward.
|
||||
assert.fail("context alignment failed");
|
||||
}
|
||||
assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0);
|
||||
assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, -2);
|
||||
});
|
||||
|
||||
it("properly matches and aligns when an implied 'wordbreak' occurs (as when following \"'\")", function() {
|
||||
|
|
@ -166,17 +189,21 @@ describe('ContextTracker', function() {
|
|||
|
||||
let baseState = new ContextState(existingContext, plainModel);
|
||||
let newContextMatch = ContextTracker.attemptMatchContext(existingContext, plainModel, baseState, toWrapperDistribution(transform));
|
||||
assert.isNotNull(newContextMatch?.state);
|
||||
assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
assert.isNotNull(newContextMatch?.final);
|
||||
assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
assert.deepEqual(newContextMatch.preservationTransform, { insert: '', deleteLeft: 0 });
|
||||
|
||||
// The 'wordbreak' transform
|
||||
let state = newContextMatch.state;
|
||||
let state = newContextMatch.final;
|
||||
assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence);
|
||||
assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence);
|
||||
|
||||
assert.equal(newContextMatch.headTokensRemoved, 0);
|
||||
assert.equal(newContextMatch.tailTokensAdded, 1);
|
||||
if(!newContextMatch.final.tokenization.alignment.canAlign) {
|
||||
// Done this way b/c TS can infer types correctly afterward.
|
||||
assert.fail("context alignment failed");
|
||||
}
|
||||
assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0);
|
||||
assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 1);
|
||||
})
|
||||
|
||||
// Needs improved context-state management (due to 2x tokens)
|
||||
|
|
@ -199,18 +226,22 @@ describe('ContextTracker', function() {
|
|||
|
||||
let baseState = new ContextState(existingContext, plainModel);
|
||||
let newContextMatch = ContextTracker.attemptMatchContext(newContext, plainModel, baseState, toWrapperDistribution(transform));
|
||||
assert.isNotNull(newContextMatch?.state);
|
||||
assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
assert.isNotNull(newContextMatch?.final);
|
||||
assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
// We want to preserve the added whitespace when predicting a token that follows after it.
|
||||
assert.deepEqual(newContextMatch.preservationTransform, { insert: ' ', deleteLeft: 0 });
|
||||
|
||||
// The 'wordbreak' transform
|
||||
let state = newContextMatch.state;
|
||||
let state = newContextMatch.final;
|
||||
assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence);
|
||||
assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence);
|
||||
|
||||
assert.equal(newContextMatch.headTokensRemoved, 2);
|
||||
assert.equal(newContextMatch.tailTokensAdded, 2);
|
||||
if(!newContextMatch.final.tokenization.alignment.canAlign) {
|
||||
// Done this way b/c TS can infer types correctly afterward.
|
||||
assert.fail("context alignment failed");
|
||||
}
|
||||
assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, -2);
|
||||
assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 2);
|
||||
});
|
||||
|
||||
it("properly matches and aligns when initial token is modified AND a 'wordbreak' is added'", function() {
|
||||
|
|
@ -232,18 +263,22 @@ describe('ContextTracker', function() {
|
|||
baseState,
|
||||
[{sample: transform, p: 1}]
|
||||
);
|
||||
assert.isNotNull(newContextMatch?.state);
|
||||
assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
assert.isNotNull(newContextMatch?.final);
|
||||
assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
// We want to preserve all text preceding the new token when applying a suggestion.
|
||||
assert.deepEqual(newContextMatch.preservationTransform, { insert: 'd ', deleteLeft: 0});
|
||||
|
||||
// The 'wordbreak' transform
|
||||
let state = newContextMatch.state;
|
||||
let state = newContextMatch.final;
|
||||
assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence);
|
||||
assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence);
|
||||
|
||||
assert.equal(newContextMatch.headTokensRemoved, 0);
|
||||
assert.equal(newContextMatch.tailTokensAdded, 2);
|
||||
if(!newContextMatch.final.tokenization.alignment.canAlign) {
|
||||
// Done this way b/c TS can infer types correctly afterward.
|
||||
assert.fail("context alignment failed");
|
||||
}
|
||||
assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0);
|
||||
assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 2);
|
||||
});
|
||||
|
||||
it("properly matches and aligns when tail token is modified AND a 'wordbreak' is added'", function() {
|
||||
|
|
@ -265,18 +300,22 @@ describe('ContextTracker', function() {
|
|||
baseState,
|
||||
[{sample: transform, p: 1}]
|
||||
);
|
||||
assert.isNotNull(newContextMatch?.state);
|
||||
assert.deepEqual(newContextMatch.state.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
assert.isNotNull(newContextMatch?.final);
|
||||
assert.deepEqual(newContextMatch.final.tokenization.tokens.map(token => token.exampleInput), rawTokens);
|
||||
// We want to preserve all text preceding the new token when applying a suggestion.
|
||||
assert.deepEqual(newContextMatch.preservationTransform, { insert: 'tor ', deleteLeft: 0 });
|
||||
|
||||
// The 'wordbreak' transform
|
||||
let state = newContextMatch.state;
|
||||
let state = newContextMatch.final;
|
||||
assert.isNotEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 2].searchSpace.inputSequence);
|
||||
assert.isEmpty(state.tokenization.tokens[state.tokenization.tokens.length - 1].searchSpace.inputSequence);
|
||||
|
||||
assert.equal(newContextMatch.headTokensRemoved, 0);
|
||||
assert.equal(newContextMatch.tailTokensAdded, 2);
|
||||
if(!newContextMatch.final.tokenization.alignment.canAlign) {
|
||||
// Done this way b/c TS can infer types correctly afterward.
|
||||
assert.fail("context alignment failed");
|
||||
}
|
||||
assert.equal(newContextMatch.final.tokenization.alignment.leadTokenShift, 0);
|
||||
assert.equal(newContextMatch.final.tokenization.alignment.tailTokenShift, 2);
|
||||
});
|
||||
|
||||
it('rejects hard-to-handle case: tail token is split into three rather than two', function() {
|
||||
|
|
@ -377,12 +416,12 @@ describe('ContextTracker', function() {
|
|||
let compositor = new ModelCompositor(model);
|
||||
let baseContextMatch = compositor.contextTracker.analyzeState(model, baseContext);
|
||||
|
||||
baseContextMatch.state.tokenization.tail.suggestions = [ baseSuggestion ];
|
||||
baseContextMatch.final.tokenization.tail.suggestions = [ baseSuggestion ];
|
||||
|
||||
let reversion = compositor.acceptSuggestion(baseSuggestion, baseContext, postTransform);
|
||||
|
||||
// Actual test assertion - was the replacement tracked?
|
||||
assert.equal(baseContextMatch.state.tokenization.tail.appliedSuggestionId, baseSuggestion.id);
|
||||
assert.equal(baseContextMatch.final.tokenization.tail.appliedSuggestionId, baseSuggestion.id);
|
||||
assert.equal(reversion.id, -baseSuggestion.id);
|
||||
|
||||
// Next step - on the followup context, is the replacement still active?
|
||||
|
|
@ -390,10 +429,10 @@ describe('ContextTracker', function() {
|
|||
let postContextMatch = compositor.contextTracker.analyzeState(model, postContext);
|
||||
|
||||
// Penultimate token corresponds to whitespace, which does not have a 'raw' representation.
|
||||
assert.equal(postContextMatch.state.tokenization.tokens[postContextMatch.state.tokenization.tokens.length - 2].exampleInput, ' ');
|
||||
assert.equal(postContextMatch.final.tokenization.tokens[postContextMatch.final.tokenization.tokens.length - 2].exampleInput, ' ');
|
||||
|
||||
// Final token is empty (follows a wordbreak)
|
||||
assert.equal(postContextMatch.state.tokenization.tail.exampleInput, '');
|
||||
assert.equal(postContextMatch.final.tokenization.tail.exampleInput, '');
|
||||
});
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Reference in a new issue