mirror of
https://github.com/keymanapp/keyman.git
synced 2026-10-17 03:17:33 +00:00
refactor(web): move computeAlignment core to standalone method
Relates-to: #14480 This is being done to prepare for a fix to #14480, which is needed to better handle complex input transforms and tokenization scenarios. The easiest way to consistently index the tokenized transforms... is to perform context alignment on the results of transform application.
This commit is contained in:
Joshua Horton
parent
e794f34dc9
commit
625f734223
5 changed files with 715 additions and 752 deletions
|
|
@ -10,6 +10,47 @@
|
|||
|
||||
import { ClassicalDistanceCalculation, EditOperation } from "./classical-calculation.js";
|
||||
|
||||
/**
|
||||
* Represents token-count values resulting from an alignment attempt between two
|
||||
* different modeled context states.
|
||||
*/
|
||||
export type ContextStateAlignment = {
|
||||
/**
|
||||
* Denotes whether or not alignment is possible between two contexts.
|
||||
*/
|
||||
canAlign: false
|
||||
} | {
|
||||
/**
|
||||
* Denotes whether or not alignment is possible between two contexts.
|
||||
*/
|
||||
canAlign: true,
|
||||
/**
|
||||
* Notes the number of tokens added to the head of the 'incoming'/'new' context
|
||||
* of the contexts being aligned. If negative, the incoming context deleted
|
||||
* a token found in the 'original' / base context.
|
||||
*
|
||||
* For the alignment, [base context index] + leadTokenShift = [incoming context index].
|
||||
*/
|
||||
leadTokenShift: number,
|
||||
/**
|
||||
* The count of tokens perfectly aligned, with no need for edits, for two successfully-
|
||||
* alignable contexts.
|
||||
*/
|
||||
matchLength: number,
|
||||
/**
|
||||
* The count of tokens at the tail perfectly aligned (existing in both contexts) but
|
||||
* edited for two successfully-alignable contexts. These tokens directly follow those
|
||||
* that need no edits.
|
||||
*/
|
||||
tailEditLength: number,
|
||||
/**
|
||||
* The count of new tokens added at the end of the incoming context for two aligned contexts.
|
||||
* If negative, the incoming context deleted a previously-existing token from the original.
|
||||
*/
|
||||
tailTokenShift: number
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* Determines the proper 'last match' index for a tokenized sequence based on its edit path.
|
||||
*
|
||||
|
|
@ -141,4 +182,224 @@ export function isSubstitutionAlignable(
|
|||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines the alignment between a new, incoming tokenization source and the
|
||||
* tokenization modeled by the current instance.
|
||||
* @param tokenizationToMatch Raw strings corresponding to the tokenization of the original context
|
||||
* @param incomingTokenization Raw strings corresponding to the tokenization of the incoming context
|
||||
* @param isSliding Notes if the context window is full (and sliding-alignment is particularly needed)
|
||||
* @param noSubVerify When true, this disables inspection of 'substitute' transitions that avoids
|
||||
* wholesale replacement of the original token.
|
||||
* @returns Alignment data that details if and how the incoming tokenization aligns with
|
||||
* the tokenization modeled by this instance.
|
||||
*/
|
||||
export function computeAlignment(
|
||||
tokenizationToMatch: string[],
|
||||
incomingTokenization: string[],
|
||||
isSliding: boolean,
|
||||
noSubVerify?: boolean
|
||||
): ContextStateAlignment {
|
||||
// Inverted order, since 'match' existed before our new context.
|
||||
let mapping = ClassicalDistanceCalculation.computeDistance(
|
||||
tokenizationToMatch.map(value => ({key: value})),
|
||||
incomingTokenization.map(value => ({key: value})),
|
||||
// Diagonal width to consider must be at least 2, as adding a single
|
||||
// whitespace after a token tends to add two tokens: one for whitespace,
|
||||
// one for the empty token to follow it.
|
||||
3
|
||||
);
|
||||
|
||||
let editPath = mapping.editPath();
|
||||
// Special case: new context bootstrapping - first token often substitutes.
|
||||
// The text length is small enough that no words should be able to rotate out the start of the context.
|
||||
// Special handling needed in case of no 'match'; the rest of the method assumes at least one 'match'.
|
||||
if(editPath.length <= 3 && (editPath[0] == 'substitute' || editPath[0] == 'match')) {
|
||||
let matchCount = 0;
|
||||
let subCount = 0;
|
||||
for(let i = 0; i < editPath.length; i++) {
|
||||
if(editPath[i] == 'substitute') {
|
||||
subCount++;
|
||||
if(!noSubVerify && !isSubstitutionAlignable(incomingTokenization[i], tokenizationToMatch[i], true)) {
|
||||
return {
|
||||
canAlign: false
|
||||
};
|
||||
}
|
||||
} else if(editPath[i] == 'match') {
|
||||
// If a substitution is already recorded, treat the 'match' as a substitution.
|
||||
if(subCount > 0) {
|
||||
subCount++;
|
||||
} else {
|
||||
matchCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const insertCount = editPath.filter((entry) => entry == 'insert').length;
|
||||
const deleteCount = editPath.filter((entry) => entry == 'delete').length;
|
||||
|
||||
return {
|
||||
canAlign: true,
|
||||
matchLength: matchCount,
|
||||
leadTokenShift: 0,
|
||||
tailEditLength: subCount,
|
||||
tailTokenShift: insertCount - deleteCount
|
||||
}
|
||||
}
|
||||
|
||||
// From here on assumes that at least one 'match' exists on the path.
|
||||
// It all works great... once the context is long enough for at least one stable token.
|
||||
const firstMatch = editPath.indexOf('match');
|
||||
|
||||
if(firstMatch == -1) {
|
||||
// If there are no matches, there's no alignment.
|
||||
return {
|
||||
canAlign: false
|
||||
};
|
||||
}
|
||||
|
||||
// Transpositions are not allowed at the token level during context alignment.
|
||||
if(editPath.find((entry) => entry.indexOf('transpose') > -1)) {
|
||||
return {
|
||||
canAlign: false
|
||||
};
|
||||
}
|
||||
|
||||
const lastMatch = getEditPathLastMatch(editPath);
|
||||
|
||||
// Assertion: for a long context, the bulk of the edit path should be a
|
||||
// continuous block of 'match' entries. If there's anything else in
|
||||
// the middle, we have a context mismatch.
|
||||
if(lastMatch == -1) {
|
||||
return {
|
||||
canAlign: false
|
||||
};
|
||||
}
|
||||
|
||||
let matchLength = lastMatch - firstMatch + 1;
|
||||
let tailInsertLength = 0;
|
||||
let tailDeleteLength = 0;
|
||||
for(let i = lastMatch; i < editPath.length; i++) {
|
||||
if(editPath[i] == 'insert') {
|
||||
tailInsertLength++;
|
||||
} else if(editPath[i] == 'delete') {
|
||||
tailDeleteLength++;
|
||||
}
|
||||
}
|
||||
if(tailInsertLength > 0 && tailDeleteLength > 0) {
|
||||
// Something's gone weird if this happens; that should appear as a substitution instead.
|
||||
// Otherwise, we have a VERY niche edit scenario.
|
||||
return {
|
||||
canAlign: false
|
||||
};
|
||||
}
|
||||
const tailSubstituteLength = (editPath.length - 1 - lastMatch) - tailInsertLength - tailDeleteLength;
|
||||
|
||||
// If we have a perfect match with a pre-existing context, no mutations have
|
||||
// happened; we have a 100% perfect match.
|
||||
if(firstMatch == 0 && lastMatch == editPath.length - 1) {
|
||||
return {
|
||||
canAlign: true,
|
||||
leadTokenShift: 0,
|
||||
matchLength,
|
||||
tailEditLength: tailSubstituteLength,
|
||||
tailTokenShift: tailInsertLength - tailDeleteLength
|
||||
};
|
||||
}
|
||||
|
||||
// The edit path calc tries to put substitutes first, before inserts.
|
||||
// We don't want that on the leading edge.
|
||||
const lastEarlyInsert = editPath.lastIndexOf('insert', firstMatch);
|
||||
const firstSubstitute = editPath.indexOf('substitute');
|
||||
if(firstSubstitute > -1 && firstSubstitute < firstMatch && firstSubstitute < lastEarlyInsert) {
|
||||
editPath[firstSubstitute] = 'insert';
|
||||
editPath[lastEarlyInsert] = 'substitute';
|
||||
}
|
||||
|
||||
// If mutations HAVE happened, we need to double-check the context-state alignment.
|
||||
let priorEdit: typeof editPath[0];
|
||||
let leadTokensRemoved = 0;
|
||||
let leadSubstitutions = 0;
|
||||
|
||||
// The `i` index below aligns based upon the index within the `tokenizationToMatch` sequence
|
||||
// and how it would have to be edited to align to the `incomingTokenization` sequence.
|
||||
for(let i = 0; i < firstMatch; i++) {
|
||||
switch(editPath[i]) {
|
||||
case 'delete':
|
||||
// All deletions should appear at the sliding window edge; if a deletion appears
|
||||
// after the edge, but before the first match, something's wrong.
|
||||
if(priorEdit && priorEdit != 'delete') {
|
||||
return {
|
||||
canAlign: false
|
||||
};
|
||||
}
|
||||
leadTokensRemoved++;
|
||||
break;
|
||||
case 'substitute':
|
||||
// We only allow for one leading token to be substituted.
|
||||
//
|
||||
// Any extras in the front would be pure inserts, not substitutions, due to
|
||||
// the sliding context window and its implications.
|
||||
if(leadSubstitutions++ > 0) {
|
||||
return {
|
||||
canAlign: false
|
||||
};
|
||||
}
|
||||
|
||||
// Find the word before and after substitution.
|
||||
const incomingIndex = i - (leadTokensRemoved > 0 ? leadTokensRemoved : 0);
|
||||
const matchingIndex = i + (leadTokensRemoved < 0 ? leadTokensRemoved : 0);
|
||||
const incomingSub = incomingTokenization[incomingIndex];
|
||||
const matchingSub = tokenizationToMatch[matchingIndex];
|
||||
|
||||
const atSlidePoint = isSliding && (incomingIndex == 0 || matchingIndex == 0);
|
||||
|
||||
// Double-check the word - does the 'substituted' word itself align?
|
||||
//
|
||||
// Exception: if the word is at the start of the context window and the
|
||||
// context window is likely sliding, don't check it.
|
||||
if(!noSubVerify && !atSlidePoint && !isSubstitutionAlignable(incomingSub, matchingSub)) {
|
||||
return {
|
||||
canAlign: false
|
||||
};
|
||||
}
|
||||
|
||||
// There's no major need to drop parts of a token being 'slid' out of the context window.
|
||||
// We'll leave it intact and treat it as a 'match'
|
||||
matchLength++;
|
||||
break;
|
||||
case 'insert':
|
||||
// Only allow an insert at the leading edge, as with 'delete's.
|
||||
if(priorEdit && priorEdit != 'insert') {
|
||||
return {
|
||||
canAlign: false
|
||||
};
|
||||
}
|
||||
// In case of backspaces, it's also possible to 'insert' a 'new'
|
||||
// token - an old one that's slid back into view.
|
||||
leadTokensRemoved--;
|
||||
break;
|
||||
default:
|
||||
// No 'match' can exist before the first found index for a 'match'.
|
||||
// No 'transpose-' edits should exist within this section, either.
|
||||
return {
|
||||
canAlign: false
|
||||
};
|
||||
}
|
||||
priorEdit = editPath[i];
|
||||
}
|
||||
|
||||
// If we need some form of tail-token substitution verification, add that here.
|
||||
|
||||
return {
|
||||
canAlign: true,
|
||||
// leadTokensRemoved represents the number of tokens that must be removed from the base context
|
||||
// when aligning the contexts. Externally, it's more helpful to think in terms of the count added
|
||||
// to the incoming context.
|
||||
leadTokenShift: -leadTokensRemoved + 0, // add 0 in case of a 'negative zero', which affects unit tests.
|
||||
matchLength,
|
||||
tailEditLength: tailSubstituteLength,
|
||||
tailTokenShift: tailInsertLength - tailDeleteLength
|
||||
};
|
||||
}
|
||||
|
|
@ -8,8 +8,7 @@
|
|||
*/
|
||||
|
||||
import { ContextToken } from './context-token.js';
|
||||
import { ClassicalDistanceCalculation } from './classical-calculation.js';
|
||||
import { getEditPathLastMatch, isSubstitutionAlignable } from './alignment-helpers.js';
|
||||
import { computeAlignment, ContextStateAlignment } from './alignment-helpers.js';
|
||||
import { Token } from '@keymanapp/models-templates';
|
||||
|
||||
import { LexicalModelTypes } from '@keymanapp/common-types';
|
||||
|
|
@ -18,46 +17,6 @@ import LexicalModel = LexicalModelTypes.LexicalModel;
|
|||
import Transform = LexicalModelTypes.Transform;
|
||||
import TransformUtils from '../transformUtils.js';
|
||||
|
||||
/**
|
||||
* Represents token-count values resulting from an alignment attempt between two
|
||||
* different modeled context states.
|
||||
*/
|
||||
export type ContextStateAlignment = {
|
||||
/**
|
||||
* Denotes whether or not alignment is possible between two contexts.
|
||||
*/
|
||||
canAlign: false
|
||||
} | {
|
||||
/**
|
||||
* Denotes whether or not alignment is possible between two contexts.
|
||||
*/
|
||||
canAlign: true,
|
||||
/**
|
||||
* Notes the number of tokens added to the head of the 'incoming'/'new' context
|
||||
* of the contexts being aligned. If negative, the incoming context deleted
|
||||
* a token found in the 'original' / base context.
|
||||
*
|
||||
* For the alignment, [base context index] + leadTokenShift = [incoming context index].
|
||||
*/
|
||||
leadTokenShift: number,
|
||||
/**
|
||||
* The count of tokens perfectly aligned, with no need for edits, for two successfully-
|
||||
* alignable contexts.
|
||||
*/
|
||||
matchLength: number,
|
||||
/**
|
||||
* The count of tokens at the tail perfectly aligned (existing in both contexts) but
|
||||
* edited for two successfully-alignable contexts. These tokens directly follow those
|
||||
* that need no edits.
|
||||
*/
|
||||
tailEditLength: number,
|
||||
/**
|
||||
* The count of new tokens added at the end of the incoming context for two aligned contexts.
|
||||
* If negative, the incoming context deleted a previously-existing token from the original.
|
||||
*/
|
||||
tailTokenShift: number
|
||||
};
|
||||
|
||||
/**
|
||||
* This class represents the sequence of tokens (words and whitespace blocks)
|
||||
* held within the active sliding context-window at a single point in time.
|
||||
|
|
@ -109,210 +68,7 @@ export class ContextTokenization {
|
|||
* the tokenization modeled by this instance.
|
||||
*/
|
||||
computeAlignment(incomingTokenization: string[], isSliding: boolean, noSubVerify?: boolean): ContextStateAlignment {
|
||||
// Map the tokenized state to an edit-distance friendly version.
|
||||
const tokenizationToMatch = this.exampleInput;
|
||||
|
||||
// Inverted order, since 'match' existed before our new context.
|
||||
let mapping = ClassicalDistanceCalculation.computeDistance(
|
||||
tokenizationToMatch.map(value => ({key: value})),
|
||||
incomingTokenization.map(value => ({key: value})),
|
||||
// Diagonal width to consider must be at least 2, as adding a single
|
||||
// whitespace after a token tends to add two tokens: one for whitespace,
|
||||
// one for the empty token to follow it.
|
||||
3
|
||||
);
|
||||
|
||||
let editPath = mapping.editPath();
|
||||
// Special case: new context bootstrapping - first token often substitutes.
|
||||
// The text length is small enough that no words should be able to rotate out the start of the context.
|
||||
// Special handling needed in case of no 'match'; the rest of the method assumes at least one 'match'.
|
||||
if(editPath.length <= 3 && (editPath[0] == 'substitute' || editPath[0] == 'match')) {
|
||||
let matchCount = 0;
|
||||
let subCount = 0;
|
||||
for(let i = 0; i < editPath.length; i++) {
|
||||
if(editPath[i] == 'substitute') {
|
||||
subCount++;
|
||||
if(!noSubVerify && !isSubstitutionAlignable(incomingTokenization[i], tokenizationToMatch[i], true)) {
|
||||
return {
|
||||
canAlign: false
|
||||
};
|
||||
}
|
||||
} else if(editPath[i] == 'match') {
|
||||
// If a substitution is already recorded, treat the 'match' as a substitution.
|
||||
if(subCount > 0) {
|
||||
subCount++;
|
||||
} else {
|
||||
matchCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const insertCount = editPath.filter((entry) => entry == 'insert').length;
|
||||
const deleteCount = editPath.filter((entry) => entry == 'delete').length;
|
||||
|
||||
return {
|
||||
canAlign: true,
|
||||
matchLength: matchCount,
|
||||
leadTokenShift: 0,
|
||||
tailEditLength: subCount,
|
||||
tailTokenShift: insertCount - deleteCount
|
||||
}
|
||||
}
|
||||
|
||||
// From here on assumes that at least one 'match' exists on the path.
|
||||
// It all works great... once the context is long enough for at least one stable token.
|
||||
const firstMatch = editPath.indexOf('match');
|
||||
|
||||
if(firstMatch == -1) {
|
||||
// If there are no matches, there's no alignment.
|
||||
return {
|
||||
canAlign: false
|
||||
};
|
||||
}
|
||||
|
||||
// Transpositions are not allowed at the token level during context alignment.
|
||||
if(editPath.find((entry) => entry.indexOf('transpose') > -1)) {
|
||||
return {
|
||||
canAlign: false
|
||||
};
|
||||
}
|
||||
|
||||
const lastMatch = getEditPathLastMatch(editPath);
|
||||
|
||||
// Assertion: for a long context, the bulk of the edit path should be a
|
||||
// continuous block of 'match' entries. If there's anything else in
|
||||
// the middle, we have a context mismatch.
|
||||
if(lastMatch == -1) {
|
||||
return {
|
||||
canAlign: false
|
||||
};
|
||||
}
|
||||
|
||||
let matchLength = lastMatch - firstMatch + 1;
|
||||
let tailInsertLength = 0;
|
||||
let tailDeleteLength = 0;
|
||||
for(let i = lastMatch; i < editPath.length; i++) {
|
||||
if(editPath[i] == 'insert') {
|
||||
tailInsertLength++;
|
||||
} else if(editPath[i] == 'delete') {
|
||||
tailDeleteLength++;
|
||||
}
|
||||
}
|
||||
if(tailInsertLength > 0 && tailDeleteLength > 0) {
|
||||
// Something's gone weird if this happens; that should appear as a substitution instead.
|
||||
// Otherwise, we have a VERY niche edit scenario.
|
||||
return {
|
||||
canAlign: false
|
||||
};
|
||||
}
|
||||
const tailSubstituteLength = (editPath.length - 1 - lastMatch) - tailInsertLength - tailDeleteLength;
|
||||
|
||||
// If we have a perfect match with a pre-existing context, no mutations have
|
||||
// happened; we have a 100% perfect match.
|
||||
if(firstMatch == 0 && lastMatch == editPath.length - 1) {
|
||||
return {
|
||||
canAlign: true,
|
||||
leadTokenShift: 0,
|
||||
matchLength,
|
||||
tailEditLength: tailSubstituteLength,
|
||||
tailTokenShift: tailInsertLength - tailDeleteLength
|
||||
};
|
||||
}
|
||||
|
||||
// The edit path calc tries to put substitutes first, before inserts.
|
||||
// We don't want that on the leading edge.
|
||||
const lastEarlyInsert = editPath.lastIndexOf('insert', firstMatch);
|
||||
const firstSubstitute = editPath.indexOf('substitute');
|
||||
if(firstSubstitute > -1 && firstSubstitute < firstMatch && firstSubstitute < lastEarlyInsert) {
|
||||
editPath[firstSubstitute] = 'insert';
|
||||
editPath[lastEarlyInsert] = 'substitute';
|
||||
}
|
||||
|
||||
// If mutations HAVE happened, we need to double-check the context-state alignment.
|
||||
let priorEdit: typeof editPath[0];
|
||||
let leadTokensRemoved = 0;
|
||||
let leadSubstitutions = 0;
|
||||
|
||||
// The `i` index below aligns based upon the index within the `tokenizationToMatch` sequence
|
||||
// and how it would have to be edited to align to the `incomingTokenization` sequence.
|
||||
for(let i = 0; i < firstMatch; i++) {
|
||||
switch(editPath[i]) {
|
||||
case 'delete':
|
||||
// All deletions should appear at the sliding window edge; if a deletion appears
|
||||
// after the edge, but before the first match, something's wrong.
|
||||
if(priorEdit && priorEdit != 'delete') {
|
||||
return {
|
||||
canAlign: false
|
||||
};
|
||||
}
|
||||
leadTokensRemoved++;
|
||||
break;
|
||||
case 'substitute':
|
||||
// We only allow for one leading token to be substituted.
|
||||
//
|
||||
// Any extras in the front would be pure inserts, not substitutions, due to
|
||||
// the sliding context window and its implications.
|
||||
if(leadSubstitutions++ > 0) {
|
||||
return {
|
||||
canAlign: false
|
||||
};
|
||||
}
|
||||
|
||||
// Find the word before and after substitution.
|
||||
const incomingIndex = i - (leadTokensRemoved > 0 ? leadTokensRemoved : 0);
|
||||
const matchingIndex = i + (leadTokensRemoved < 0 ? leadTokensRemoved : 0);
|
||||
const incomingSub = incomingTokenization[incomingIndex];
|
||||
const matchingSub = tokenizationToMatch[matchingIndex];
|
||||
|
||||
const atSlidePoint = isSliding && (incomingIndex == 0 || matchingIndex == 0);
|
||||
|
||||
// Double-check the word - does the 'substituted' word itself align?
|
||||
//
|
||||
// Exception: if the word is at the start of the context window and the
|
||||
// context window is likely sliding, don't check it.
|
||||
if(!noSubVerify && !atSlidePoint && !isSubstitutionAlignable(incomingSub, matchingSub)) {
|
||||
return {
|
||||
canAlign: false
|
||||
};
|
||||
}
|
||||
|
||||
// There's no major need to drop parts of a token being 'slid' out of the context window.
|
||||
// We'll leave it intact and treat it as a 'match'
|
||||
matchLength++;
|
||||
break;
|
||||
case 'insert':
|
||||
// Only allow an insert at the leading edge, as with 'delete's.
|
||||
if(priorEdit && priorEdit != 'insert') {
|
||||
return {
|
||||
canAlign: false
|
||||
};
|
||||
}
|
||||
// In case of backspaces, it's also possible to 'insert' a 'new'
|
||||
// token - an old one that's slid back into view.
|
||||
leadTokensRemoved--;
|
||||
break;
|
||||
default:
|
||||
// No 'match' can exist before the first found index for a 'match'.
|
||||
// No 'transpose-' edits should exist within this section, either.
|
||||
return {
|
||||
canAlign: false
|
||||
};
|
||||
}
|
||||
priorEdit = editPath[i];
|
||||
}
|
||||
|
||||
// If we need some form of tail-token substitution verification, add that here.
|
||||
|
||||
return {
|
||||
canAlign: true,
|
||||
// leadTokensRemoved represents the number of tokens that must be removed from the base context
|
||||
// when aligning the contexts. Externally, it's more helpful to think in terms of the count added
|
||||
// to the incoming context.
|
||||
leadTokenShift: -leadTokensRemoved + 0, // add 0 in case of a 'negative zero', which affects unit tests.
|
||||
matchLength,
|
||||
tailEditLength: tailSubstituteLength,
|
||||
tailTokenShift: tailInsertLength - tailDeleteLength
|
||||
};
|
||||
return computeAlignment(this.exampleInput, incomingTokenization, isSliding, noSubVerify);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ export * as correction from './correction/index.js';
|
|||
export * from './model-helpers.js';
|
||||
export * as models from './models/index.js';
|
||||
export { ModelCompositor } from './model-compositor.js';
|
||||
export { tokenizeTransform, tokenizeTransformDistribution } from './correction/transform-tokenization.js';
|
||||
export * from './correction/transform-tokenization.js';
|
||||
export * from './predict-helpers.js';
|
||||
export { default as TransformUtils } from './transformUtils.js'
|
||||
export { default as LMLayerWorker } from './index.js'
|
||||
|
|
@ -9,7 +9,7 @@
|
|||
*/
|
||||
|
||||
import { assert } from 'chai';
|
||||
import { EditOperation, getEditPathLastMatch, isSubstitutionAlignable } from '@keymanapp/lm-worker/test-index';
|
||||
import { computeAlignment, EditOperation, getEditPathLastMatch, isSubstitutionAlignable } from '@keymanapp/lm-worker/test-index';
|
||||
|
||||
describe('getEditPathLastMatch', () => {
|
||||
it('returns the last match when no substitutions exist', () => {
|
||||
|
|
@ -116,4 +116,454 @@ describe('isSubstitutionAlignable', () => {
|
|||
// The double-p adds a fun complication once the first gets dropped.
|
||||
assert.isTrue(isSubstitutionAlignable('applesauce', 'plesauce'));
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
describe('computeAlignment', () => {
|
||||
it("properly matches and aligns when contexts match", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [...baseContext];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: 0,
|
||||
matchLength: 5,
|
||||
tailEditLength: 0,
|
||||
tailTokenShift: 0
|
||||
});
|
||||
});
|
||||
|
||||
it("properly matches and aligns with applied-suggestion contexts", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'o'
|
||||
];
|
||||
const newContext = [...baseContext];
|
||||
newContext[4] = 'over';
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: 0,
|
||||
matchLength: 4,
|
||||
tailEditLength: 1,
|
||||
tailTokenShift: 0
|
||||
});
|
||||
});
|
||||
|
||||
it("properly matches and aligns with applied-suggestion at start of context", () => {
|
||||
const baseContext = [
|
||||
'te'
|
||||
];
|
||||
const newContext = [
|
||||
'testing',
|
||||
' ',
|
||||
''
|
||||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false, true);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: 0,
|
||||
matchLength: 0,
|
||||
tailEditLength: 1,
|
||||
tailTokenShift: 2
|
||||
});
|
||||
});
|
||||
|
||||
it("detects unalignable contexts - no matching tokens", () => {
|
||||
const baseContext = [
|
||||
'swift', 'tan', 'wolf', 'leaped', 'across'
|
||||
];
|
||||
const newContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {canAlign: false});
|
||||
});
|
||||
|
||||
it("detects unalignable contexts - too many mismatching tokens", () => {
|
||||
const baseContext = [
|
||||
'swift', 'tan', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {canAlign: false});
|
||||
});
|
||||
|
||||
it("fails alignment for leading-edge word substitutions", () => {
|
||||
const baseContext = [
|
||||
'swift', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {canAlign: false});
|
||||
});
|
||||
|
||||
it("fails alignment for small leading-edge word substitutions", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'sick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {canAlign: false});
|
||||
});
|
||||
|
||||
it("properly matches and aligns when lead token is modified", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'uick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: 0,
|
||||
matchLength: 5,
|
||||
tailEditLength: 0,
|
||||
tailTokenShift: 0
|
||||
});
|
||||
});
|
||||
|
||||
it("properly matches and aligns when lead token is removed", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: -1,
|
||||
matchLength: 4,
|
||||
tailEditLength: 0,
|
||||
tailTokenShift: 0
|
||||
});
|
||||
});
|
||||
|
||||
it("properly matches and aligns when lead token is added", () => {
|
||||
const baseContext = [
|
||||
'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: 1,
|
||||
matchLength: 4,
|
||||
tailEditLength: 0,
|
||||
tailTokenShift: 0
|
||||
});
|
||||
});
|
||||
|
||||
it("properly matches and aligns when lead tokens are removed and modified", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'ox', 'jumped', 'over'
|
||||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: -2,
|
||||
matchLength: 3,
|
||||
tailEditLength: 0,
|
||||
tailTokenShift: 0
|
||||
});
|
||||
});
|
||||
|
||||
it("properly matches and aligns when lead tokens are added and modified", () => {
|
||||
const baseContext = [
|
||||
'rown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: 1,
|
||||
matchLength: 4,
|
||||
tailEditLength: 0,
|
||||
tailTokenShift: 0
|
||||
});
|
||||
});
|
||||
|
||||
it("properly matches and aligns when lead token is removed and tail token is added", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'brown', 'fox', 'jumped', 'over', 'the'
|
||||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: -1,
|
||||
matchLength: 4,
|
||||
tailEditLength: 0,
|
||||
tailTokenShift: 1
|
||||
});
|
||||
});
|
||||
|
||||
it("properly matches and aligns when lead token and tail token are modified", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'ove'
|
||||
];
|
||||
const newContext = [
|
||||
'uick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: 0,
|
||||
matchLength: 4, // we treat 'quick' and 'uick' as the same
|
||||
tailEditLength: 1,
|
||||
tailTokenShift: 0
|
||||
});
|
||||
});
|
||||
|
||||
it("properly matches and aligns when lead token and tail token are modified + new token appended", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'ove'
|
||||
];
|
||||
const newContext = [
|
||||
'uick', 'brown', 'fox', 'jumped', 'over', 't'
|
||||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: 0,
|
||||
matchLength: 4, // we treat 'quick' and 'uick' as the same
|
||||
tailEditLength: 1,
|
||||
tailTokenShift: 1
|
||||
});
|
||||
});
|
||||
|
||||
it("properly handles context window sliding backward", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'e', 'quick', 'brown', 'fox', 'jumped', 'ove'
|
||||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: 1,
|
||||
matchLength: 4, // we treat 'quick' and 'uick' as the same
|
||||
tailEditLength: 1,
|
||||
tailTokenShift: 0
|
||||
});
|
||||
});
|
||||
|
||||
it("properly handles context window sliding far backward", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'the', 'quick', 'brown', 'fox', 'jumped'
|
||||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: 1,
|
||||
matchLength: 4, // we treat 'quick' and 'uick' as the same
|
||||
tailEditLength: 0,
|
||||
tailTokenShift: -1
|
||||
});
|
||||
});
|
||||
|
||||
it("properly handles context window sliding farther backward", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'the', 'quick', 'brown', 'fox', 'jumpe'
|
||||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: 1,
|
||||
matchLength: 3, // we treat 'quick' and 'uick' as the same
|
||||
tailEditLength: 1,
|
||||
tailTokenShift: -1
|
||||
});
|
||||
});
|
||||
|
||||
it("fails alignment for mid-head deletion", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'quick', 'fox', 'jumped', 'over'
|
||||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {canAlign: false});
|
||||
});
|
||||
|
||||
it("fails alignment for mid-head insertion", () => {
|
||||
const baseContext = [
|
||||
'quick', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {canAlign: false});
|
||||
});
|
||||
|
||||
it("fails alignment for mid-tail deletion", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'quick', 'brown', 'fox', 'over'
|
||||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {canAlign: false});
|
||||
});
|
||||
|
||||
it("fails alignment for mid-tail insertion", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'far', 'over'
|
||||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {canAlign: false});
|
||||
});
|
||||
|
||||
it("handles late-context suggestion application after backspace", () => {
|
||||
const baseContext = [
|
||||
'quick', ' ', 'brown', ' ', 'fox', ' ', 'jumped', ' ', 'oven', ' ', ''
|
||||
];
|
||||
const newContext = [
|
||||
'quick', ' ', 'brown', ' ', 'fox', ' ', 'jumped', ' ', 'over', ' ', ''
|
||||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: 0,
|
||||
matchLength: 8,
|
||||
tailEditLength: 3,
|
||||
tailTokenShift: 0
|
||||
});
|
||||
});
|
||||
|
||||
it("handles sliding context-window scenarios", () => {
|
||||
// // Explicitly-defined window, though it's not needed directly by the method.
|
||||
// const config = {
|
||||
// leftContextCodePoints: 64,
|
||||
// rightContextCodePoints: 64
|
||||
// };
|
||||
|
||||
const baseContext1 = [
|
||||
// "ap" prefix not in actual view, but preserved by prior tokenization rounds.
|
||||
"applesauce", " ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ",
|
||||
"like", " ", "they'd", " ", "make", " ", "for", " ", "the", " ", "be"
|
||||
];
|
||||
|
||||
const incomingContext1 = [
|
||||
"plesauce", " ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ",
|
||||
"like", " ", "they'd", " ", "make", " ", "for", " ", "the", " ", "bes"
|
||||
];
|
||||
|
||||
// 66 chars above, vs a sliding window of length 64.
|
||||
assert.equal(baseContext1.reduce((accum, curr) => accum + curr.length, 0), 66);
|
||||
// Actual window + one newly-typed character
|
||||
assert.equal(incomingContext1.reduce((accum, curr) => accum + curr.length, 0), 65);
|
||||
|
||||
assert.deepEqual(computeAlignment(baseContext1, incomingContext1, true), {
|
||||
canAlign: true,
|
||||
leadTokenShift: 0,
|
||||
matchLength: 22,
|
||||
tailEditLength: 1,
|
||||
tailTokenShift: 0
|
||||
});
|
||||
|
||||
// Our tokenization scheme remembers the full original word before any of it slid out of
|
||||
// the context window.
|
||||
const baseContext2 = [
|
||||
"applesauce", " ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ",
|
||||
// +2 +1 +4
|
||||
"like", " ", "they'd", " ", "make", " ", "for", " ", "the", " ", "best", " ", "brea"
|
||||
];
|
||||
|
||||
const incomingContext2 = [
|
||||
// "plesauce" => "e": -7 chars.
|
||||
"e", " ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ",
|
||||
"like", " ", "they'd", " ", "make", " ", "for", " ", "the", " ", "best", " ", "break"
|
||||
];
|
||||
|
||||
// 73 chars above, vs a sliding window of length 64.
|
||||
assert.equal(baseContext2.reduce((accum, curr) => accum + curr.length, 0), 73);
|
||||
// Actual window + one newly-typed character
|
||||
assert.equal(incomingContext2.reduce((accum, curr) => accum + curr.length, 0), 65);
|
||||
|
||||
assert.deepEqual(computeAlignment(baseContext2, incomingContext2, true), {
|
||||
canAlign: true,
|
||||
leadTokenShift: 0,
|
||||
matchLength: 24,
|
||||
tailEditLength: 1,
|
||||
tailTokenShift: 0
|
||||
});
|
||||
|
||||
const baseContext3 = [
|
||||
"applesauce", " ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ", "like", " ",
|
||||
"they'd", " ", "make", " ", "for", " ", "the", " ", "best", " ", "break"
|
||||
];
|
||||
|
||||
const incomingContext3 = [
|
||||
" ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ", "like", " ",
|
||||
"they'd", " ", "make", " ", "for", " ", "the", " ", "best", " ", "breakf"
|
||||
];
|
||||
|
||||
// 74 chars above, vs a sliding window of length 64.
|
||||
assert.equal(baseContext3.reduce((accum, curr) => accum + curr.length, 0), 74);
|
||||
// Actual window + one newly-typed character
|
||||
assert.equal(incomingContext3.reduce((accum, curr) => accum + curr.length, 0), 65);
|
||||
|
||||
assert.deepEqual(computeAlignment(baseContext3, incomingContext3, true), {
|
||||
canAlign: true,
|
||||
leadTokenShift: -1,
|
||||
matchLength: 23,
|
||||
tailEditLength: 1,
|
||||
tailTokenShift: 0
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -20,11 +20,6 @@ import TrieModel = models.TrieModel;
|
|||
var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'),
|
||||
{wordBreaker: defaultBreaker});
|
||||
|
||||
function buildBaseTokenization(textTokens: string[]) {
|
||||
const tokens = textTokens.map((entry) => toToken(entry));
|
||||
return new ContextTokenization(tokens);
|
||||
}
|
||||
|
||||
function toToken(text: string) {
|
||||
let isWhitespace = text == ' ';
|
||||
let token = new ContextToken(plainModel, text);
|
||||
|
|
@ -100,505 +95,6 @@ describe('ContextTokenization', function() {
|
|||
assert.deepEqual(tokenization.exampleInput, rawTextTokens);
|
||||
});
|
||||
|
||||
describe('computeAlignment', () => {
|
||||
it("properly matches and aligns when contexts match", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [...baseContext];
|
||||
|
||||
const baseTokenization = buildBaseTokenization(baseContext);
|
||||
const computedAlignment = baseTokenization.computeAlignment(newContext, false);
|
||||
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: 0,
|
||||
matchLength: 5,
|
||||
tailEditLength: 0,
|
||||
tailTokenShift: 0
|
||||
});
|
||||
});
|
||||
|
||||
it("properly matches and aligns with applied-suggestion contexts", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'o'
|
||||
];
|
||||
const newContext = [...baseContext];
|
||||
newContext[4] = 'over';
|
||||
|
||||
const baseTokenization = buildBaseTokenization(baseContext);
|
||||
const computedAlignment = baseTokenization.computeAlignment(newContext, false);
|
||||
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: 0,
|
||||
matchLength: 4,
|
||||
tailEditLength: 1,
|
||||
tailTokenShift: 0
|
||||
});
|
||||
});
|
||||
|
||||
it("properly matches and aligns with applied-suggestion at start of context", () => {
|
||||
const baseContext = [
|
||||
'te'
|
||||
];
|
||||
const newContext = [
|
||||
'testing',
|
||||
' ',
|
||||
''
|
||||
];
|
||||
|
||||
const baseTokenization = buildBaseTokenization(baseContext);
|
||||
const computedAlignment = baseTokenization.computeAlignment(newContext, false, true);
|
||||
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: 0,
|
||||
matchLength: 0,
|
||||
tailEditLength: 1,
|
||||
tailTokenShift: 2
|
||||
});
|
||||
});
|
||||
|
||||
it("detects unalignable contexts - no matching tokens", () => {
|
||||
const baseContext = [
|
||||
'swift', 'tan', 'wolf', 'leaped', 'across'
|
||||
];
|
||||
const newContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
|
||||
const baseTokenization = buildBaseTokenization(baseContext);
|
||||
const computedAlignment = baseTokenization.computeAlignment(newContext, false);
|
||||
|
||||
assert.deepEqual(computedAlignment, {canAlign: false});
|
||||
});
|
||||
|
||||
it("detects unalignable contexts - too many mismatching tokens", () => {
|
||||
const baseContext = [
|
||||
'swift', 'tan', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
|
||||
const baseTokenization = buildBaseTokenization(baseContext);
|
||||
const computedAlignment = baseTokenization.computeAlignment(newContext, false);
|
||||
|
||||
assert.deepEqual(computedAlignment, {canAlign: false});
|
||||
});
|
||||
|
||||
it("fails alignment for leading-edge word substitutions", () => {
|
||||
const baseContext = [
|
||||
'swift', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
|
||||
const baseTokenization = buildBaseTokenization(baseContext);
|
||||
const computedAlignment = baseTokenization.computeAlignment(newContext, false);
|
||||
|
||||
assert.deepEqual(computedAlignment, {canAlign: false});
|
||||
});
|
||||
|
||||
it("fails alignment for small leading-edge word substitutions", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'sick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
|
||||
const baseTokenization = buildBaseTokenization(baseContext);
|
||||
const computedAlignment = baseTokenization.computeAlignment(newContext, false);
|
||||
|
||||
assert.deepEqual(computedAlignment, {canAlign: false});
|
||||
});
|
||||
|
||||
it("properly matches and aligns when lead token is modified", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'uick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
|
||||
const baseTokenization = buildBaseTokenization(baseContext);
|
||||
const computedAlignment = baseTokenization.computeAlignment(newContext, false);
|
||||
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: 0,
|
||||
matchLength: 5,
|
||||
tailEditLength: 0,
|
||||
tailTokenShift: 0
|
||||
});
|
||||
});
|
||||
|
||||
it("properly matches and aligns when lead token is removed", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
|
||||
const baseTokenization = buildBaseTokenization(baseContext);
|
||||
const computedAlignment = baseTokenization.computeAlignment(newContext, false);
|
||||
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: -1,
|
||||
matchLength: 4,
|
||||
tailEditLength: 0,
|
||||
tailTokenShift: 0
|
||||
});
|
||||
});
|
||||
|
||||
it("properly matches and aligns when lead token is added", () => {
|
||||
const baseContext = [
|
||||
'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
|
||||
const baseTokenization = buildBaseTokenization(baseContext);
|
||||
const computedAlignment = baseTokenization.computeAlignment(newContext, false);
|
||||
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: 1,
|
||||
matchLength: 4,
|
||||
tailEditLength: 0,
|
||||
tailTokenShift: 0
|
||||
});
|
||||
});
|
||||
|
||||
it("properly matches and aligns when lead tokens are removed and modified", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'ox', 'jumped', 'over'
|
||||
];
|
||||
|
||||
const baseTokenization = buildBaseTokenization(baseContext);
|
||||
const computedAlignment = baseTokenization.computeAlignment(newContext, false);
|
||||
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: -2,
|
||||
matchLength: 3,
|
||||
tailEditLength: 0,
|
||||
tailTokenShift: 0
|
||||
});
|
||||
});
|
||||
|
||||
it("properly matches and aligns when lead tokens are added and modified", () => {
|
||||
const baseContext = [
|
||||
'rown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
|
||||
const baseTokenization = buildBaseTokenization(baseContext);
|
||||
const computedAlignment = baseTokenization.computeAlignment(newContext, false);
|
||||
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: 1,
|
||||
matchLength: 4,
|
||||
tailEditLength: 0,
|
||||
tailTokenShift: 0
|
||||
});
|
||||
});
|
||||
|
||||
it("properly matches and aligns when lead token is removed and tail token is added", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'brown', 'fox', 'jumped', 'over', 'the'
|
||||
];
|
||||
|
||||
const baseTokenization = buildBaseTokenization(baseContext);
|
||||
const computedAlignment = baseTokenization.computeAlignment(newContext, false);
|
||||
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: -1,
|
||||
matchLength: 4,
|
||||
tailEditLength: 0,
|
||||
tailTokenShift: 1
|
||||
});
|
||||
});
|
||||
|
||||
it("properly matches and aligns when lead token and tail token are modified", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'ove'
|
||||
];
|
||||
const newContext = [
|
||||
'uick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
|
||||
const baseTokenization = buildBaseTokenization(baseContext);
|
||||
const computedAlignment = baseTokenization.computeAlignment(newContext, false);
|
||||
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: 0,
|
||||
matchLength: 4, // we treat 'quick' and 'uick' as the same
|
||||
tailEditLength: 1,
|
||||
tailTokenShift: 0
|
||||
});
|
||||
});
|
||||
|
||||
it("properly matches and aligns when lead token and tail token are modified + new token appended", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'ove'
|
||||
];
|
||||
const newContext = [
|
||||
'uick', 'brown', 'fox', 'jumped', 'over', 't'
|
||||
];
|
||||
|
||||
const baseTokenization = buildBaseTokenization(baseContext);
|
||||
const computedAlignment = baseTokenization.computeAlignment(newContext, false);
|
||||
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: 0,
|
||||
matchLength: 4, // we treat 'quick' and 'uick' as the same
|
||||
tailEditLength: 1,
|
||||
tailTokenShift: 1
|
||||
});
|
||||
});
|
||||
|
||||
it("properly handles context window sliding backward", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'e', 'quick', 'brown', 'fox', 'jumped', 'ove'
|
||||
];
|
||||
|
||||
const baseTokenization = buildBaseTokenization(baseContext);
|
||||
const computedAlignment = baseTokenization.computeAlignment(newContext, false);
|
||||
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: 1,
|
||||
matchLength: 4, // we treat 'quick' and 'uick' as the same
|
||||
tailEditLength: 1,
|
||||
tailTokenShift: 0
|
||||
});
|
||||
});
|
||||
|
||||
it("properly handles context window sliding far backward", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'the', 'quick', 'brown', 'fox', 'jumped'
|
||||
];
|
||||
|
||||
const baseTokenization = buildBaseTokenization(baseContext);
|
||||
const computedAlignment = baseTokenization.computeAlignment(newContext, false);
|
||||
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: 1,
|
||||
matchLength: 4, // we treat 'quick' and 'uick' as the same
|
||||
tailEditLength: 0,
|
||||
tailTokenShift: -1
|
||||
});
|
||||
});
|
||||
|
||||
it("properly handles context window sliding farther backward", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'the', 'quick', 'brown', 'fox', 'jumpe'
|
||||
];
|
||||
|
||||
const baseTokenization = buildBaseTokenization(baseContext);
|
||||
const computedAlignment = baseTokenization.computeAlignment(newContext, false);
|
||||
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: 1,
|
||||
matchLength: 3, // we treat 'quick' and 'uick' as the same
|
||||
tailEditLength: 1,
|
||||
tailTokenShift: -1
|
||||
});
|
||||
});
|
||||
|
||||
it("fails alignment for mid-head deletion", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'quick', 'fox', 'jumped', 'over'
|
||||
];
|
||||
|
||||
const baseTokenization = buildBaseTokenization(baseContext);
|
||||
const computedAlignment = baseTokenization.computeAlignment(newContext, false);
|
||||
|
||||
assert.deepEqual(computedAlignment, {canAlign: false});
|
||||
});
|
||||
|
||||
it("fails alignment for mid-head insertion", () => {
|
||||
const baseContext = [
|
||||
'quick', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
|
||||
const baseTokenization = buildBaseTokenization(baseContext);
|
||||
const computedAlignment = baseTokenization.computeAlignment(newContext, false);
|
||||
|
||||
assert.deepEqual(computedAlignment, {canAlign: false});
|
||||
});
|
||||
|
||||
it("fails alignment for mid-tail deletion", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'quick', 'brown', 'fox', 'over'
|
||||
];
|
||||
|
||||
const baseTokenization = buildBaseTokenization(baseContext);
|
||||
const computedAlignment = baseTokenization.computeAlignment(newContext, false);
|
||||
|
||||
assert.deepEqual(computedAlignment, {canAlign: false});
|
||||
});
|
||||
|
||||
it("fails alignment for mid-tail insertion", () => {
|
||||
const baseContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'over'
|
||||
];
|
||||
const newContext = [
|
||||
'quick', 'brown', 'fox', 'jumped', 'far', 'over'
|
||||
];
|
||||
|
||||
const baseTokenization = buildBaseTokenization(baseContext);
|
||||
const computedAlignment = baseTokenization.computeAlignment(newContext, false);
|
||||
|
||||
assert.deepEqual(computedAlignment, {canAlign: false});
|
||||
});
|
||||
|
||||
it("handles late-context suggestion application after backspace", () => {
|
||||
const baseContext = [
|
||||
'quick', ' ', 'brown', ' ', 'fox', ' ', 'jumped', ' ', 'oven', ' ', ''
|
||||
];
|
||||
const newContext = [
|
||||
'quick', ' ', 'brown', ' ', 'fox', ' ', 'jumped', ' ', 'over', ' ', ''
|
||||
];
|
||||
|
||||
const baseTokenization = buildBaseTokenization(baseContext);
|
||||
const computedAlignment = baseTokenization.computeAlignment(newContext, false);
|
||||
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
leadTokenShift: 0,
|
||||
matchLength: 8,
|
||||
tailEditLength: 3,
|
||||
tailTokenShift: 0
|
||||
});
|
||||
});
|
||||
|
||||
it("handles sliding context-window scenarios", () => {
|
||||
// // Explicitly-defined window, though it's not needed directly by the method.
|
||||
// const config = {
|
||||
// leftContextCodePoints: 64,
|
||||
// rightContextCodePoints: 64
|
||||
// };
|
||||
|
||||
const baseContext1 = [
|
||||
// "ap" prefix not in actual view, but preserved by prior tokenization rounds.
|
||||
"applesauce", " ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ",
|
||||
"like", " ", "they'd", " ", "make", " ", "for", " ", "the", " ", "be"
|
||||
];
|
||||
|
||||
const incomingContext1 = [
|
||||
"plesauce", " ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ",
|
||||
"like", " ", "they'd", " ", "make", " ", "for", " ", "the", " ", "bes"
|
||||
];
|
||||
|
||||
// 66 chars above, vs a sliding window of length 64.
|
||||
assert.equal(baseContext1.reduce((accum, curr) => accum + curr.length, 0), 66);
|
||||
// Actual window + one newly-typed character
|
||||
assert.equal(incomingContext1.reduce((accum, curr) => accum + curr.length, 0), 65);
|
||||
|
||||
const tokenization1 = new ContextTokenization(buildBaseTokenization(baseContext1));
|
||||
|
||||
assert.deepEqual(tokenization1.computeAlignment(incomingContext1, true), {
|
||||
canAlign: true,
|
||||
leadTokenShift: 0,
|
||||
matchLength: 22,
|
||||
tailEditLength: 1,
|
||||
tailTokenShift: 0
|
||||
});
|
||||
|
||||
// Our tokenization scheme remembers the full original word before any of it slid out of
|
||||
// the context window.
|
||||
const baseContext2 = [
|
||||
"applesauce", " ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ",
|
||||
// +2 +1 +4
|
||||
"like", " ", "they'd", " ", "make", " ", "for", " ", "the", " ", "best", " ", "brea"
|
||||
];
|
||||
|
||||
const incomingContext2 = [
|
||||
// "plesauce" => "e": -7 chars.
|
||||
"e", " ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ",
|
||||
"like", " ", "they'd", " ", "make", " ", "for", " ", "the", " ", "best", " ", "break"
|
||||
];
|
||||
|
||||
// 73 chars above, vs a sliding window of length 64.
|
||||
assert.equal(baseContext2.reduce((accum, curr) => accum + curr.length, 0), 73);
|
||||
// Actual window + one newly-typed character
|
||||
assert.equal(incomingContext2.reduce((accum, curr) => accum + curr.length, 0), 65);
|
||||
|
||||
const tokenization2 = new ContextTokenization(buildBaseTokenization(baseContext2));
|
||||
assert.deepEqual(tokenization2.computeAlignment(incomingContext2, true), {
|
||||
canAlign: true,
|
||||
leadTokenShift: 0,
|
||||
matchLength: 24,
|
||||
tailEditLength: 1,
|
||||
tailTokenShift: 0
|
||||
});
|
||||
|
||||
const baseContext3 = [
|
||||
"applesauce", " ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ", "like", " ",
|
||||
"they'd", " ", "make", " ", "for", " ", "the", " ", "best", " ", "break"
|
||||
];
|
||||
|
||||
const incomingContext3 = [
|
||||
" ", "and", " ", "orange", " ", "juice", " ", "don't", " ", "seem", " ", "like", " ",
|
||||
"they'd", " ", "make", " ", "for", " ", "the", " ", "best", " ", "breakf"
|
||||
];
|
||||
|
||||
// 74 chars above, vs a sliding window of length 64.
|
||||
assert.equal(baseContext3.reduce((accum, curr) => accum + curr.length, 0), 74);
|
||||
// Actual window + one newly-typed character
|
||||
assert.equal(incomingContext3.reduce((accum, curr) => accum + curr.length, 0), 65);
|
||||
|
||||
const tokenization3 = new ContextTokenization(buildBaseTokenization(baseContext3));
|
||||
assert.deepEqual(tokenization3.computeAlignment(incomingContext3, true), {
|
||||
canAlign: true,
|
||||
leadTokenShift: -1,
|
||||
matchLength: 23,
|
||||
tailEditLength: 1,
|
||||
tailTokenShift: 0
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('transitionTo', function() {
|
||||
it('simple case - new whitespace + new empty token', () => {
|
||||
const baseTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue