mirror of
https://github.com/keymanapp/keyman.git
synced 2026-08-31 20:57:41 +00:00
Merge pull request #14697 from keymanapp/change/web/edit-path-edge-data
change(web): edit path edges should report alignment indices instead of texts 🚂
This commit is contained in:
Joshua Horton
GitHub
commit
3dba25cf7d
7 changed files with 638 additions and 168 deletions
|
|
@ -9,7 +9,8 @@
|
|||
*/
|
||||
|
||||
import { SENTINEL_CODE_UNIT } from '@keymanapp/models-templates';
|
||||
import { ClassicalDistanceCalculation, EditOperation } from "./classical-calculation.js";
|
||||
import { ClassicalDistanceCalculation, EditTuple } from "./classical-calculation.js";
|
||||
import { ExtendedEditOperation } from './segmentable-calculation.js';
|
||||
|
||||
/**
|
||||
* Represents token-count values resulting from an alignment attempt between two
|
||||
|
|
@ -26,12 +27,20 @@ export type ContextStateAlignment = {
|
|||
*
|
||||
* The edit path does not include actual user text and is sanitized.
|
||||
*/
|
||||
editPath: EditOperation[];
|
||||
editPath: EditTuple<ExtendedEditOperation>[];
|
||||
} | {
|
||||
/**
|
||||
* Denotes whether or not alignment is possible between two contexts.
|
||||
*/
|
||||
canAlign: true,
|
||||
|
||||
/**
|
||||
* Indicates the edit path that could not be handled. (Useful for error reporting)
|
||||
*
|
||||
* The edit path does not include actual user text and is sanitized.
|
||||
*/
|
||||
editPath: EditTuple<ExtendedEditOperation>[];
|
||||
|
||||
/**
|
||||
* Notes the number of tokens added to the head of the 'incoming'/'new' context
|
||||
* of the contexts being aligned. If negative, the incoming context deleted
|
||||
|
|
@ -88,7 +97,7 @@ export type ContextStateAlignment = {
|
|||
* @param editPath
|
||||
* @returns
|
||||
*/
|
||||
export function getEditPathLastMatch(editPath: EditOperation[], forAppliedSuggestion?: boolean) {
|
||||
export function getEditPathLastMatch(editPath: ExtendedEditOperation[], forAppliedSuggestion?: boolean) {
|
||||
// Assertion: for a long context, the bulk of the edit path should be a
|
||||
// continuous block of 'match' entries. If there's anything but a substitution
|
||||
// in the middle, we have a context mismatch.
|
||||
|
|
@ -261,7 +270,7 @@ export function computeAlignment(
|
|||
|
||||
const failure: ContextStateAlignment = {
|
||||
canAlign: false,
|
||||
editPath
|
||||
editPath: editPaths[0]
|
||||
};
|
||||
|
||||
// Special case: new context bootstrapping - first token often substitutes.
|
||||
|
|
@ -291,6 +300,7 @@ export function computeAlignment(
|
|||
|
||||
return {
|
||||
canAlign: true,
|
||||
editPath: editPaths[0],
|
||||
matchLength: matchCount,
|
||||
leadTokenShift: 0,
|
||||
leadEditLength: 0,
|
||||
|
|
@ -344,6 +354,7 @@ export function computeAlignment(
|
|||
if(firstMatch == 0 && lastMatch == editPath.length - 1) {
|
||||
return {
|
||||
canAlign: true,
|
||||
editPath: editPaths[0],
|
||||
leadTokenShift: 0,
|
||||
leadEditLength: 0,
|
||||
matchLength,
|
||||
|
|
@ -418,6 +429,7 @@ export function computeAlignment(
|
|||
|
||||
return {
|
||||
canAlign: true,
|
||||
editPath: editPaths[0],
|
||||
// leadTokensRemoved represents the number of tokens that must be removed from the base context
|
||||
// when aligning the contexts. Externally, it's more helpful to think in terms of the count added
|
||||
// to the incoming context.
|
||||
|
|
|
|||
|
|
@ -9,13 +9,13 @@ export type EditOperation = 'insert' | 'delete' | 'match' | 'substitute' | 'tran
|
|||
* Represents individual nodes on calculated edit paths and the relevant
|
||||
* edited value(s) at each step.
|
||||
*/
|
||||
export interface EditTuple<TUnit, TOpSet = EditOperation> {
|
||||
export interface EditTuple<TOpSet = EditOperation> {
|
||||
/** The edit operation taking place at this position in the edit path */
|
||||
op: TOpSet | EditOperation,
|
||||
/** The value for the `input` source at this position in the edit path */
|
||||
input?: TUnit,
|
||||
/** The value for the `match` source at this position in the edit path */
|
||||
match?: TUnit
|
||||
/** The index for the `input` source at this position in the edit path */
|
||||
input?: number,
|
||||
/** The index for the `match` source at this position in the edit path */
|
||||
match?: number
|
||||
}
|
||||
|
||||
// Implemented externally from the class so that it may be more easily
|
||||
|
|
@ -53,7 +53,7 @@ export interface EditTuple<TUnit, TOpSet = EditOperation> {
|
|||
* visualization
|
||||
* @returns
|
||||
*/
|
||||
export function visualizeCalculation<TUnit, TOpSet>(calc: ClassicalDistanceCalculation<TUnit, TOpSet>, path?: EditTuple<TUnit, TOpSet>[]) {
|
||||
export function visualizeCalculation<TUnit, TOpSet>(calc: ClassicalDistanceCalculation<TUnit, TOpSet>, path?: EditTuple<TOpSet>[]) {
|
||||
path = (path ?? calc.editPath()[0]).slice();
|
||||
|
||||
const inputs = calc.inputSequence.map(i => '' + i);
|
||||
|
|
@ -107,27 +107,29 @@ export function visualizeCalculation<TUnit, TOpSet>(calc: ClassicalDistanceCalcu
|
|||
rowStr += " ".repeat(sparseCount > 0 ? sparseCount : 1);
|
||||
|
||||
let edits: string[] = [];
|
||||
const printEdit = (edit: EditTuple<TUnit, TOpSet>) => {
|
||||
const printEdit = (edit: EditTuple<TOpSet>) => {
|
||||
let tokenText: string;
|
||||
const input = calc.inputSequence[edit.input];
|
||||
const match = calc.matchSequence[edit.match];
|
||||
switch(edit.op) {
|
||||
case 'delete':
|
||||
case 'transpose-delete':
|
||||
tokenText = `'${edit.input}'`;
|
||||
tokenText = `'${input}'`;
|
||||
break;
|
||||
case 'insert':
|
||||
case 'transpose-insert':
|
||||
tokenText = `'${edit.match}'`;
|
||||
tokenText = `'${match}'`;
|
||||
break;
|
||||
case 'substitute':
|
||||
case 'match':
|
||||
case 'split':
|
||||
case 'merge':
|
||||
const op = edit.op == 'match' ? '==' : '=>';
|
||||
tokenText = tokenText || `'${edit.input}' ${op} '${edit.match}'`;
|
||||
tokenText = tokenText || `'${input}' ${op} '${match}'`;
|
||||
break;
|
||||
// transpose-start, transpose-end
|
||||
default:
|
||||
tokenText = `'${edit.input}' vs '${edit.match}'`;
|
||||
tokenText = `'${input}' vs '${match}'`;
|
||||
}
|
||||
return `${edit.op}(${tokenText})`;
|
||||
}
|
||||
|
|
@ -337,7 +339,7 @@ export class ClassicalDistanceCalculation<TUnit = string, TOpSet = EditOperation
|
|||
* @param row
|
||||
* @param col
|
||||
*/
|
||||
public editPath(): EditTuple<TUnit, TOpSet>[][] {
|
||||
public editPath(): EditTuple<TOpSet>[][] {
|
||||
const results = this._buildPath();
|
||||
|
||||
if(results.length <= 1) {
|
||||
|
|
@ -415,7 +417,7 @@ export class ClassicalDistanceCalculation<TUnit = string, TOpSet = EditOperation
|
|||
* @param row
|
||||
* @param col
|
||||
*/
|
||||
protected _buildPath(pathBuilder?: PathBuilder<TUnit, TOpSet>): EditTuple<TUnit, TOpSet>[][] {
|
||||
protected _buildPath(pathBuilder?: PathBuilder<TUnit, TOpSet>): EditTuple<TOpSet>[][] {
|
||||
pathBuilder = pathBuilder ?? new PathBuilder(this, []);
|
||||
pathBuilder.addEdgeFinder(findBaseEdges);
|
||||
pathBuilder.addEdgeFinder(findTransposeEdges);
|
||||
|
|
@ -825,21 +827,21 @@ export function findTransposeEdges<TUnit, TOpSet>(
|
|||
// This transposition includes either 'transpose-insert' or 'transpose-delete' operations.
|
||||
let i = row;
|
||||
let m = col;
|
||||
let ops: EditTuple<TUnit, TOpSet>[] = [];
|
||||
let ops: EditTuple<TOpSet>[] = [];
|
||||
|
||||
if(lastInputIndex != row-1) {
|
||||
let count = row - lastInputIndex;
|
||||
ops.push({
|
||||
op: 'transpose-start',
|
||||
input: calc.inputSequence[i-count],
|
||||
match: calc.matchSequence[lastMatchIndex]
|
||||
input: i-count,
|
||||
match: lastMatchIndex
|
||||
});
|
||||
// Intentional fallthrough on 0 - index 0 is covered by 'transpose-end'
|
||||
// after the if-else.
|
||||
for(let x=count-1; x > 0; x--) {
|
||||
ops.push({
|
||||
op: 'transpose-delete',
|
||||
input: calc.inputSequence[i-x]
|
||||
input: i-x
|
||||
});
|
||||
}
|
||||
expectedCost += count-1;
|
||||
|
|
@ -847,15 +849,15 @@ export function findTransposeEdges<TUnit, TOpSet>(
|
|||
let count = col - lastMatchIndex;
|
||||
ops.push({
|
||||
op: 'transpose-start',
|
||||
input: calc.inputSequence[lastInputIndex],
|
||||
match: calc.matchSequence[m-count]
|
||||
input: lastInputIndex,
|
||||
match: m-count
|
||||
});
|
||||
// Intentional fallthrough on 0 - index 0 is covered by 'transpose-end'
|
||||
// after the if-else.
|
||||
for(let y=count-1; y > 0; y--) {
|
||||
ops.push({
|
||||
op: 'transpose-insert',
|
||||
match: calc.matchSequence[m-y]
|
||||
match: m-y
|
||||
});
|
||||
}
|
||||
expectedCost += count - 1;
|
||||
|
|
@ -863,8 +865,8 @@ export function findTransposeEdges<TUnit, TOpSet>(
|
|||
|
||||
ops.push({
|
||||
op: 'transpose-end',
|
||||
input: calc.inputSequence[i],
|
||||
match: calc.matchSequence[m]
|
||||
input: i,
|
||||
match: m
|
||||
});
|
||||
|
||||
// Double-check our expectations.
|
||||
|
|
@ -899,28 +901,28 @@ export function findBaseEdges<TUnit, TOpSet>(
|
|||
|
||||
const insertParentCost = calc.getCostAt(row, col-1);
|
||||
if(insertParentCost == currentCost - 1) {
|
||||
pathBuilder.backtracePath(row, col-1, [{op: 'insert', match}]);
|
||||
pathBuilder.backtracePath(row, col-1, [{op: 'insert', match: col}]);
|
||||
}
|
||||
|
||||
const deleteParentCost = calc.getCostAt(row-1, col);
|
||||
if(deleteParentCost == currentCost - 1) {
|
||||
pathBuilder.backtracePath(row-1, col, [{op: 'delete', input}]);
|
||||
pathBuilder.backtracePath(row-1, col, [{op: 'delete', input: row}]);
|
||||
}
|
||||
|
||||
const substitutionParentCost = calc.getCostAt(row-1, col-1);
|
||||
if(substitutionParentCost == currentCost - 1) {
|
||||
pathBuilder.backtracePath(row-1, col-1, [{op: 'substitute', input, match}]);
|
||||
pathBuilder.backtracePath(row-1, col-1, [{op: 'substitute', input: row, match: col}]);
|
||||
// VERY IMPORTANT: validate the match. The path can go "off the rails" if
|
||||
// we don't validate this!
|
||||
} else if(substitutionParentCost == currentCost && input == match) {
|
||||
pathBuilder.backtracePath(row-1, col-1, [{op: 'match', input, match}]);
|
||||
pathBuilder.backtracePath(row-1, col-1, [{op: 'match', input: row, match: col}]);
|
||||
}
|
||||
}
|
||||
|
||||
export class PathBuilder<TUnit, TOpSet = EditOperation> {
|
||||
readonly calc: ClassicalDistanceCalculation<TUnit, TOpSet>;
|
||||
readonly edgeFinders: (EdgeFinder<TUnit, TOpSet>)[];
|
||||
readonly validPaths: EditTuple<TUnit, TOpSet>[][] = [];
|
||||
readonly validPaths: EditTuple<TOpSet>[][] = [];
|
||||
|
||||
constructor(calc: ClassicalDistanceCalculation<TUnit, TOpSet>, edgeFinders: (EdgeFinder<TUnit, TOpSet>)[]) {
|
||||
this.calc = calc;
|
||||
|
|
@ -931,11 +933,11 @@ export class PathBuilder<TUnit, TOpSet = EditOperation> {
|
|||
this.edgeFinders.push(finder);
|
||||
}
|
||||
|
||||
backtracePath(row: number, col: number, recentEdge: EditTuple<TUnit, TOpSet>[]) {
|
||||
backtracePath(row: number, col: number, recentEdge: EditTuple<TOpSet>[]) {
|
||||
const calc = this.calc;
|
||||
|
||||
// Recursively build the edit path.
|
||||
let results: EditTuple<TUnit, TOpSet>[][];
|
||||
let results: EditTuple<TOpSet>[][];
|
||||
if(row >= 0 && col >= 0) {
|
||||
const parentBuilder = new PathBuilder<TUnit, TOpSet>(calc, this.edgeFinders);
|
||||
if(calc.getCostAt(row, col) == Number.MAX_VALUE) {
|
||||
|
|
@ -951,14 +953,14 @@ export class PathBuilder<TUnit, TOpSet = EditOperation> {
|
|||
// There are initial deletions.
|
||||
result.push({
|
||||
op: 'delete',
|
||||
input: calc.inputSequence[r]
|
||||
input: r
|
||||
});
|
||||
}
|
||||
for(let c = 0; c <= col; c++) {
|
||||
// There are initial insertions.
|
||||
result.push({
|
||||
op: 'insert',
|
||||
match: calc.matchSequence[c]
|
||||
match: c
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -89,7 +89,7 @@ export class SegmentableDistanceCalculation extends ClassicalDistanceCalculation
|
|||
throw new Error("Not yet supported for this edit-distance calculation type.");
|
||||
}
|
||||
|
||||
protected _buildPath(pathBuilder?: PathBuilder<string, ExtendedEditOperation>): EditTuple<string, ExtendedEditOperation>[][] {
|
||||
protected _buildPath(pathBuilder?: PathBuilder<string, ExtendedEditOperation>): EditTuple<ExtendedEditOperation>[][] {
|
||||
pathBuilder = pathBuilder ?? new PathBuilder<string, ExtendedEditOperation>(this, []);
|
||||
pathBuilder.addEdgeFinder(findSplitMergeEdges);
|
||||
super._buildPath(pathBuilder); // actually evaluates the edit-path.
|
||||
|
|
@ -159,22 +159,19 @@ export function findSplitMergeEdges<TOpSet>(
|
|||
throw new Error("Cannot find path - diagonal width is not large enough.")
|
||||
}
|
||||
|
||||
const input = calc.inputSequence[row];
|
||||
const match = calc.matchSequence[col];
|
||||
|
||||
const [lastMergeIndex, lastSplitIndex] = getMergeSplitParent(calc, row, col);
|
||||
if(lastMergeIndex != -1) {
|
||||
const ops: EditTuple<string, ExtendedEditOperation>[] = [];
|
||||
const ops: EditTuple<ExtendedEditOperation>[] = [];
|
||||
for(let r = lastMergeIndex; r <= row; r++) {
|
||||
ops.push({ input: calc.inputSequence[r], match, op: 'merge' });
|
||||
ops.push({ input: r, match: col, op: 'merge' });
|
||||
}
|
||||
pathBuilder.backtracePath(lastMergeIndex - 1, col - 1, ops);
|
||||
}
|
||||
|
||||
if(lastSplitIndex != -1) {
|
||||
const ops: EditTuple<string, ExtendedEditOperation>[] = [];
|
||||
const ops: EditTuple<ExtendedEditOperation>[] = [];
|
||||
for(let c = lastSplitIndex; c <= col; c++) {
|
||||
ops.push({ input, match: calc.matchSequence[c], op: 'split' });
|
||||
ops.push({ input: row, match: c, op: 'split' });
|
||||
}
|
||||
pathBuilder.backtracePath(row - 1, lastSplitIndex - 1, ops);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -133,6 +133,13 @@ describe('computeAlignment', () => {
|
|||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'match', input: 0, match: 0},
|
||||
{op: 'match', input: 1, match: 1},
|
||||
{op: 'match', input: 2, match: 2},
|
||||
{op: 'match', input: 3, match: 3},
|
||||
{op: 'match', input: 4, match: 4}
|
||||
],
|
||||
leadTokenShift: 0,
|
||||
leadEditLength: 0,
|
||||
matchLength: 5,
|
||||
|
|
@ -151,6 +158,13 @@ describe('computeAlignment', () => {
|
|||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'match', input: 0, match: 0},
|
||||
{op: 'match', input: 1, match: 1},
|
||||
{op: 'match', input: 2, match: 2},
|
||||
{op: 'match', input: 3, match: 3},
|
||||
{op: 'substitute', input: 4, match: 4}
|
||||
],
|
||||
leadTokenShift: 0,
|
||||
leadEditLength: 0,
|
||||
matchLength: 4,
|
||||
|
|
@ -172,6 +186,11 @@ describe('computeAlignment', () => {
|
|||
const computedAlignment = computeAlignment(baseContext, newContext, false, true);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'substitute', input: 0, match: 0},
|
||||
{op: 'insert', match: 1},
|
||||
{op: 'insert', match: 2}
|
||||
],
|
||||
leadTokenShift: 0,
|
||||
leadEditLength: 0,
|
||||
matchLength: 0,
|
||||
|
|
@ -189,7 +208,16 @@ describe('computeAlignment', () => {
|
|||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {canAlign: false, editPath: ['substitute', 'substitute', 'substitute', 'substitute', 'substitute']});
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: false,
|
||||
editPath: [
|
||||
{ op: 'substitute', input: 0, match: 0 },
|
||||
{ op: 'substitute', input: 1, match: 1 },
|
||||
{ op: 'substitute', input: 2, match: 2 },
|
||||
{ op: 'substitute', input: 3, match: 3 },
|
||||
{ op: 'substitute', input: 4, match: 4 }
|
||||
]
|
||||
});
|
||||
});
|
||||
|
||||
it("detects unalignable contexts - too many mismatching tokens", () => {
|
||||
|
|
@ -201,7 +229,16 @@ describe('computeAlignment', () => {
|
|||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {canAlign: false, editPath: ['substitute', 'substitute', 'match', 'match', 'match']});
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: false,
|
||||
editPath: [
|
||||
{op: 'substitute', input: 0, match: 0},
|
||||
{op: 'substitute', input: 1, match: 1},
|
||||
{op: 'match', input: 2, match: 2},
|
||||
{op: 'match', input: 3, match: 3},
|
||||
{op: 'match', input: 4, match: 4}
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("fails alignment for leading-edge word substitutions", () => {
|
||||
|
|
@ -213,7 +250,16 @@ describe('computeAlignment', () => {
|
|||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {canAlign: false, editPath: ['substitute', 'match', 'match', 'match', 'match']});
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: false,
|
||||
editPath: [
|
||||
{op: 'substitute', input: 0, match: 0},
|
||||
{op: 'match', input: 1, match: 1},
|
||||
{op: 'match', input: 2, match: 2},
|
||||
{op: 'match', input: 3, match: 3},
|
||||
{op: 'match', input: 4, match: 4}
|
||||
]
|
||||
});
|
||||
});
|
||||
|
||||
it("fails alignment for small leading-edge word substitutions", () => {
|
||||
|
|
@ -225,7 +271,16 @@ describe('computeAlignment', () => {
|
|||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {canAlign: false, editPath: ['substitute', 'match', 'match', 'match', 'match']});
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: false,
|
||||
editPath: [
|
||||
{op: 'substitute', input: 0, match: 0},
|
||||
{op: 'match', input: 1, match: 1},
|
||||
{op: 'match', input: 2, match: 2},
|
||||
{op: 'match', input: 3, match: 3},
|
||||
{op: 'match', input: 4, match: 4}
|
||||
]
|
||||
});
|
||||
});
|
||||
|
||||
it("properly matches and aligns when lead token is modified", () => {
|
||||
|
|
@ -239,6 +294,13 @@ describe('computeAlignment', () => {
|
|||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'substitute', input: 0, match: 0},
|
||||
{op: 'match', input: 1, match: 1},
|
||||
{op: 'match', input: 2, match: 2},
|
||||
{op: 'match', input: 3, match: 3},
|
||||
{op: 'match', input: 4, match: 4}
|
||||
],
|
||||
leadTokenShift: 0,
|
||||
leadEditLength: 1,
|
||||
matchLength: 4,
|
||||
|
|
@ -258,6 +320,13 @@ describe('computeAlignment', () => {
|
|||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'delete', input: 0},
|
||||
{op: 'match', input: 1, match: 0},
|
||||
{op: 'match', input: 2, match: 1},
|
||||
{op: 'match', input: 3, match: 2},
|
||||
{op: 'match', input: 4, match: 3}
|
||||
],
|
||||
leadTokenShift: -1,
|
||||
leadEditLength: 0,
|
||||
matchLength: 4,
|
||||
|
|
@ -277,6 +346,13 @@ describe('computeAlignment', () => {
|
|||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'insert', match: 0},
|
||||
{op: 'match', input: 0, match: 1},
|
||||
{op: 'match', input: 1, match: 2},
|
||||
{op: 'match', input: 2, match: 3},
|
||||
{op: 'match', input: 3, match: 4}
|
||||
],
|
||||
leadTokenShift: 1,
|
||||
leadEditLength: 0,
|
||||
matchLength: 4,
|
||||
|
|
@ -296,6 +372,13 @@ describe('computeAlignment', () => {
|
|||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'delete', input: 0},
|
||||
{op: 'delete', input: 1},
|
||||
{op: 'substitute', input: 2, match: 0},
|
||||
{op: 'match', input: 3, match: 1},
|
||||
{op: 'match', input: 4, match: 2},
|
||||
],
|
||||
leadTokenShift: -2,
|
||||
leadEditLength: 1,
|
||||
matchLength: 2,
|
||||
|
|
@ -315,6 +398,13 @@ describe('computeAlignment', () => {
|
|||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'insert', match: 0},
|
||||
{op: 'substitute', input: 0, match: 1},
|
||||
{op: 'match', input: 1, match: 2},
|
||||
{op: 'match', input: 2, match: 3},
|
||||
{op: 'match', input: 3, match: 4},
|
||||
],
|
||||
leadTokenShift: 1,
|
||||
leadEditLength: 1,
|
||||
matchLength: 3,
|
||||
|
|
@ -334,6 +424,14 @@ describe('computeAlignment', () => {
|
|||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'delete', input: 0},
|
||||
{op: 'match', input: 1, match: 0},
|
||||
{op: 'match', input: 2, match: 1},
|
||||
{op: 'match', input: 3, match: 2},
|
||||
{op: 'match', input: 4, match: 3},
|
||||
{op: 'insert', match: 4}
|
||||
],
|
||||
leadTokenShift: -1,
|
||||
leadEditLength: 0,
|
||||
matchLength: 4,
|
||||
|
|
@ -353,6 +451,13 @@ describe('computeAlignment', () => {
|
|||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'substitute', input: 0, match: 0},
|
||||
{op: 'match', input: 1, match: 1},
|
||||
{op: 'match', input: 2, match: 2},
|
||||
{op: 'match', input: 3, match: 3},
|
||||
{op: 'substitute', input: 4, match: 4}
|
||||
],
|
||||
leadTokenShift: 0,
|
||||
leadEditLength: 1,
|
||||
matchLength: 3,
|
||||
|
|
@ -372,6 +477,14 @@ describe('computeAlignment', () => {
|
|||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'substitute', input: 0, match: 0},
|
||||
{op: 'match', input: 1, match: 1},
|
||||
{op: 'match', input: 2, match: 2},
|
||||
{op: 'match', input: 3, match: 3},
|
||||
{op: 'substitute', input: 4, match: 4},
|
||||
{op: 'insert', match: 5}
|
||||
],
|
||||
leadTokenShift: 0,
|
||||
leadEditLength: 1,
|
||||
matchLength: 3,
|
||||
|
|
@ -391,9 +504,17 @@ describe('computeAlignment', () => {
|
|||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'insert', match: 0},
|
||||
{op: 'match', input: 0, match: 1},
|
||||
{op: 'match', input: 1, match: 2},
|
||||
{op: 'match', input: 2, match: 3},
|
||||
{op: 'match', input: 3, match: 4},
|
||||
{op: 'substitute', input: 4, match: 5}
|
||||
],
|
||||
leadTokenShift: 1,
|
||||
leadEditLength: 0,
|
||||
matchLength: 4, // we treat 'quick' and 'uick' as the same
|
||||
matchLength: 4,
|
||||
tailEditLength: 1,
|
||||
tailTokenShift: 0
|
||||
});
|
||||
|
|
@ -410,9 +531,17 @@ describe('computeAlignment', () => {
|
|||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'insert', match: 0},
|
||||
{op: 'match', input: 0, match: 1},
|
||||
{op: 'match', input: 1, match: 2},
|
||||
{op: 'match', input: 2, match: 3},
|
||||
{op: 'match', input: 3, match: 4},
|
||||
{op: 'delete', input: 4}
|
||||
],
|
||||
leadTokenShift: 1,
|
||||
leadEditLength: 0,
|
||||
matchLength: 4, // we treat 'quick' and 'uick' as the same
|
||||
matchLength: 4,
|
||||
tailEditLength: 0,
|
||||
tailTokenShift: -1
|
||||
});
|
||||
|
|
@ -429,9 +558,17 @@ describe('computeAlignment', () => {
|
|||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'insert', match: 0},
|
||||
{op: 'match', input: 0, match: 1},
|
||||
{op: 'match', input: 1, match: 2},
|
||||
{op: 'match', input: 2, match: 3},
|
||||
{op: 'substitute', input: 3, match: 4},
|
||||
{op: 'delete', input: 4}
|
||||
],
|
||||
leadTokenShift: 1,
|
||||
leadEditLength: 0,
|
||||
matchLength: 3, // we treat 'quick' and 'uick' as the same
|
||||
matchLength: 3,
|
||||
tailEditLength: 1,
|
||||
tailTokenShift: -1
|
||||
});
|
||||
|
|
@ -446,7 +583,16 @@ describe('computeAlignment', () => {
|
|||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {canAlign: false, editPath: ["match", "delete", "match", "match", "match"]});
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: false,
|
||||
editPath: [
|
||||
{op: 'match', input: 0, match: 0},
|
||||
{op: 'delete', input: 1},
|
||||
{op: 'match', input: 2, match: 1},
|
||||
{op: 'match', input: 3, match: 2},
|
||||
{op: 'match', input: 4, match: 3}
|
||||
]
|
||||
});
|
||||
});
|
||||
|
||||
it("fails alignment for mid-head insertion", () => {
|
||||
|
|
@ -458,7 +604,16 @@ describe('computeAlignment', () => {
|
|||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {canAlign: false, editPath: ["match", "insert", "match", "match", "match"]});
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: false,
|
||||
editPath: [
|
||||
{op: 'match', input: 0, match: 0},
|
||||
{op: 'insert', match: 1},
|
||||
{op: 'match', input: 1, match: 2},
|
||||
{op: 'match', input: 2, match: 3},
|
||||
{op: 'match', input: 3, match: 4}
|
||||
]
|
||||
});
|
||||
});
|
||||
|
||||
it("fails alignment for mid-tail deletion", () => {
|
||||
|
|
@ -470,7 +625,16 @@ describe('computeAlignment', () => {
|
|||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {canAlign: false, editPath: ["match", "match", "match", "delete", "match"]});
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: false,
|
||||
editPath: [
|
||||
{op: 'match', input: 0, match: 0},
|
||||
{op: 'match', input: 1, match: 1},
|
||||
{op: 'match', input: 2, match: 2},
|
||||
{op: 'delete', input: 3},
|
||||
{op: 'match', input: 4, match: 3}
|
||||
]
|
||||
});
|
||||
});
|
||||
|
||||
it("fails alignment for mid-tail insertion", () => {
|
||||
|
|
@ -482,7 +646,17 @@ describe('computeAlignment', () => {
|
|||
];
|
||||
|
||||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {canAlign: false, editPath: ["match", "match", "match", "match", "insert", "match"]});
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: false,
|
||||
editPath: [
|
||||
{op: 'match', input: 0, match: 0},
|
||||
{op: 'match', input: 1, match: 1},
|
||||
{op: 'match', input: 2, match: 2},
|
||||
{op: 'match', input: 3, match: 3},
|
||||
{op: 'insert', match: 4},
|
||||
{op: 'match', input: 4, match: 5}
|
||||
]
|
||||
});
|
||||
});
|
||||
|
||||
it("handles late-context suggestion application after backspace", () => {
|
||||
|
|
@ -496,6 +670,19 @@ describe('computeAlignment', () => {
|
|||
const computedAlignment = computeAlignment(baseContext, newContext, false);
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'match', input: 0, match: 0},
|
||||
{op: 'match', input: 1, match: 1},
|
||||
{op: 'match', input: 2, match: 2},
|
||||
{op: 'match', input: 3, match: 3},
|
||||
{op: 'match', input: 4, match: 4},
|
||||
{op: 'match', input: 5, match: 5},
|
||||
{op: 'match', input: 6, match: 6},
|
||||
{op: 'match', input: 7, match: 7},
|
||||
{op: 'substitute', input: 8, match: 8},
|
||||
{op: 'match', input: 9, match: 9},
|
||||
{op: 'match', input: 10, match: 10}
|
||||
],
|
||||
leadTokenShift: 0,
|
||||
leadEditLength: 0,
|
||||
matchLength: 8,
|
||||
|
|
@ -516,6 +703,21 @@ describe('computeAlignment', () => {
|
|||
|
||||
assert.deepEqual(computedAlignment, {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'match', input: 0, match: 0},
|
||||
{op: 'match', input: 1, match: 1},
|
||||
{op: 'match', input: 2, match: 2},
|
||||
{op: 'match', input: 3, match: 3},
|
||||
{op: 'match', input: 4, match: 4},
|
||||
{op: 'match', input: 5, match: 5},
|
||||
{op: 'match', input: 6, match: 6},
|
||||
{op: 'match', input: 7, match: 7},
|
||||
{op: 'match', input: 8, match: 8},
|
||||
{op: 'match', input: 9, match: 9},
|
||||
{op: 'substitute', input: 10, match: 10},
|
||||
{op: 'insert', match: 11},
|
||||
{op: 'insert', match: 12}
|
||||
],
|
||||
leadTokenShift: 0,
|
||||
leadEditLength: 0,
|
||||
matchLength: 10,
|
||||
|
|
@ -549,6 +751,31 @@ describe('computeAlignment', () => {
|
|||
|
||||
assert.deepEqual(computeAlignment(baseContext1, incomingContext1, true), {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'substitute', input: 0, match: 0},
|
||||
{op: 'match', input: 1, match: 1},
|
||||
{op: 'match', input: 2, match: 2},
|
||||
{op: 'match', input: 3, match: 3},
|
||||
{op: 'match', input: 4, match: 4},
|
||||
{op: 'match', input: 5, match: 5},
|
||||
{op: 'match', input: 6, match: 6},
|
||||
{op: 'match', input: 7, match: 7},
|
||||
{op: 'match', input: 8, match: 8},
|
||||
{op: 'match', input: 9, match: 9},
|
||||
{op: 'match', input: 10, match: 10},
|
||||
{op: 'match', input: 11, match: 11},
|
||||
{op: 'match', input: 12, match: 12},
|
||||
{op: 'match', input: 13, match: 13},
|
||||
{op: 'match', input: 14, match: 14},
|
||||
{op: 'match', input: 15, match: 15},
|
||||
{op: 'match', input: 16, match: 16},
|
||||
{op: 'match', input: 17, match: 17},
|
||||
{op: 'match', input: 18, match: 18},
|
||||
{op: 'match', input: 19, match: 19},
|
||||
{op: 'match', input: 20, match: 20},
|
||||
{op: 'match', input: 21, match: 21},
|
||||
{op: 'substitute', input: 22, match: 22}
|
||||
],
|
||||
leadTokenShift: 0,
|
||||
leadEditLength: 1,
|
||||
matchLength: 21,
|
||||
|
|
@ -577,6 +804,33 @@ describe('computeAlignment', () => {
|
|||
|
||||
assert.deepEqual(computeAlignment(baseContext2, incomingContext2, true), {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'substitute', input: 0, match: 0},
|
||||
{op: 'match', input: 1, match: 1},
|
||||
{op: 'match', input: 2, match: 2},
|
||||
{op: 'match', input: 3, match: 3},
|
||||
{op: 'match', input: 4, match: 4},
|
||||
{op: 'match', input: 5, match: 5},
|
||||
{op: 'match', input: 6, match: 6},
|
||||
{op: 'match', input: 7, match: 7},
|
||||
{op: 'match', input: 8, match: 8},
|
||||
{op: 'match', input: 9, match: 9},
|
||||
{op: 'match', input: 10, match: 10},
|
||||
{op: 'match', input: 11, match: 11},
|
||||
{op: 'match', input: 12, match: 12},
|
||||
{op: 'match', input: 13, match: 13},
|
||||
{op: 'match', input: 14, match: 14},
|
||||
{op: 'match', input: 15, match: 15},
|
||||
{op: 'match', input: 16, match: 16},
|
||||
{op: 'match', input: 17, match: 17},
|
||||
{op: 'match', input: 18, match: 18},
|
||||
{op: 'match', input: 19, match: 19},
|
||||
{op: 'match', input: 20, match: 20},
|
||||
{op: 'match', input: 21, match: 21},
|
||||
{op: 'match', input: 22, match: 22},
|
||||
{op: 'match', input: 23, match: 23},
|
||||
{op: 'substitute', input: 24, match: 24}
|
||||
],
|
||||
leadTokenShift: 0,
|
||||
leadEditLength: 1,
|
||||
matchLength: 23,
|
||||
|
|
@ -601,6 +855,33 @@ describe('computeAlignment', () => {
|
|||
|
||||
assert.deepEqual(computeAlignment(baseContext3, incomingContext3, true), {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'delete', input: 0},
|
||||
{op: 'match', input: 1, match: 0},
|
||||
{op: 'match', input: 2, match: 1},
|
||||
{op: 'match', input: 3, match: 2},
|
||||
{op: 'match', input: 4, match: 3},
|
||||
{op: 'match', input: 5, match: 4},
|
||||
{op: 'match', input: 6, match: 5},
|
||||
{op: 'match', input: 7, match: 6},
|
||||
{op: 'match', input: 8, match: 7},
|
||||
{op: 'match', input: 9, match: 8},
|
||||
{op: 'match', input: 10, match: 9},
|
||||
{op: 'match', input: 11, match: 10},
|
||||
{op: 'match', input: 12, match: 11},
|
||||
{op: 'match', input: 13, match: 12},
|
||||
{op: 'match', input: 14, match: 13},
|
||||
{op: 'match', input: 15, match: 14},
|
||||
{op: 'match', input: 16, match: 15},
|
||||
{op: 'match', input: 17, match: 16},
|
||||
{op: 'match', input: 18, match: 17},
|
||||
{op: 'match', input: 19, match: 18},
|
||||
{op: 'match', input: 20, match: 19},
|
||||
{op: 'match', input: 21, match: 20},
|
||||
{op: 'match', input: 22, match: 21},
|
||||
{op: 'match', input: 23, match: 22},
|
||||
{op: 'substitute', input: 24, match: 23}
|
||||
],
|
||||
leadTokenShift: -1,
|
||||
leadEditLength: 0,
|
||||
matchLength: 23,
|
||||
|
|
|
|||
|
|
@ -45,6 +45,15 @@ describe('ContextTokenization', function() {
|
|||
const rawTextTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'];
|
||||
let alignment: ContextStateAlignment = {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'match', input: 0, match: 0},
|
||||
{op: 'match', input: 1, match: 1},
|
||||
{op: 'match', input: 2, match: 2},
|
||||
{op: 'match', input: 3, match: 3},
|
||||
{op: 'match', input: 4, match: 4},
|
||||
{op: 'match', input: 5, match: 5},
|
||||
{op: 'match', input: 6, match: 6}
|
||||
],
|
||||
leadTokenShift: 0,
|
||||
leadEditLength: 0,
|
||||
matchLength: 6,
|
||||
|
|
@ -67,6 +76,15 @@ describe('ContextTokenization', function() {
|
|||
|
||||
let baseTokenization = new ContextTokenization(rawTextTokens.map((text => toToken(text))), {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'match', input: 0, match: 0},
|
||||
{op: 'match', input: 1, match: 1},
|
||||
{op: 'match', input: 2, match: 2},
|
||||
{op: 'match', input: 3, match: 3},
|
||||
{op: 'match', input: 4, match: 4},
|
||||
{op: 'match', input: 5, match: 5},
|
||||
{op: 'match', input: 6, match: 6}
|
||||
],
|
||||
leadTokenShift: 0,
|
||||
leadEditLength: 0,
|
||||
matchLength: 6,
|
||||
|
|
@ -112,6 +130,17 @@ describe('ContextTokenization', function() {
|
|||
const tokenization = baseTokenization.transitionTo(
|
||||
targetTokens, {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'match', input: 0, match: 0},
|
||||
{op: 'match', input: 1, match: 1},
|
||||
{op: 'match', input: 2, match: 2},
|
||||
{op: 'match', input: 3, match: 3},
|
||||
{op: 'match', input: 4, match: 4},
|
||||
{op: 'match', input: 5, match: 5},
|
||||
{op: 'match', input: 6, match: 6},
|
||||
{op: 'insert', match: 7},
|
||||
{op: 'insert', match: 8}
|
||||
],
|
||||
leadTokenShift: 0,
|
||||
leadEditLength: 0,
|
||||
matchLength: 7,
|
||||
|
|
@ -140,6 +169,15 @@ describe('ContextTokenization', function() {
|
|||
const tokenization = baseTokenization.transitionTo(
|
||||
targetTokens, {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'match', input: 0, match: 0},
|
||||
{op: 'match', input: 1, match: 1},
|
||||
{op: 'match', input: 2, match: 2},
|
||||
{op: 'match', input: 3, match: 3},
|
||||
{op: 'match', input: 4, match: 4},
|
||||
{op: 'match', input: 5, match: 5},
|
||||
{op: 'substitute', input: 6, match: 6}
|
||||
],
|
||||
leadTokenShift: 0,
|
||||
leadEditLength: 0,
|
||||
matchLength: 6,
|
||||
|
|
@ -171,6 +209,17 @@ describe('ContextTokenization', function() {
|
|||
const tokenization = baseTokenization.transitionTo(
|
||||
targetTokens, {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'match', input: 0, match: 0},
|
||||
{op: 'match', input: 1, match: 1},
|
||||
{op: 'match', input: 2, match: 2},
|
||||
{op: 'match', input: 3, match: 3},
|
||||
{op: 'match', input: 4, match: 4},
|
||||
{op: 'match', input: 5, match: 5},
|
||||
{op: 'match', input: 6, match: 6},
|
||||
{op: 'substitute', input: 7, match: 7},
|
||||
{op: 'substitute', input: 8, match: 8}
|
||||
],
|
||||
leadTokenShift: 0,
|
||||
leadEditLength: 0,
|
||||
matchLength: 7,
|
||||
|
|
@ -210,6 +259,31 @@ describe('ContextTokenization', function() {
|
|||
const tokenization = baseTokenization.transitionTo(
|
||||
targetTokens, {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'substitute', input: 0, match: 0},
|
||||
{op: 'match', input: 1, match: 1},
|
||||
{op: 'match', input: 2, match: 2},
|
||||
{op: 'match', input: 3, match: 3},
|
||||
{op: 'match', input: 4, match: 4},
|
||||
{op: 'match', input: 5, match: 5},
|
||||
{op: 'match', input: 6, match: 6},
|
||||
{op: 'match', input: 7, match: 7},
|
||||
{op: 'match', input: 8, match: 8},
|
||||
{op: 'match', input: 9, match: 9},
|
||||
{op: 'match', input: 10, match: 10},
|
||||
{op: 'match', input: 11, match: 11},
|
||||
{op: 'match', input: 12, match: 12},
|
||||
{op: 'match', input: 13, match: 13},
|
||||
{op: 'match', input: 14, match: 14},
|
||||
{op: 'match', input: 15, match: 15},
|
||||
{op: 'match', input: 16, match: 16},
|
||||
{op: 'match', input: 17, match: 17},
|
||||
{op: 'match', input: 18, match: 18},
|
||||
{op: 'match', input: 19, match: 19},
|
||||
{op: 'match', input: 20, match: 20},
|
||||
{op: 'match', input: 21, match: 21},
|
||||
{op: 'substitute', input: 22, match: 22}
|
||||
],
|
||||
leadTokenShift: 0,
|
||||
leadEditLength: 1,
|
||||
matchLength: 21,
|
||||
|
|
@ -247,6 +321,33 @@ describe('ContextTokenization', function() {
|
|||
const tokenization = baseTokenization.transitionTo(
|
||||
targetTokens, {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'substitute', input: 0, match: 0},
|
||||
{op: 'match', input: 1, match: 1},
|
||||
{op: 'match', input: 2, match: 2},
|
||||
{op: 'match', input: 3, match: 3},
|
||||
{op: 'match', input: 4, match: 4},
|
||||
{op: 'match', input: 5, match: 5},
|
||||
{op: 'match', input: 6, match: 6},
|
||||
{op: 'match', input: 7, match: 7},
|
||||
{op: 'match', input: 8, match: 8},
|
||||
{op: 'match', input: 9, match: 9},
|
||||
{op: 'match', input: 10, match: 10},
|
||||
{op: 'match', input: 11, match: 11},
|
||||
{op: 'match', input: 12, match: 12},
|
||||
{op: 'match', input: 13, match: 13},
|
||||
{op: 'match', input: 14, match: 14},
|
||||
{op: 'match', input: 15, match: 15},
|
||||
{op: 'match', input: 16, match: 16},
|
||||
{op: 'match', input: 17, match: 17},
|
||||
{op: 'match', input: 18, match: 18},
|
||||
{op: 'match', input: 19, match: 19},
|
||||
{op: 'match', input: 20, match: 20},
|
||||
{op: 'match', input: 21, match: 21},
|
||||
{op: 'match', input: 22, match: 22},
|
||||
{op: 'match', input: 23, match: 23},
|
||||
{op: 'substitute', input: 24, match: 24}
|
||||
],
|
||||
leadTokenShift: 0,
|
||||
leadEditLength: 1,
|
||||
matchLength: 23,
|
||||
|
|
@ -282,6 +383,19 @@ describe('ContextTokenization', function() {
|
|||
// properly.
|
||||
targetTokens, {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'substitute', input: 0, match: 0},
|
||||
{op: 'match', input: 1, match: 1},
|
||||
{op: 'match', input: 2, match: 2},
|
||||
{op: 'match', input: 3, match: 3},
|
||||
{op: 'match', input: 4, match: 4},
|
||||
{op: 'match', input: 5, match: 5},
|
||||
{op: 'match', input: 6, match: 6},
|
||||
{op: 'match', input: 7, match: 7},
|
||||
{op: 'match', input: 8, match: 8},
|
||||
{op: 'match', input: 9, match: 9},
|
||||
{op: 'match', input: 10, match: 10}
|
||||
],
|
||||
leadTokenShift: 0,
|
||||
leadEditLength: 1,
|
||||
matchLength: baseTexts.length - 1,
|
||||
|
|
@ -321,6 +435,21 @@ describe('ContextTokenization', function() {
|
|||
// properly.
|
||||
targetTokens, {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'insert', match: 0},
|
||||
{op: 'insert', match: 1},
|
||||
{op: 'match', input: 0, match: 2},
|
||||
{op: 'match', input: 1, match: 3},
|
||||
{op: 'match', input: 2, match: 4},
|
||||
{op: 'match', input: 3, match: 5},
|
||||
{op: 'match', input: 4, match: 6},
|
||||
{op: 'match', input: 5, match: 7},
|
||||
{op: 'match', input: 6, match: 8},
|
||||
{op: 'match', input: 7, match: 9},
|
||||
{op: 'match', input: 8, match: 10},
|
||||
{op: 'match', input: 9, match: 11},
|
||||
{op: 'match', input: 10, match: 12}
|
||||
],
|
||||
leadTokenShift: 2, // "applesauce", " "
|
||||
leadEditLength: 1, // "nd" / "and"
|
||||
matchLength: baseTexts.length - 2,
|
||||
|
|
@ -360,6 +489,21 @@ describe('ContextTokenization', function() {
|
|||
const tokenization = baseTokenization.transitionTo(
|
||||
targetTokens, {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'delete', input: 0},
|
||||
{op: 'substitute', input: 1, match: 0},
|
||||
{op: 'substitute', input: 2, match: 1},
|
||||
{op: 'substitute', input: 3, match: 2},
|
||||
{op: 'substitute', input: 4, match: 3},
|
||||
{op: 'substitute', input: 5, match: 4},
|
||||
{op: 'substitute', input: 6, match: 5},
|
||||
{op: 'substitute', input: 7, match: 6},
|
||||
{op: 'substitute', input: 8, match: 7},
|
||||
{op: 'match', input: 9, match: 8},
|
||||
{op: 'match', input: 10, match: 9},
|
||||
{op: 'match', input: 11, match: 10},
|
||||
{op: 'match', input: 12, match: 11}
|
||||
],
|
||||
leadTokenShift: -1, // "'",
|
||||
leadEditLength: 1, // "t" / "n't"
|
||||
matchLength: baseTexts.length - 3,
|
||||
|
|
@ -399,6 +543,23 @@ describe('ContextTokenization', function() {
|
|||
const tokenization = baseTokenization.transitionTo(
|
||||
targetTokens, {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'delete', input: 0},
|
||||
{op: 'substitute', input: 1, match: 0},
|
||||
{op: 'substitute', input: 2, match: 1},
|
||||
{op: 'substitute', input: 3, match: 2},
|
||||
{op: 'substitute', input: 4, match: 3},
|
||||
{op: 'substitute', input: 5, match: 4},
|
||||
{op: 'substitute', input: 6, match: 5},
|
||||
{op: 'substitute', input: 7, match: 6},
|
||||
{op: 'substitute', input: 8, match: 7},
|
||||
{op: 'match', input: 9, match: 8},
|
||||
{op: 'match', input: 10, match: 9},
|
||||
{op: 'match', input: 11, match: 10},
|
||||
{op: 'match', input: 12, match: 11},
|
||||
{op: 'delete', input: 13},
|
||||
{op: 'delete', input: 14}
|
||||
],
|
||||
leadTokenShift: -1, // "'",
|
||||
leadEditLength: 1, // "t" / "n't"
|
||||
matchLength: baseTexts.length - 4,
|
||||
|
|
@ -439,6 +600,23 @@ describe('ContextTokenization', function() {
|
|||
const tokenization = baseTokenization.transitionTo(
|
||||
targetTokens, {
|
||||
canAlign: true,
|
||||
editPath: [
|
||||
{op: 'delete', input: 0},
|
||||
{op: 'substitute', input: 1, match: 0},
|
||||
{op: 'substitute', input: 2, match: 1},
|
||||
{op: 'substitute', input: 3, match: 2},
|
||||
{op: 'substitute', input: 4, match: 3},
|
||||
{op: 'substitute', input: 5, match: 4},
|
||||
{op: 'substitute', input: 6, match: 5},
|
||||
{op: 'substitute', input: 7, match: 6},
|
||||
{op: 'substitute', input: 8, match: 7},
|
||||
{op: 'match', input: 9, match: 8},
|
||||
{op: 'match', input: 10, match: 9},
|
||||
{op: 'match', input: 11, match: 10},
|
||||
{op: 'match', input: 12, match: 11},
|
||||
{op: 'insert', match: 12},
|
||||
{op: 'insert', match: 13}
|
||||
],
|
||||
leadTokenShift: 1, // "'",
|
||||
leadEditLength: 1, // "n't" / "t"
|
||||
matchLength: baseTexts.length - 1,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { assert } from 'chai';
|
||||
import { ClassicalDistanceCalculation, EditTuple, forNewIndices } from '@keymanapp/lm-worker/test-index';
|
||||
import { ClassicalDistanceCalculation, EditOperation, EditTuple, forNewIndices } from '@keymanapp/lm-worker/test-index';
|
||||
import sinon from 'sinon';
|
||||
|
||||
// Very useful for diagnosing unit test issues when path inspection is needed.
|
||||
|
|
@ -596,30 +596,30 @@ describe('Classical Damerau-Levenshtein edit-distance calculation', function() {
|
|||
|
||||
// While the two 'm's are interchangable, the path will favor
|
||||
// a result with the longest contiguous set of matches.
|
||||
let editSequences: EditTuple<string>[][] = [[
|
||||
{ op: 'match', input: 'a', match: 'a'},
|
||||
{ op: 'match', input: 'c', match: 'c'},
|
||||
{ op: 'match', input: 'c', match: 'c'},
|
||||
{ op: 'match', input: 'o', match: 'o'},
|
||||
{ op: 'insert', match: 'm'}, // insert is earlier
|
||||
{ op: 'match', input: 'm', match: 'm'}, // longer contiguous match sequence
|
||||
{ op: 'match', input: 'o', match: 'o'},
|
||||
{ op: 'match', input: 'd', match: 'd'},
|
||||
{ op: 'match', input: 'a', match: 'a'},
|
||||
{ op: 'match', input: 't', match: 't'},
|
||||
{ op: 'match', input: 'e', match: 'e'},
|
||||
let editSequences: EditTuple<EditOperation>[][] = [[
|
||||
{ op: 'match', input: 0 /* 'a' */, match: 0 /* 'a' */},
|
||||
{ op: 'match', input: 1 /* 'c' */, match: 1 /* 'c' */},
|
||||
{ op: 'match', input: 2 /* 'c' */, match: 2 /* 'c' */},
|
||||
{ op: 'match', input: 3 /* 'o' */, match: 3 /* 'o' */},
|
||||
{ op: 'insert', match: 4 /* 'm' */}, // insert is earlier
|
||||
{ op: 'match', input: 4 /* 'm' */, match: 5 /* 'm' */}, // longer contiguous match sequence
|
||||
{ op: 'match', input: 5 /* 'o' */, match: 6 /* 'o' */},
|
||||
{ op: 'match', input: 6 /* 'd' */, match: 7 /* 'd' */},
|
||||
{ op: 'match', input: 7 /* 'a' */, match: 8 /* 'a' */},
|
||||
{ op: 'match', input: 8 /* 't' */, match: 9 /* 't' */},
|
||||
{ op: 'match', input: 9 /* 'e' */, match: 10 /* 'e' */}
|
||||
], [
|
||||
{ op: 'match', input: 'a', match: 'a'},
|
||||
{ op: 'match', input: 'c', match: 'c'},
|
||||
{ op: 'match', input: 'c', match: 'c'},
|
||||
{ op: 'match', input: 'o', match: 'o'},
|
||||
{ op: 'match', input: 'm', match: 'm'},
|
||||
{ op: 'insert', match: 'm'}, // now inserting later.
|
||||
{ op: 'match', input: 'o', match: 'o'},
|
||||
{ op: 'match', input: 'd', match: 'd'},
|
||||
{ op: 'match', input: 'a', match: 'a'},
|
||||
{ op: 'match', input: 't', match: 't'},
|
||||
{ op: 'match', input: 'e', match: 'e'},
|
||||
{ op: 'match', input: 0 /* 'a' */, match: 0 /* 'a' */},
|
||||
{ op: 'match', input: 1 /* 'c' */, match: 1 /* 'c' */},
|
||||
{ op: 'match', input: 2 /* 'c' */, match: 2 /* 'c' */},
|
||||
{ op: 'match', input: 3 /* 'o' */, match: 3 /* 'o' */},
|
||||
{ op: 'match', input: 4 /* 'm' */, match: 4 /* 'm' */},
|
||||
{ op: 'insert', match: 5 /* 'm' */}, // now inserting later.
|
||||
{ op: 'match', input: 5 /* 'o' */, match: 6 /* 'o' */},
|
||||
{ op: 'match', input: 6 /* 'd' */, match: 7 /* 'd' */},
|
||||
{ op: 'match', input: 7 /* 'a' */, match: 8 /* 'a' */},
|
||||
{ op: 'match', input: 8 /* 't' */, match: 9 /* 't' */},
|
||||
{ op: 'match', input: 9 /* 'e' */, match: 10 /* 'e' */}
|
||||
]];
|
||||
|
||||
assert.sameDeepOrderedMembers(buffer.editPath(), editSequences);
|
||||
|
|
@ -629,22 +629,22 @@ describe('Classical Damerau-Levenshtein edit-distance calculation', function() {
|
|||
let buffer = compute("harras", "harass", "InputThenMatch");
|
||||
|
||||
let bestEditSequence: EditTuple<string>[] = [
|
||||
{ op: 'match', input: 'h', match: 'h'},
|
||||
{ op: 'match', input: 'a', match: 'a'},
|
||||
{ op: 'match', input: 'r', match: 'r'},
|
||||
{ op: 'substitute', input: 'r', match: 'a'},
|
||||
{ op: 'substitute', input: 'a', match: 's'},
|
||||
{ op: 'match', input: 's', match: 's'}
|
||||
{ op: 'match', input: 0, match: 0}, // h, h
|
||||
{ op: 'match', input: 1, match: 1}, // a, a
|
||||
{ op: 'match', input: 2, match: 2}, // r, r
|
||||
{ op: 'substitute', input: 3, match: 3}, // r => a
|
||||
{ op: 'substitute', input: 4, match: 4}, // a => s
|
||||
{ op: 'match', input: 5, match: 5}
|
||||
];
|
||||
|
||||
let altSequence: EditTuple<string>[] = [
|
||||
{ op: 'match', input: 'h', match: 'h'},
|
||||
{ op: 'match', input: 'a', match: 'a'},
|
||||
{ op: 'match', input: 'r', match: 'r'},
|
||||
{ op: 'delete', input: 'r'},
|
||||
{ op: 'match', input: 'a', match: 'a'},
|
||||
{ op: 'match', input: 's', match: 's'},
|
||||
{ op: 'insert', match: 's'}
|
||||
{ op: 'match', input: 0, match: 0},
|
||||
{ op: 'match', input: 1, match: 1},
|
||||
{ op: 'match', input: 2, match: 2},
|
||||
{ op: 'delete', input: 3},
|
||||
{ op: 'match', input: 4, match: 3},
|
||||
{ op: 'match', input: 5, match: 4},
|
||||
{ op: 'insert', match: 5}
|
||||
];
|
||||
|
||||
const viablePaths = buffer.editPath();
|
||||
|
|
@ -660,12 +660,12 @@ describe('Classical Damerau-Levenshtein edit-distance calculation', function() {
|
|||
let buffer = compute("access", "assess", "InputThenMatch");
|
||||
|
||||
let editSequence: EditTuple<string>[] = [
|
||||
{ op: 'match', input: 'a', match: 'a'},
|
||||
{ op: 'substitute', input: 'c', match: 's'},
|
||||
{ op: 'substitute', input: 'c', match: 's'},
|
||||
{ op: 'match', input: 'e', match: 'e'},
|
||||
{ op: 'match', input: 's', match: 's'},
|
||||
{ op: 'match', input: 's', match: 's'}
|
||||
{ op: 'match', input: 0, match: 0},
|
||||
{ op: 'substitute', input: 1, match: 1},
|
||||
{ op: 'substitute', input: 2, match: 2},
|
||||
{ op: 'match', input: 3, match: 3},
|
||||
{ op: 'match', input: 4, match: 4},
|
||||
{ op: 'match', input: 5, match: 5}
|
||||
];
|
||||
|
||||
const viablePaths = buffer.editPath();
|
||||
|
|
@ -677,10 +677,10 @@ describe('Classical Damerau-Levenshtein edit-distance calculation', function() {
|
|||
let buffer = compute("ifhs", "fish");
|
||||
|
||||
let editSequence: EditTuple<string>[] = [
|
||||
{ op: 'transpose-start', input: 'i', match: 'f'},
|
||||
{ op: 'transpose-end', input: 'f', match: 'i'},
|
||||
{ op: 'transpose-start', input: 'h', match: 's'},
|
||||
{ op: 'transpose-end', input: 's', match: 'h'}
|
||||
{ op: 'transpose-start', input: 0, match: 0},
|
||||
{ op: 'transpose-end', input: 1, match: 1},
|
||||
{ op: 'transpose-start', input: 2, match: 2},
|
||||
{ op: 'transpose-end', input: 3, match: 3}
|
||||
];
|
||||
|
||||
const viablePaths = buffer.editPath();
|
||||
|
|
@ -692,19 +692,19 @@ describe('Classical Damerau-Levenshtein edit-distance calculation', function() {
|
|||
let buffer = compute("then", "their");
|
||||
|
||||
let editSequence: EditTuple<string>[] = [
|
||||
{ op: 'match', input: 't', match: 't'},
|
||||
{ op: 'match', input: 'h', match: 'h'}, // best sequence: continguous matches
|
||||
{ op: 'match', input: 'e', match: 'e'}, // + adjacent substitutes
|
||||
{ op: 'substitute', input: 'n', match: 'i'},
|
||||
{ op: 'insert', match: 'r'},
|
||||
{ op: 'match', input: 0, match: 0},
|
||||
{ op: 'match', input: 1, match: 1}, // best sequence: continguous matches
|
||||
{ op: 'match', input: 2, match: 2}, // + adjacent substitutes
|
||||
{ op: 'substitute', input: 3, match: 3},
|
||||
{ op: 'insert', match: 4},
|
||||
];
|
||||
|
||||
let altSequence: EditTuple<string>[] = [
|
||||
{ op: 'match', input: 't', match: 't'},
|
||||
{ op: 'match', input: 'h', match: 'h'},
|
||||
{ op: 'match', input: 'e', match: 'e'},
|
||||
{ op: 'insert', match: 'i'},
|
||||
{ op: 'substitute', input: 'n', match: 'r'},
|
||||
{ op: 'match', input: 0, match: 0},
|
||||
{ op: 'match', input: 1, match: 1},
|
||||
{ op: 'match', input: 2, match: 2},
|
||||
{ op: 'insert', match: 3},
|
||||
{ op: 'substitute', input: 3, match: 4},
|
||||
];
|
||||
|
||||
const viablePaths = buffer.editPath();
|
||||
|
|
@ -718,16 +718,16 @@ describe('Classical Damerau-Levenshtein edit-distance calculation', function() {
|
|||
let buffer = compute("abczdefig", "cazefghi", "InputThenMatch", 2);
|
||||
|
||||
let editSequence: EditTuple<string>[] = [
|
||||
{ op: 'transpose-start', input: 'a', match: 'c'},
|
||||
{ op: 'transpose-delete', input: 'b'},
|
||||
{ op: 'transpose-end', input: 'c', match: 'a'},
|
||||
{ op: 'match', input: 'z', match: 'z'},
|
||||
{ op: 'delete', input: 'd'},
|
||||
{ op: 'match', input: 'e', match: 'e'},
|
||||
{ op: 'match', input: 'f', match: 'f'},
|
||||
{ op: 'transpose-start', input: 'i', match: 'g'},
|
||||
{ op: 'transpose-insert', match: 'h'},
|
||||
{ op: 'transpose-end', input: 'g', match: 'i'}
|
||||
{ op: 'transpose-start', input: 0, match: 0},
|
||||
{ op: 'transpose-delete', input: 1},
|
||||
{ op: 'transpose-end', input: 2, match: 1},
|
||||
{ op: 'match', input: 3, match: 2},
|
||||
{ op: 'delete', input: 4},
|
||||
{ op: 'match', input: 5, match: 3},
|
||||
{ op: 'match', input: 6, match: 4},
|
||||
{ op: 'transpose-start', input: 7, match: 5},
|
||||
{ op: 'transpose-insert', match: 6},
|
||||
{ op: 'transpose-end', input: 8, match: 7}
|
||||
];
|
||||
|
||||
const viablePaths = buffer.editPath();
|
||||
|
|
@ -744,39 +744,39 @@ describe('Classical Damerau-Levenshtein edit-distance calculation', function() {
|
|||
const buffer = compute(src, dst, "InputThenMatch", 4);
|
||||
|
||||
let editSequence: EditTuple<string>[] = [
|
||||
{ op: 'delete', input: 'a'},
|
||||
{ op: 'delete', input: ' '},
|
||||
{ op: 'substitute', input: 'b', match: 'q'},
|
||||
{ op: 'match', input: ' ', match: ' '},
|
||||
{ op: 'match', input: 'c', match: 'c'},
|
||||
{ op: 'match', input: ' ', match: ' '},
|
||||
{ op: 'match', input: 'd', match: 'd'},
|
||||
{ op: 'match', input: '?', match: '?'},
|
||||
{ op: 'match', input: ';', match: ';'},
|
||||
{ op: 'match', input: 'e', match: 'e'},
|
||||
{ op: 'match', input: ' ', match: ' '},
|
||||
{ op: 'match', input: 'f', match: 'f'},
|
||||
{ op: 'match', input: ' ', match: ' '},
|
||||
{ op: 'match', input: 'g', match: 'g'},
|
||||
{ op: 'match', input: ',', match: ','},
|
||||
{ op: 'match', input: ' ', match: ' '},
|
||||
{ op: 'match', input: 'h', match: 'h'},
|
||||
{ op: 'match', input: ' ', match: ' '},
|
||||
{ op: 'match', input: 'i', match: 'i'},
|
||||
{ op: 'match', input: ' ', match: ' '},
|
||||
{ op: 'match', input: 'j', match: 'j'},
|
||||
{ op: 'match', input: ' ', match: ' '},
|
||||
{ op: 'match', input: 'k', match: 'k'},
|
||||
{ op: 'match', input: ' ', match: ' '},
|
||||
{ op: 'match', input: 'l', match: 'l'},
|
||||
{ op: 'match', input: ' ', match: ' '},
|
||||
{ op: 'match', input: 'm', match: 'm'},
|
||||
{ op: 'match', input: ' ', match: ' '},
|
||||
{ op: 'match', input: 'n', match: 'n'},
|
||||
{ op: 'match', input: ' ', match: ' '},
|
||||
{ op: 'match', input: 'o', match: 'o'},
|
||||
{ op: 'match', input: ' ', match: ' '},
|
||||
{ op: 'substitute', input: 'p', match: 'r'},
|
||||
{ op: 'delete', input: 0},
|
||||
{ op: 'delete', input: 1},
|
||||
{ op: 'substitute', input: 2, match: 0},
|
||||
{ op: 'match', input: 3, match: 1},
|
||||
{ op: 'match', input: 4, match: 2},
|
||||
{ op: 'match', input: 5, match: 3},
|
||||
{ op: 'match', input: 6, match: 4},
|
||||
{ op: 'match', input: 7, match: 5},
|
||||
{ op: 'match', input: 8, match: 6},
|
||||
{ op: 'match', input: 9, match: 7},
|
||||
{ op: 'match', input: 10, match: 8},
|
||||
{ op: 'match', input: 11, match: 9},
|
||||
{ op: 'match', input: 12, match: 10},
|
||||
{ op: 'match', input: 13, match: 11},
|
||||
{ op: 'match', input: 14, match: 12},
|
||||
{ op: 'match', input: 15, match: 13},
|
||||
{ op: 'match', input: 16, match: 14},
|
||||
{ op: 'match', input: 17, match: 15},
|
||||
{ op: 'match', input: 18, match: 16},
|
||||
{ op: 'match', input: 19, match: 17},
|
||||
{ op: 'match', input: 20, match: 18},
|
||||
{ op: 'match', input: 21, match: 19},
|
||||
{ op: 'match', input: 22, match: 20},
|
||||
{ op: 'match', input: 23, match: 21},
|
||||
{ op: 'match', input: 24, match: 22},
|
||||
{ op: 'match', input: 25, match: 23},
|
||||
{ op: 'match', input: 26, match: 24},
|
||||
{ op: 'match', input: 27, match: 25},
|
||||
{ op: 'match', input: 28, match: 26},
|
||||
{ op: 'match', input: 29, match: 27},
|
||||
{ op: 'match', input: 30, match: 28},
|
||||
{ op: 'match', input: 31, match: 29},
|
||||
{ op: 'substitute', input: 32, match: 30},
|
||||
];
|
||||
|
||||
const viablePaths = buffer.editPath();
|
||||
|
|
|
|||
|
|
@ -69,13 +69,13 @@ describe('Split/merge aware edit-distance calculation', () => {
|
|||
const editPaths = calc.editPath();
|
||||
assert.equal(editPaths.length, 1);
|
||||
assert.sameDeepOrderedMembers(editPaths[0], [
|
||||
{ op: 'split', input: 'ab', match: 'a' },
|
||||
{ op: 'split', input: 'ab', match: 'b' },
|
||||
{ op: 'transpose-start', input: 'd', match: 'c' },
|
||||
{ op: 'transpose-end', input: 'c', match: 'd' },
|
||||
{ op: 'substitute', input: 'e', match: 'f' },
|
||||
{ op: 'merge', input: 'g', match: 'gh' },
|
||||
{ op: 'merge', input: 'h', match: 'gh' }
|
||||
{ op: 'split', input: 0, match: 0 },
|
||||
{ op: 'split', input: 0, match: 1 },
|
||||
{ op: 'transpose-start', input: 1, match: 2 },
|
||||
{ op: 'transpose-end', input: 2, match: 3 },
|
||||
{ op: 'substitute', input: 3, match: 4 },
|
||||
{ op: 'merge', input: 4, match: 5 },
|
||||
{ op: 'merge', input: 5, match: 5 }
|
||||
]);
|
||||
});
|
||||
|
||||
|
|
@ -91,13 +91,13 @@ describe('Split/merge aware edit-distance calculation', () => {
|
|||
const editPaths = calc.editPath();
|
||||
assert.equal(editPaths.length, 1);
|
||||
assert.sameDeepOrderedMembers(editPaths[0], [
|
||||
{ op: 'split', input: 'abc', match: 'a' },
|
||||
{ op: 'split', input: 'abc', match: 'b' },
|
||||
{ op: 'split', input: 'abc', match: 'c' },
|
||||
{ op: 'match', input: 'd', match: 'd' },
|
||||
{ op: 'merge', input: 'f', match: 'fgh' },
|
||||
{ op: 'merge', input: 'g', match: 'fgh' },
|
||||
{ op: 'merge', input: 'h', match: 'fgh' }
|
||||
{ op: 'split', input: 0, match: 0 },
|
||||
{ op: 'split', input: 0, match: 1 },
|
||||
{ op: 'split', input: 0, match: 2 },
|
||||
{ op: 'match', input: 1, match: 3 },
|
||||
{ op: 'merge', input: 2, match: 4 },
|
||||
{ op: 'merge', input: 3, match: 4 },
|
||||
{ op: 'merge', input: 4, match: 4 }
|
||||
]);
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Reference in a new issue