erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-10-17 11:27:39 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

change(web): map tokenized transform by index relative to context tail

Browse source 
With this change, the transform-tokenization methods now indicate which token (relative to the original context's tail token) each of the resulting tokenized transforms affects (and/or creates) in the output tokenization.

Fixes: #14361.
Fixes: #14480.
This commit is contained in:
Joshua Horton 2025-08-20 12:21:13 -05:00
parent 625f734223
commit ef99dd49a9
5 changed files with 338 additions and 156 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
web/src/engine/predictive-text/worker-thread/src/main/correction/context-state.ts
View file

@ -224,6 +224,8 @@ export class ContextState {
// Used to construct and represent the part of the incoming transform that
// does not land as part of the final token in the resulting context. This
// component should be preserved by any suggestions that get applied.
//
// undefined by default, which asserts we're still affecting the same token.
let preservationTransform: Transform;
// Handling for non-whitespace word boundaries - for example,
@ -235,10 +237,15 @@ export class ContextState {
preservationTransform = { insert: '', deleteLeft: 0 };
}
// Leave out the final entry!
for(let i = 0; i < transformSequenceDistribution?.[0].sample.length - 1; i++) {
const primaryInput = transformSequenceDistribution[0].sample[i];
preservationTransform = preservationTransform ? buildMergedTransform(preservationTransform, primaryInput): primaryInput;
if(transformSequenceDistribution) {
const transformKeys = [...transformSequenceDistribution[0].sample.keys()];
// Leave out the final entry - that part is replaceable by suggestions.
transformKeys.pop();
for(let i of transformKeys) {
const primaryInput = transformSequenceDistribution[0].sample.get(i);
preservationTransform = preservationTransform ? buildMergedTransform(preservationTransform, primaryInput): primaryInput;
}
}
const state = new ContextState(postContext, lexicalModel);

42
web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts
View file

@ -14,6 +14,7 @@ import { Token } from '@keymanapp/models-templates';
import { LexicalModelTypes } from '@keymanapp/common-types';
import Distribution = LexicalModelTypes.Distribution;
import LexicalModel = LexicalModelTypes.LexicalModel;
import ProbabilityMass = LexicalModelTypes.ProbabilityMass;
import Transform = LexicalModelTypes.Transform;
import TransformUtils from '../transformUtils.js';
@ -89,7 +90,7 @@ export class ContextTokenization {
lexicalModel: LexicalModel,
// FUTURE NOTE: especially for epic-dict-breaker, we'll want an array of these - to align across multiple transitions
// in case word boundaries shift back and forth.
alignedTransformDistribution: Distribution<Transform[]>
alignedTransformDistribution: Distribution<Map<number, Transform>>
): ContextTokenization {
if(!alignment.canAlign) {
return null;
@ -156,17 +157,26 @@ export class ContextTokenization {
// first index: original sample's tokenization
// second index: token index within original sample
const tokenDistribution = alignedTransformDistribution.map((entry) => {
return entry.sample.map((sample) => {
return {
sample: sample,
const remap: Map<number, ProbabilityMass<Transform>> = new Map();
for(let pair of entry.sample.entries()) {
remap.set(pair[0], {
sample: pair[1],
p: entry.p
}
});
})
}
return remap;
});
// Using these as base indices...
let tailIndex = 0;
// The original tail token should match index 0. If tokens have been deleted, that
// left-shifts our base indices; we start left of 0. If more than one token was
// edited, those edits occur to the left as well - and further left of whatever
// the new tail token is *if* tokens were removed.
const firstTailEditIndex = Math.min((1 - tailEditLength), 0) + Math.min(tailTokenShift, 0);
for(let i = 0; i < tailEditLength; i++) {
const tailIndex = firstTailEditIndex + i;
// do tail edits
const incomingIndex = i + incomingTailUpdateIndex;
const matchingIndex = i + matchingTailUpdateIndex;
@ -174,7 +184,7 @@ export class ContextTokenization {
const incomingToken = tokenizedContext[incomingIndex];
const matchedToken = this.tokens[matchingIndex];
let primaryInput = hasDistribution ? tokenizedPrimaryInput[i] : null;
let primaryInput = hasDistribution ? tokenizedPrimaryInput.get(tailIndex) : null;
const isBackspace = primaryInput && TransformUtils.isBackspace(primaryInput);
let token: ContextToken;
@ -187,11 +197,10 @@ export class ContextTokenization {
token = new ContextToken(matchedToken);
// Erase any applied-suggestion transition ID; it is no longer valid.
token.appliedTransitionId = undefined;
token.searchSpace.addInput(tokenDistribution.map((seq) => seq[tailIndex] ?? { sample: { insert: '', deleteLeft: 0 }, p: 1 }));
token.searchSpace.addInput(tokenDistribution.map((seq) => seq.get(tailIndex) ?? { sample: { insert: '', deleteLeft: 0 }, p: 1 }));
}
tokenization[incomingIndex] = token;
tailIndex++;
}
if(tailTokenShift < 0) {
@ -205,14 +214,16 @@ export class ContextTokenization {
tokenization.pop();
}
} else {
for(let i = tailEditLength; i < tailEditLength + tailTokenShift; i++) {
// First tail insertion index within the tokenized-transform map is always 1.
for(let i = 1; i <= tailTokenShift; i++) {
// create tail tokens
const incomingIndex = i + incomingTailUpdateIndex;
// same original base after all edited
const incomingIndex = incomingTailUpdateIndex + tailEditLength + (i - 1);
const incomingToken = tokenizedContext[incomingIndex];
// // Assertion: there should be no matching token; this should be a newly-appended token.
// const matchingIndex = i + tailEditLength + matchingTailUpdateIndex;
const primaryInput = hasDistribution ? tokenizedPrimaryInput[i] : null;
const primaryInput = hasDistribution ? tokenizedPrimaryInput.get(i) : null;
let pushedToken = new ContextToken(lexicalModel);
// TODO: assumes that there was no shift in wordbreaking from the
@ -228,7 +239,7 @@ export class ContextTokenization {
// tokenized `butter` `fli`. (e.g: `fli` leads to `flight`.) How do
// we know to properly relocate the `f` and `l` transforms?
let tokenDistribComponent = tokenDistribution.map((seq) => {
const entry = seq[tailIndex];
const entry = seq.get(i);
if(!entry || TransformUtils.isEmpty(entry.sample)) {
return null;
} else {
@ -250,7 +261,6 @@ export class ContextTokenization {
// Auto-replaces the search space to correspond with the new token.
tokenization.push(pushedToken);
tailIndex++;
}
}

145
web/src/engine/predictive-text/worker-thread/src/main/correction/transform-tokenization.ts
View file

@ -1,26 +1,34 @@
import { LexicalModelTypes } from '@keymanapp/common-types';
import { applyTransform, type Tokenization } from "@keymanapp/models-templates";
import { determineModelTokenizer } from '../model-helpers.js';
import Context = LexicalModelTypes.Context;
import Distribution = LexicalModelTypes.Distribution;
import LexicalModel = LexicalModelTypes.LexicalModel;
import Transform = LexicalModelTypes.Transform;
import { applyTransform, type Tokenization } from "@keymanapp/models-templates";
import { determineModelTokenizer } from '#./model-helpers.js';
import { computeAlignment } from './alignment-helpers.js';
import { KMWString } from '@keymanapp/web-utils';
/**
* Determines a tokenization-aware sequence of (`Transform`) edits, one per
* token that would result after applying the incoming keystroke's `Transform`
* to its base `Context`. This sequence reproduces the same net effect as the
* original incoming `transform` when applied in sequence.
* original incoming `transform` when applied in sequence. The component
* transforms are then indexed relative to the position of the corresponding
* token in the base `Context`, with 0 matching the original token left of
* the text insertion point, negative indices affecting previous tokens, and
* positive indicies affecting new tokens.
*
* For example, using English and standard whitespace-based tokenization:
* - context: `the quick blue`
* - transform: `{ insert: 'rown fox', deleteLeft: 3 }`
* - Resulting context: `the quick brown fox`
* - Output: `[
* { insert: 'rown', deleteLeft: 3 },
* { insert: ' ', deleteLeft: 0 },
* { insert: 'fox', deleteLeft: 0 }
* ]`
* - Output: `Map {
* 0 => { insert: 'rown', deleteLeft: 3 },
* 1 => { insert: ' ', deleteLeft: 0 },
* 2 => { insert: 'fox', deleteLeft: 0 }
* }`
* @param tokenize The tokenization function to utilize, as determined by the
* active lexical model or its settings.
* @param context The original, unmodified context
@ -31,38 +39,82 @@ export function tokenizeTransform(
tokenize: (context: Context) => Tokenization,
context: Context,
transform: Transform
): Transform[] {
): Map<number, Transform> {
if(transform.insert == '' && transform.deleteLeft == 0) {
const map = new Map<number, Transform>();
map.set(0, { insert: '', deleteLeft: 0 });
return map;
}
// Context does not slide within this function.
const postContext = applyTransform(transform, context);
const preTokenization = tokenize(context).left;
const postTokenization = tokenize(postContext).left;
if(preTokenization.length == 0) {
preTokenization.push({text: ''});
}
if(postTokenization.length == 0) {
postTokenization.push({text: ''});
}
let insert = transform.insert;
const alignment = computeAlignment(
preTokenization.map(t => t.text),
postTokenization.map(t => t.text),
false,
true
);
if(!alignment.canAlign || alignment.leadTokenShift) {
throw new Error(`Could not align context ${JSON.stringify(context)} before and after transform ${JSON.stringify(transform)}`);
}
const baseIndex = Math.min(1, 1 - alignment.tailEditLength) + Math.min(0, alignment.tailTokenShift);
let deleteLeft = transform.deleteLeft;
const tokenizedTransforms: Transform[] = [];
for(let index = postTokenization.length - 1; index >= 0; index--) {
const currentToken = postTokenization[index];
const textLen = currentToken.text.length;
// Create deletion transforms for deleted ...
for(let index = -alignment.tailTokenShift; index > 0; index--) {
const deletedToken = preTokenization.pop();
const srcLen = KMWString.length(deletedToken.text);
deleteLeft -= KMWString.length(deletedToken.text);
if(textLen < insert.length) {
tokenizedTransforms.unshift({
tokenizedTransforms.push({ insert: '', deleteLeft: srcLen });
}
let insert = transform.insert;
// Avoid emitting an empty transform if we land right at the end of a previous
// token (say, after a backspace)
if(insert.length > 0 || deleteLeft > 0) {
for(let index = postTokenization.length - 1; index >= 0; index--) {
const currentToken = postTokenization[index];
const srcLen = KMWString.length(preTokenization[index]?.text ?? '');
const curLen = KMWString.length(currentToken.text);
if(srcLen >= deleteLeft && curLen >= KMWString.length(insert)) {
tokenizedTransforms.push({
insert: insert,
deleteLeft: deleteLeft
});
break;
}
insert = insert.substring(0, insert.length - currentToken.text.length);
deleteLeft = Math.max(0, deleteLeft - srcLen);
tokenizedTransforms.push({
insert: currentToken.text,
deleteLeft: 0
deleteLeft: srcLen
});
insert = insert.substring(0, insert.length - textLen);
} else {
tokenizedTransforms.unshift({
insert: insert,
deleteLeft: transform.deleteLeft
});
break;
}
}
return tokenizedTransforms;
const returnedMap = new Map<number, Transform>();
let pushedCount: number = 0;
while(tokenizedTransforms.length > 0) {
returnedMap.set(baseIndex + pushedCount, tokenizedTransforms.pop());
pushedCount++;
}
return returnedMap;
}
/**
@ -83,7 +135,7 @@ export function tokenizeTransformDistribution(
tokenize: (context: Context) => Tokenization,
context: Context,
transformDistribution: Distribution<Transform>
): Distribution<Transform[]> {
): Distribution<Map<number, Transform>> {
return transformDistribution.map((transform) => {
return {
sample: tokenizeTransform(tokenize, context, transform.sample),
@ -97,7 +149,7 @@ export function tokenizeTransformDistribution(
* `tokenizeTransform` for each, mapping each transform to its tokenized form in
* the returned distribution.
*
* It then filters out all incoming Transforms that do not result in the same
* It then filters out all incoming Transforms that do not result in the same final
* number of tokens as the "primary input" when applied, as the context-tracker
* and predictive-text engine cannot handle word-breaking divergence well at
* this time.
@ -110,34 +162,23 @@ export function tokenizeAndFilterDistribution(
context: Context,
model: LexicalModel,
transformDistribution?: Distribution<Transform>
) {
): Distribution<Map<number, Transform>> {
let tokenize = determineModelTokenizer(model);
const inputTransform = transformDistribution?.[0];
let transformTokenLength = 0;
let tokenizedDistribution: Distribution<Transform[]> = null;
if(inputTransform) {
// These two methods apply transforms internally; do not mutate context here.
// This particularly matters for the 'distribution' variant.
// What if a pre-whitespace token has a final substitution as PART of an edit?
// Say, ['apple', ' ', ''] => ['apply', ' ', 'n']
// For now... we can't really handle that case well - modeling the 'e' => 'y' part.
// Will likely require improvements to tokenizeTransform(), which doesn't yet handle
// deleteLeft tokenization for transforms spanning tokens & whitespace.
//
// See: #14361.
// There's a good shot attemptTokenizedAlignment would be useful for it.
transformTokenLength = tokenizeTransform(tokenize, context, inputTransform.sample).length;
tokenizedDistribution = tokenizeTransformDistribution(tokenize, context, transformDistribution);
// Now we update the context used for context-state management based upon our input.
context = applyTransform(inputTransform.sample, context);
// While we lack phrase-based / phrase-oriented prediction support, we'll just extract the
// set that matches the token length that results from our input.
tokenizedDistribution = tokenizedDistribution.filter((entry) => entry.sample.length == transformTokenLength);
if(!inputTransform) {
return null;
}
return tokenizedDistribution;
// These two methods apply transforms internally; do not mutate context here.
// This particularly matters for the 'distribution' variant.
const tokenizedInputTransform = tokenizeTransform(tokenize, context, inputTransform.sample);
const lastTokenizedInputIndex = [...tokenizedInputTransform.keys()].reverse()[0];
const tokenizedDistribution = tokenizeTransformDistribution(tokenize, context, transformDistribution);
// While we lack phrase-based / phrase-oriented prediction support, we'll just extract the
// set that matches the token length that results from our input.
return tokenizedDistribution.filter((entry) =>
entry.sample.has(lastTokenizedInputIndex) && !entry.sample.has(lastTokenizedInputIndex + 1)
);
}

23
web/src/test/auto/headless/engine/predictive-text/worker-thread/context/context-tokenization.tests.ts
View file

@ -12,9 +12,11 @@ import { assert } from 'chai';
import { default as defaultBreaker } from '@keymanapp/models-wordbreakers';
import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs';
import { LexicalModelTypes } from '@keymanapp/common-types';
import { ContextToken, ContextTokenization, models } from '@keymanapp/lm-worker/test-index';
import Transform = LexicalModelTypes.Transform;
import TrieModel = models.TrieModel;
var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'),
@ -101,6 +103,10 @@ describe('ContextTokenization', function() {
const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t)), null);
const targetTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day', ' ', ''].map((t) => ({text: t, isWhitespace: t == ' '}));
const inputTransformMap: Map<number, Transform> = new Map();
inputTransformMap.set(1, { insert: ' ', deleteLeft: 0 });
inputTransformMap.set(2, { insert: '', deleteLeft: 0 });
const tokenization = baseTokenization.transitionTo(
targetTokens, {
canAlign: true,
@ -110,7 +116,7 @@ describe('ContextTokenization', function() {
tailTokenShift: 2
},
plainModel,
[{ sample: [{ insert: ' ', deleteLeft: 0 }, { insert: '', deleteLeft: 0 }], p: 1}]
[{ sample: inputTransformMap, p: 1}]
);
assert.isOk(tokenization);
@ -125,6 +131,9 @@ describe('ContextTokenization', function() {
const baseTokenization = new ContextTokenization(baseTokens.map(t => toToken(t)), null);
const targetTokens = ['an', ' ', 'apple', ' ', 'a', ' ', 'day'].map((t) => ({text: t, isWhitespace: t == ' '}));
const inputTransformMap: Map<number, Transform> = new Map();
inputTransformMap.set(0, { insert: 'y', deleteLeft: 0 });
const tokenization = baseTokenization.transitionTo(
targetTokens, {
canAlign: true,
@ -134,7 +143,7 @@ describe('ContextTokenization', function() {
tailTokenShift: 0
},
plainModel,
[{ sample: [{ insert: 'y', deleteLeft: 0 }], p: 1}]
[{ sample: inputTransformMap, p: 1}]
);
assert.isOk(tokenization);
@ -160,6 +169,9 @@ describe('ContextTokenization', function() {
"seem", " ", "like", " ", "they'd", " ", "make", " ", "for", " ", "the", " ", "b"
];
const targetTokens = targetTexts.map((t) => ({text: t, isWhitespace: t == ' '}));
const inputTransformMap: Map<number, Transform> = new Map();
inputTransformMap.set(0, { insert: 'b', deleteLeft: 0 });
const tokenization = baseTokenization.transitionTo(
targetTokens, {
canAlign: true,
@ -169,7 +181,7 @@ describe('ContextTokenization', function() {
tailTokenShift: 0
},
plainModel,
[{ sample: [{ insert: 'b', deleteLeft: 0 }], p: 1}]
[{ sample: inputTransformMap, p: 1}]
);
assert.isOk(tokenization);
@ -192,6 +204,9 @@ describe('ContextTokenization', function() {
"like", " ", "they'd", " ", "make", " ", "for", " ", "the", " ", "best", " ", "break"
];
const targetTokens = targetTexts.map((t) => ({text: t, isWhitespace: t == ' '}));
const inputTransformMap: Map<number, Transform> = new Map();
inputTransformMap.set(0, { insert: 'k', deleteLeft: 0 });
const tokenization = baseTokenization.transitionTo(
targetTokens, {
canAlign: true,
@ -201,7 +216,7 @@ describe('ContextTokenization', function() {
tailTokenShift: 0
},
plainModel,
[{ sample: [{ insert: 'k', deleteLeft: 0 }], p: 1}]
[{ sample: inputTransformMap, p: 1}]
);
assert.isOk(tokenization);

269
web/src/test/auto/headless/engine/predictive-text/worker-thread/transform-tokenization.tests.ts
View file

@ -7,11 +7,35 @@ import { LexicalModelTypes } from '@keymanapp/common-types';
import { tokenizeTransform } from '@keymanapp/lm-worker/test-index';
import Context = LexicalModelTypes.Context;
import Transform = LexicalModelTypes.Transform;
const defaultTokenize = (context: Context) => tokenize(defaultBreaker, context);
describe('tokenizeTransform', () => {
describe('with default wordbreaking', () => {
it('produces a single empty transform at index 0 when an empty transform is input', () => {
const context: Context = {
left: 'an apple a date',
right: '',
startOfBuffer: true,
endOfBuffer: true
};
const editTransform = {
insert: '',
deleteLeft: 0
};
const result = tokenizeTransform(
defaultTokenize,
context,
editTransform
);
assert.equal(result.size, 1);
assert.deepEqual(result.get(0), editTransform);
});
it('properly handles simple token-edit transform', () => {
const context: Context = {
left: 'an apple a date',
@ -31,8 +55,8 @@ describe('tokenizeTransform', () => {
editTransform
);
assert.equal(result.length, 1);
assert.deepEqual(result[0], editTransform);
assert.equal(result.size, 1);
assert.deepEqual(result.get(0), editTransform);
});
it('properly handles simple token-replacing transform', () => {
@ -54,8 +78,8 @@ describe('tokenizeTransform', () => {
editTransform
);
assert.equal(result.length, 1);
assert.deepEqual(result[0], editTransform);
assert.equal(result.size, 1);
assert.deepEqual(result.get(0), editTransform);
});
it('handles simple token-replacing transform with cross-token deleteLeft', () => {
@ -69,7 +93,7 @@ describe('tokenizeTransform', () => {
// 'an apple any'
const editTransform = {
insert: 'ny',
deleteLeft: 6
deleteLeft: 5
};
const result = tokenizeTransform(
@ -78,8 +102,22 @@ describe('tokenizeTransform', () => {
editTransform
);
assert.equal(result.length, 1);
assert.deepEqual(result[0], editTransform);
const expectedMap = new Map<number, Transform>();
expectedMap.set(-2, {
insert: 'ny',
deleteLeft: 0
});
expectedMap.set(-1, {
insert: '',
deleteLeft: 1
});
expectedMap.set(0, {
insert: '',
deleteLeft: 4
});
assert.equal(result.size, expectedMap.size);
assert.deepEqual(result, expectedMap);
});
it('properly handles a simple appended whitespace', () => {
@ -101,14 +139,15 @@ describe('tokenizeTransform', () => {
editTransform
);
assert.equal(result.length, 2);
assert.deepEqual(result, [
// The whitespace belongs on the whitespace token that will be added.
editTransform,
// The default-breaker adds an empty token after whitespace, hence this
// empty transform.
{ insert: '', deleteLeft: 0 }
]);
const expectedMap = new Map<number, Transform>();
// The whitespace belongs on the whitespace token that will be added.
expectedMap.set(1, editTransform);
// The default-breaker adds an empty token after whitespace, hence this
// empty transform.
expectedMap.set(2, { insert: '', deleteLeft: 0 });
assert.equal(result.size, expectedMap.size);
assert.deepEqual(result, expectedMap);
});
it('properly handles a simple appended period', () => {
@ -132,10 +171,40 @@ describe('tokenizeTransform', () => {
// The default wordbreaker does not (currently) append a blank token
// after standard English punctuation.
assert.equal(result.length, 1);
assert.deepEqual(result, [
const expectedMap = new Map<number, Transform>();
expectedMap.set(1, editTransform);
assert.equal(result.size, expectedMap.size);
assert.deepEqual(result, expectedMap);
});
it('properly deletes a simple appended whitespace', () => {
const context = {
left: 'an apple a day ',
right: '',
startOfBuffer: true,
endOfBuffer: true
};
const editTransform = {
insert: '',
deleteLeft: 1
};
const result = tokenizeTransform(
defaultTokenize,
context,
editTransform
]);
);
const expectedMap = new Map<number, Transform>();
// The whitespace belongs on the whitespace token that will be added.
expectedMap.set(-1, editTransform);
// The default-breaker adds an empty token after whitespace, hence this
// empty transform.
expectedMap.set(0, { insert: '', deleteLeft: 0 });
assert.equal(result.size, expectedMap.size);
assert.deepEqual(result, expectedMap);
});
it('handles word-breakable transforms (case 1)', () => {
@ -157,12 +226,15 @@ describe('tokenizeTransform', () => {
editTransform
);
assert.equal(result.length, 3);
assert.deepEqual(result, [
{ insert: 'y', deleteLeft: 1 },
{ insert: ' ', deleteLeft: 0 },
{ insert: 'k', deleteLeft: 0 }
]);
const expectedMap = new Map<number, Transform>();
// dat => day
expectedMap.set(0, { insert: 'y', deleteLeft: 1 });
// new whitespace
expectedMap.set(1, { insert: ' ', deleteLeft: 0 });
// new 'k' token
expectedMap.set(2, { insert: 'k', deleteLeft: 0 });
assert.equal(result.size, expectedMap.size);
assert.deepEqual(result, expectedMap);
});
it('handles word-breakable transforms (case 2)', () => {
@ -184,18 +256,18 @@ describe('tokenizeTransform', () => {
editTransform
);
assert.equal(result.length, 4);
assert.deepEqual(result, [
{ insert: 'y', deleteLeft: 1 },
{ insert: '.', deleteLeft: 0 },
{ insert: ' ', deleteLeft: 0 },
{ insert: '', deleteLeft: 0 }
]);
const expectedMap = new Map<number, Transform>();
expectedMap.set(0, { insert: 'y', deleteLeft: 1 });
expectedMap.set(1, { insert: '.', deleteLeft: 0 });
expectedMap.set(2, { insert: ' ', deleteLeft: 0 });
expectedMap.set(3, { insert: '', deleteLeft: 0 });
assert.equal(result.size, 4);
assert.deepEqual(result, expectedMap);
});
it('handles complex breakable cases', () => {
const context = {
left: 'an apple a date',
left: 'an apple as date',
right: '',
startOfBuffer: true,
endOfBuffer: true
@ -213,22 +285,31 @@ describe('tokenizeTransform', () => {
editTransform
);
assert.equal(result.length, 3);
assert.deepEqual(result, [
{ insert: 'ny', deleteLeft: 6 },
{ insert: ' ', deleteLeft: 0 },
{ insert: 'day', deleteLeft: 0 }
]);
const expectedMap = new Map<number, Transform>();
// as => any
expectedMap.set(-2, { insert: 'ny', deleteLeft: 1 }); // 2 back from the last token before the text insertion point.
// ' ' replaced with another ' ' (but still edited)
expectedMap.set(-1, { insert: ' ', deleteLeft: 1 });
// date => day, but with full replacement due to the large deleteLeft.
expectedMap.set( 0, { insert: 'day', deleteLeft: 4 }); // The original token before the text insertion point.
assert.equal(result.size, expectedMap.size);
assert.deepEqual(result, expectedMap);
});
});
describe('with mocked dictionary-based wordbreaking', () => {
function mockedTokenization(entries: string[]) {
return {
left: entries.map((text) => {
return {text: text} as Token
})
} as Tokenization;
function mockedTokenization(map: Map<string, string[]>) {
return (context: Context) => {
let tokens = map.get(context.left);
if(!tokens) {
assert.fail("Mocked tokenization was not properly constructed");
}
return {
left: tokens.map((text) => {
return {text: text} as Token
})
} as Tokenization;
}
}
it('properly handles simple token-edit transform', () => {
@ -244,14 +325,19 @@ describe('tokenizeTransform', () => {
deleteLeft: 2
};
const mockMap = new Map<string, string[]>();
mockMap.set('anappleadate', ['an', 'apple', 'a', 'date']);
mockMap.set('anappleaday', ['an', 'apple', 'a', 'day']);
const result = tokenizeTransform(
() => mockedTokenization(['an', 'apple', 'a', 'day']),
mockedTokenization(mockMap),
context,
editTransform
);
assert.equal(result.length, 1);
assert.deepEqual(result[0], editTransform);
const expectedMap = new Map<number, Transform>();
expectedMap.set(0, editTransform); // The original token before the text insertion point.
assert.equal(result.size, expectedMap.size);
assert.deepEqual(result, expectedMap);
});
it('properly handles simple token-replacing transform', () => {
@ -267,14 +353,19 @@ describe('tokenizeTransform', () => {
deleteLeft: 4
};
const mockMap = new Map<string, string[]>();
mockMap.set('anappleadate', ['an', 'apple', 'a', 'date']);
mockMap.set('anappleaweek', ['an', 'apple', 'a', 'week']);
const result = tokenizeTransform(
() => mockedTokenization(['an', 'apple', 'a', 'week']),
mockedTokenization(mockMap),
context,
editTransform
);
assert.equal(result.length, 1);
assert.deepEqual(result[0], editTransform);
const expectedMap = new Map<number, Transform>();
expectedMap.set(0, editTransform); // The original token before the text insertion point.
assert.equal(result.size, expectedMap.size);
assert.deepEqual(result, expectedMap);
});
it('handles simple token-replacing transform with cross-token deleteLeft', () => {
@ -288,17 +379,23 @@ describe('tokenizeTransform', () => {
// 'an apple any'
const editTransform = {
insert: 'ny',
deleteLeft: 5
deleteLeft: 4
};
const mockMap = new Map<string, string[]>();
mockMap.set('anappleadate', ['an', 'apple', 'a', 'date']);
mockMap.set('anappleany', ['an', 'apple', 'any']);
const result = tokenizeTransform(
() => mockedTokenization(['an', 'apple', 'any']),
mockedTokenization(mockMap),
context,
editTransform
);
assert.equal(result.length, 1);
assert.deepEqual(result[0], editTransform);
const expectedMap = new Map<number, Transform>();
expectedMap.set(-1, { insert: 'ny', deleteLeft: 0 });
expectedMap.set( 0, { insert: '', deleteLeft: 4 });
assert.equal(result.size, expectedMap.size);
assert.deepEqual(result, expectedMap);
});
it('handles word-breakable transforms (case 1)', () => {
@ -314,17 +411,20 @@ describe('tokenizeTransform', () => {
deleteLeft: 1
};
const mockMap = new Map<string, string[]>();
mockMap.set('anappleadat', ['an', 'apple', 'a', 'dat']);
mockMap.set('anappleadayk', ['an', 'apple', 'a', 'day', 'k']);
const result = tokenizeTransform(
() => mockedTokenization(['an', 'apple', 'any', 'k']),
mockedTokenization(mockMap),
context,
editTransform
);
assert.equal(result.length, 2);
assert.deepEqual(result, [
{ insert: 'y', deleteLeft: 1 },
{ insert: 'k', deleteLeft: 0 }
]);
const expectedMap = new Map<number, Transform>();
expectedMap.set(0, { insert: 'y', deleteLeft: 1 });
expectedMap.set(1, { insert: 'k', deleteLeft: 0 });
assert.equal(result.size, expectedMap.size);
assert.deepEqual(result, expectedMap);
});
it('handles word-breakable transforms (case 2)', () => {
@ -340,17 +440,20 @@ describe('tokenizeTransform', () => {
deleteLeft: 1
};
const mockMap = new Map<string, string[]>();
mockMap.set('anappleadat', ['an', 'apple', 'a', 'dat']);
mockMap.set('anappleaday.', ['an', 'apple', 'a', 'day', '.']);
const result = tokenizeTransform(
() => mockedTokenization(['an', 'apple', 'any', 'day', '.']),
mockedTokenization(mockMap),
context,
editTransform
);
assert.equal(result.length, 2);
assert.deepEqual(result, [
{ insert: 'y', deleteLeft: 1 },
{ insert: '.', deleteLeft: 0 }
]);
const expectedMap = new Map<number, Transform>();
expectedMap.set(0, { insert: 'y', deleteLeft: 1 });
expectedMap.set(1, { insert: '.', deleteLeft: 0 });
assert.equal(result.size, expectedMap.size);
assert.deepEqual(result, expectedMap);
});
it('handles word-breakable transforms (case 2 alternate output)', () => {
@ -366,18 +469,21 @@ describe('tokenizeTransform', () => {
deleteLeft: 1
};
const mockMap = new Map<string, string[]>();
mockMap.set('anappleadat', ['an', 'apple', 'a', 'dat']);
mockMap.set('anappleaday.', ['an', 'apple', 'a', 'day', '.', '']);
const result = tokenizeTransform(
() => mockedTokenization(['an', 'apple', 'any', 'day', '.', '']),
mockedTokenization(mockMap),
context,
editTransform
);
assert.equal(result.length, 3);
assert.deepEqual(result, [
{ insert: 'y', deleteLeft: 1 },
{ insert: '.', deleteLeft: 0 },
{ insert: '', deleteLeft: 0 }
]);
const expectedMap = new Map<number, Transform>();
expectedMap.set(0, { insert: 'y', deleteLeft: 1 });
expectedMap.set(1, { insert: '.', deleteLeft: 0 });
expectedMap.set(2, { insert: '', deleteLeft: 0});
assert.equal(result.size, expectedMap.size);
assert.deepEqual(result, expectedMap);
});
it('handles complex breakable cases', () => {
@ -391,20 +497,23 @@ describe('tokenizeTransform', () => {
// 'an apple any'
const editTransform = {
insert: 'nyday',
deleteLeft: 5
deleteLeft: 4
};
const mockMap = new Map<string, string[]>();
mockMap.set('anappleadate', ['an', 'apple', 'a', 'date']);
mockMap.set('anappleanyday', ['an', 'apple', 'any', 'day']);
const result = tokenizeTransform(
() => mockedTokenization(['an', 'apple', 'any', 'day']),
mockedTokenization(mockMap),
context,
editTransform
);
assert.equal(result.length, 2);
assert.deepEqual(result, [
{ insert: 'ny', deleteLeft: 5 },
{ insert: 'day', deleteLeft: 0 }
]);
const expectedMap = new Map<number, Transform>();
expectedMap.set(-1, { insert: 'ny', deleteLeft: 0 });
expectedMap.set( 0, { insert: 'day', deleteLeft: 4 });
assert.equal(result.size, expectedMap.size);
assert.deepEqual(result, expectedMap);
});
});
});