erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-11-08 08:57:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

fix(web): adjust tokenization unit test expectations to match

Browse source
This commit is contained in:
Joshua Horton 2026-05-08 15:45:19 -05:00
parent 4cfac7b19a
commit ef8593b049
2 changed files with 27 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
web/src/test/auto/headless/engine/predictive-text/templates/tokenization.tests.ts
View file

@ -175,7 +175,7 @@ describe('Tokenization functions', function() {
});
it('properly handles empty-context cases', function() {
// Wordbreaking on a empty space => no word.
// Wordbreaking on a empty space => no word, but empty initial token.
let context = {
left: '', startOfBuffer: true,
right: '', endOfBuffer: true
@ -184,7 +184,7 @@ describe('Tokenization functions', function() {
let tokenization = models.tokenize(wordBreakers.default, context);
let expectedResult: models.Tokenization = {
left: [],
left: [{text: '', isWhitespace: false}],
right: [],
caretSplitsToken: false
};
@ -193,11 +193,11 @@ describe('Tokenization functions', function() {
});
it('properly handles null context cases', function() {
// Wordbreaking on a empty space => no word.
// Wordbreaking on a empty space => no word, but empty initial token.
let tokenization = models.tokenize(wordBreakers.default, null);
let expectedResult: models.Tokenization = {
left: [],
left: [{text: '', isWhitespace: false}],
right: [],
caretSplitsToken: false
};

33
web/src/test/auto/headless/engine/predictive-text/worker-thread/worker-custom-punctuation.tests.ts
View file

@ -87,17 +87,30 @@ describe('Custom Punctuation', function () {
// the tests run smoothly.
wordbreaker: (text) => {
const textLen = text.length;
if(text.charAt(textLen - 1) == " ") {
return [
{text: text.substring(0, 1), start: 0, end: 1, length: 1},
{text: text.substring(1, textLen-2), start: 1, end: textLen-1, length: textLen-2},
{text: text.substring(textLen-1), start: textLen-1, end: textLen, length: 1}
];
if(text.charAt(0) == "᚛") { // ensure the prior token component (the '᚛') wordbreaks.
if(text.charAt(textLen - 1) == " ") { // ensure the insert-after component word-breaks.
return [
{text: text.substring(0, 1), start: 0, end: 1, length: 1},
{text: text.substring(1, textLen-2), start: 1, end: textLen-1, length: textLen-2},
{text: text.substring(textLen-1), start: textLen-1, end: textLen, length: 1}
];
} else {
return [
{text: text.substring(0, 1), start: 0, end: 1, length: 1},
{text: text.substring(1), start: 1, end: textLen, length: textLen-1}
];
}
} else {
return [
{text: text.substring(0, 1), start: 0, end: 1, length: 1},
{text: text.substring(1), start: 1, end: textLen, length: textLen-1}
];
if(text.charAt(textLen - 1) == " ") {
return [
{text: text.substring(0, textLen-2), start: 0, end: textLen-1, length: textLen-1},
{text: text.substring(textLen-1), start: textLen-1, end: textLen, length: 1}
];
} else {
return [
{text: text, start: 0, end: textLen, length: textLen}
];
}
}
}
});