refactor(common/lmlayer): move placeholder word breaker to @keymanapp/lexical-model-word-breakers

This commit is contained in:
Eddie Antonio Santos 2020-05-13 10:41:38 -06:00
parent 6aabd10c70
commit 73b16ae33e
No known key found for this signature in database
GPG key ID: DD411EA4D9509D87
4 changed files with 25 additions and 27 deletions

View file

@ -1,5 +1,6 @@
import {ascii} from "./ascii-word-breaker";
import {placeholder} from './placeholder';
import {default_} from "./default";
export {ascii};
export {ascii, placeholder};
export {default_ as default};

View file

@ -0,0 +1,21 @@
/**
* A **VERY** dumb word breaker that simply splits at words. Do not use this
* word breaker!
*
* @param phrase The phrase in which to break words.
* @deprecated Use a word breaker tailored to your language instead!
*/
export function placeholder(phrase: string): Span[] {
let nextStart = 0;
return phrase.split(/\s+/).map(utterance => {
// XXX: The indices are NOT accurate to the original phrase!
let span = {
start: nextStart,
end: nextStart + utterance.length,
text: utterance,
length: utterance.length
};
nextStart = span.end;
return span;
});
}

View file

@ -1,13 +1,13 @@
/// <reference types="@keymanapp/lexical-model-types" />
import {assert} from 'chai';
import {placeholder as breakWords} from '../'
import {placeholder as breakWords} from '../src';
describe('The placeholder word breaker', function () {
it('should break simple English sentences', function () {
let breaks = breakWords('Look! -- The quick brown fox jumps... over the lazy dog!');
let words = breaks.map(span => span.text);
assert.deepEqual(words, ['Look', 'The', 'quick', 'brown', 'fox', 'jumps', 'over', 'the', 'lazy', 'dog']);
assert.deepEqual(words, ['Look!', '--', 'The', 'quick', 'brown', 'fox', 'jumps...', 'over', 'the', 'lazy', 'dog!']);
});
});

View file

@ -1,24 +0,0 @@
namespace wordBreakers {
/**
* A **VERY** dumb word breaker that simply splits at words. Do not use this
* word breaker!
*
* @param phrase The phrase in which to break words.
* @deprecated Use a word breaker tailored to your language instead!
*/
export function placeholder(phrase: string): Span[] {
let nextStart = 0;
return phrase.split(/\s+/).map(utterance => {
// XXX: The indices are NOT accurate to the original phrase!
let span = {
start: nextStart,
end: nextStart + utterance.length,
text: utterance,
length: utterance.length
};
nextStart = span.end;
return span;
});
}
}