From 73b16ae33eb62f08247fbcb1afe0fca032f3df97 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Wed, 13 May 2020 10:41:38 -0600 Subject: [PATCH] refactor(common/lmlayer): move placeholder word breaker to @keymanapp/lexical-model-word-breakers --- .../lexical-model-word-breakers/src/index.ts | 3 ++- .../src/placeholder.ts | 21 ++++++++++++++++ .../test/test-placeholder-word-breaker.ts | 4 ++-- .../word_breaking/placeholder-word-breaker.ts | 24 ------------------- 4 files changed, 25 insertions(+), 27 deletions(-) create mode 100644 common/lexical-model-word-breakers/src/placeholder.ts delete mode 100644 common/predictive-text/worker/word_breaking/placeholder-word-breaker.ts diff --git a/common/lexical-model-word-breakers/src/index.ts b/common/lexical-model-word-breakers/src/index.ts index 22c6cd74a9..824e9aab75 100644 --- a/common/lexical-model-word-breakers/src/index.ts +++ b/common/lexical-model-word-breakers/src/index.ts @@ -1,5 +1,6 @@ import {ascii} from "./ascii-word-breaker"; +import {placeholder} from './placeholder'; import {default_} from "./default"; -export {ascii}; +export {ascii, placeholder}; export {default_ as default}; diff --git a/common/lexical-model-word-breakers/src/placeholder.ts b/common/lexical-model-word-breakers/src/placeholder.ts new file mode 100644 index 0000000000..7924b196e4 --- /dev/null +++ b/common/lexical-model-word-breakers/src/placeholder.ts @@ -0,0 +1,21 @@ +/** + * A **VERY** dumb word breaker that simply splits at words. Do not use this + * word breaker! + * + * @param phrase The phrase in which to break words. + * @deprecated Use a word breaker tailored to your language instead! + */ +export function placeholder(phrase: string): Span[] { + let nextStart = 0; + return phrase.split(/\s+/).map(utterance => { + // XXX: The indices are NOT accurate to the original phrase! + let span = { + start: nextStart, + end: nextStart + utterance.length, + text: utterance, + length: utterance.length + }; + nextStart = span.end; + return span; + }); +} diff --git a/common/lexical-model-word-breakers/test/test-placeholder-word-breaker.ts b/common/lexical-model-word-breakers/test/test-placeholder-word-breaker.ts index 75470106f7..923d60b7cb 100644 --- a/common/lexical-model-word-breakers/test/test-placeholder-word-breaker.ts +++ b/common/lexical-model-word-breakers/test/test-placeholder-word-breaker.ts @@ -1,13 +1,13 @@ /// import {assert} from 'chai'; -import {placeholder as breakWords} from '../' +import {placeholder as breakWords} from '../src'; describe('The placeholder word breaker', function () { it('should break simple English sentences', function () { let breaks = breakWords('Look! -- The quick brown fox jumps... over the lazy dog!'); let words = breaks.map(span => span.text); - assert.deepEqual(words, ['Look', 'The', 'quick', 'brown', 'fox', 'jumps', 'over', 'the', 'lazy', 'dog']); + assert.deepEqual(words, ['Look!', '--', 'The', 'quick', 'brown', 'fox', 'jumps...', 'over', 'the', 'lazy', 'dog!']); }); }); diff --git a/common/predictive-text/worker/word_breaking/placeholder-word-breaker.ts b/common/predictive-text/worker/word_breaking/placeholder-word-breaker.ts deleted file mode 100644 index 1105b89fb9..0000000000 --- a/common/predictive-text/worker/word_breaking/placeholder-word-breaker.ts +++ /dev/null @@ -1,24 +0,0 @@ -namespace wordBreakers { - - /** - * A **VERY** dumb word breaker that simply splits at words. Do not use this - * word breaker! - * - * @param phrase The phrase in which to break words. - * @deprecated Use a word breaker tailored to your language instead! - */ - export function placeholder(phrase: string): Span[] { - let nextStart = 0; - return phrase.split(/\s+/).map(utterance => { - // XXX: The indices are NOT accurate to the original phrase! - let span = { - start: nextStart, - end: nextStart + utterance.length, - text: utterance, - length: utterance.length - }; - nextStart = span.end; - return span; - }); - } -}