feat(common/lmlayer): compile join decorator

This commit is contained in:
Eddie Antonio Santos 2020-04-27 16:19:25 -06:00
parent 9bdc1b7f0e
commit 438a90fa4f
No known key found for this signature in database
GPG key ID: DD411EA4D9509D87
2 changed files with 16 additions and 1 deletions

View file

@ -93,11 +93,19 @@ export class ModelSourceError extends Error {
* breaking function.
*/
function compileWordBreaker(spec: WordBreakerSpec): string {
return compileInnerWordBreaker(spec.use);
let baseWordBreakerCode = compileInnerWordBreaker(spec.use);
if (spec.joinWordsAt == undefined) {
// No need to decorate; return it as-is
return baseWordBreakerCode;
}
// Let's decorate it with the join word breaker!
return `wordBreakers.join_(${baseWordBreakerCode},${JSON.stringify(spec.joinWordsAt)})`;
}
/**
* Compiles the base word breaker, that may be decorated later.
* Returns the source code of a JavaScript expression.
*/
function compileInnerWordBreaker(spec: SimpleWordBreakerSpec): string {
if (typeof spec === "string") {

View file

@ -17,6 +17,13 @@ interface LexicalModelDeclaration {
*/
interface WordBreakerSpec {
readonly use: SimpleWordBreakerSpec;
/**
* If present, joins words that were split by the word breaker
* together at the given strings. e.g.,
*
* joinWordsAt: ['-'] // to keep hyphenated items together
*/
readonly joinWordsAt?: string[];
}
/**