fix(common/models): forgot DummyModel.tokenize

This commit is contained in:
jahorton 2020-09-01 08:45:24 +07:00
parent 2953a909e6
commit af4e03662d

View file

@ -82,6 +82,11 @@ namespace models {
}
}
tokenize(context: Context): USVString[] {
let words = wordBreakers.default_(context.left);
return words.map(span => span.text);
}
wordbreak(context: Context): USVString {
let words = wordBreakers.default_(context.left);
if (words.length > 0) {