From cf8cbbfff4f60fd63e1efdf7c55cf5b6d70d52e2 Mon Sep 17 00:00:00 2001 From: jahorton Date: Wed, 14 Oct 2020 12:32:25 +0700 Subject: [PATCH] refactor(common/models): refactors old wordbreak tests --- common/models/templates/src/tokenization.ts | 8 +- .../templates/test/test-tokenization.js | 89 ++++++++++++ .../headless/default-word-breaker.js | 130 ------------------ 3 files changed, 96 insertions(+), 131 deletions(-) delete mode 100644 common/predictive-text/unit_tests/headless/default-word-breaker.js diff --git a/common/models/templates/src/tokenization.ts b/common/models/templates/src/tokenization.ts index 8826818a28..a49cc6b975 100644 --- a/common/models/templates/src/tokenization.ts +++ b/common/models/templates/src/tokenization.ts @@ -24,7 +24,13 @@ namespace models { } export function tokenize(wordBreaker: WordBreakingFunction, context: Context): Tokenization { - let leftSpans = wordBreaker(context.left) || []; + context = context || { + left: undefined, + startOfBuffer: undefined, + endOfBuffer: undefined + }; + + let leftSpans = wordBreaker(context.left || '') || []; let rightSpans = wordBreaker(context.right || '') || []; let tokenization: Tokenization = { diff --git a/common/models/templates/test/test-tokenization.js b/common/models/templates/test/test-tokenization.js index df25bff7e7..b6dee76747 100644 --- a/common/models/templates/test/test-tokenization.js +++ b/common/models/templates/test/test-tokenization.js @@ -27,6 +27,25 @@ describe('Tokenization functions', function() { assert.deepEqual(tokenization, expectedResult); }); + it('tokenizes English using defaults, pre-whitespace caret, partial context', function() { + let context = { + left: "quick brown fox", // No "The" + right: " jumped over the lazy", // No "dog" + startOfBuffer: false, + endOfBuffer: false + }; + + let tokenization = models.tokenize(wordBreakers.default, context); + + let expectedResult = { + left: ['quick', 'brown', 'fox'], + right: ['jumped', 'over', 'the', 'lazy'], + caretSplitsToken: false + }; + + assert.deepEqual(tokenization, expectedResult); + }); + it('tokenizes English using defaults, post-whitespace caret', function() { let context = { left: "The quick brown fox ", @@ -48,6 +67,27 @@ describe('Tokenization functions', function() { assert.deepEqual(tokenization, expectedResult); }); + it('tokenizes English using defaults, post-whitespace caret, partial context', function() { + let context = { + left: "quick brown fox ", + right: "jumped over the lazy", + startOfBuffer: false, + endOfBuffer: false + }; + + let tokenization = models.tokenize(wordBreakers.default, context); + + // Technically, we're editing the start of the first token on the right + // when in this context. + let expectedResult = { + left: ['quick', 'brown', 'fox', ''], + right: ['jumped', 'over', 'the', 'lazy'], + caretSplitsToken: true + }; + + assert.deepEqual(tokenization, expectedResult); + }); + it('tokenizes English using defaults, splitting caret', function() { let context = { left: "The quick brown fox jum", @@ -67,6 +107,55 @@ describe('Tokenization functions', function() { assert.deepEqual(tokenization, expectedResult); }); + it('empty context case', function() { + // Wordbreaking on a empty space => no word. + let context = { + left: '', startOfBuffer: true, + right: '', endOfBuffer: true + }; + + let tokenization = models.tokenize(wordBreakers.default, context); + + let expectedResult = { + left: [], + right: [], + caretSplitsToken: false + }; + + assert.deepEqual(tokenization, expectedResult); + }); + + it('nil context case', function() { + // Wordbreaking on a empty space => no word. + let tokenization = models.tokenize(wordBreakers.default, null); + + let expectedResult = { + left: [], + right: [], + caretSplitsToken: false + }; + + assert.deepEqual(tokenization, expectedResult); + }); + + it('near-empty context: one space before caret', function() { + // Wordbreaking on a empty space => no word. + let context = { + left: ' ', startOfBuffer: true, + right: '', endOfBuffer: true + }; + + let tokenization = models.tokenize(wordBreakers.default, context); + + let expectedResult = { + left: [''], + right: [], + caretSplitsToken: false + }; + + assert.deepEqual(tokenization, expectedResult); + }); + // For the next few tests: a mocked wordbreaker for Khmer, a language // without whitespace between words. let mockedKhmerBreaker = function(text) { diff --git a/common/predictive-text/unit_tests/headless/default-word-breaker.js b/common/predictive-text/unit_tests/headless/default-word-breaker.js deleted file mode 100644 index 613336458e..0000000000 --- a/common/predictive-text/unit_tests/headless/default-word-breaker.js +++ /dev/null @@ -1,130 +0,0 @@ -/** - * Smoke-test the default word breaker. - */ -var assert = require('chai').assert; -var TrieModel = require('../../build/intermediate').models.TrieModel; - -var breakWords = require('../../build/intermediate').wordBreakers['default']; -const SHY = '\u00AD'; - -// The following are **integration tests** testing the interaction between the -// Trie model "template" and the default word breaker. -// -// They exercise whether the default word breaker produces the correct word -// breaks, and whether the Trie model can consume the word breaks properly. -describe('The default word breaker', function () { - it('recognizes a word at end of complete lefthand context', function () { - var model = new TrieModel(jsonFixture('tries/english-1000'), { - wordBreaker: breakWords // wordBreakers['default'] when fully integrated. - }); - - // Standard case - wordbreaking at the end of a word. - var context = { - left: 'The quick brown fox jumped', startOfBuffer: true, - right: ' over the lazy dog.', endOfBuffer: true - }; - - var broken = model.wordbreak(context); - - assert.strictEqual(broken, 'jumped'); - }); - - // Same test as before, but we want to be sure the start/end of buffer flags - // don't affect our results. - it('recognizes a word at end of incomplete lefthand context', function () { - var model = new TrieModel(jsonFixture('tries/english-1000'), { - wordBreaker: breakWords - }); - - // Standard case - wordbreaking at the end of a word. - var context = { - left: 'The quick brown fox jumped', startOfBuffer: false, - right: ' over the lazy dog.', endOfBuffer: false - }; - - var broken = model.wordbreak(context); - - assert.strictEqual(broken, 'jumped'); - }); - - it('returns text for a word in-progress', function() { - var model = new TrieModel(jsonFixture('tries/english-1000'), { - wordBreaker: breakWords - }); - - // Standard case - midword (xylophone) call - var context = { - left: 'xyl', startOfBuffer: true, - right: '', endOfBuffer: true - }; - - var broken = model.wordbreak(context); - - assert.strictEqual(broken, 'xyl'); - }); - - it('returns empty string when called without word text', function() { - var model = new TrieModel(jsonFixture('tries/english-1000'), { - wordBreaker: breakWords - }); - - // Wordbreaking on a empty space => no word. - context = { - left: 'The quick brown fox jumped ', startOfBuffer: true, - right: 'over the lazy dog.', endOfBuffer: true - }; - - broken = model.wordbreak(context); - - assert.strictEqual(broken, ''); - }); - - it('returns empty string when called with empty context', function() { - var model = new TrieModel(jsonFixture('tries/english-1000'), { - wordBreaker: breakWords - }); - - // Wordbreaking on a empty space => no word. - context = { - left: '', startOfBuffer: true, - right: '', endOfBuffer: true - }; - - broken = model.wordbreak(context); - - assert.strictEqual(broken, ''); - }); - - it('returns empty string when called with nil context', function() { - var model = new TrieModel(jsonFixture('tries/english-1000'), { - wordBreaker: breakWords - }); - - // Wordbreaking on a empty space => no word. - context = { - left: '', startOfBuffer: false, - right: '', endOfBuffer: false - }; - - broken = model.wordbreak(context); - - assert.strictEqual(broken, ''); - }); - - it.skip('correctly breaks a word when the caret is placed within it', function() { - var model = new TrieModel(jsonFixture('tries/english-1000'), { - wordBreaker: breakWords - }); - - // A limitation of the current implementation; we should fix this before release. - // Then again, when typing this is probably fine; just not when not typing. - context = { - left: 'The quick brown fox jum', startOfBuffer: true, - right: 'ped over the lazy dog.', endOfBuffer: true - }; - - broken = model.wordbreak(context); - - assert.strictEqual(broken, 'jumped'); // Current result: 'jum'. - }); -});