refactor(common/models): refactors old wordbreak tests

common/models/templates/src/tokenization.ts

@@ -24,7 +24,13 @@ namespace models {
   }
 
   export function tokenize(wordBreaker: WordBreakingFunction, context: Context): Tokenization {
-    let leftSpans  = wordBreaker(context.left) || [];
+    context = context || {
+      left: undefined,
+      startOfBuffer: undefined,
+      endOfBuffer: undefined
+    };
+
+    let leftSpans  = wordBreaker(context.left || '') || [];
     let rightSpans = wordBreaker(context.right || '') || [];
 
     let tokenization: Tokenization = {

common/models/templates/test/test-tokenization.js

@@ -27,6 +27,25 @@ describe('Tokenization functions', function() {
       assert.deepEqual(tokenization, expectedResult);
     });
 
+    it('tokenizes English using defaults, pre-whitespace caret, partial context', function() {
+      let context = {
+        left: "quick brown fox",        // No "The"
+        right: " jumped over the lazy", // No "dog"
+        startOfBuffer: false,
+        endOfBuffer: false
+      };
+
+      let tokenization = models.tokenize(wordBreakers.default, context);
+
+      let expectedResult = {
+        left: ['quick', 'brown', 'fox'],
+        right: ['jumped', 'over', 'the', 'lazy'],
+        caretSplitsToken: false
+      };
+
+      assert.deepEqual(tokenization, expectedResult);
+    });
+
     it('tokenizes English using defaults, post-whitespace caret', function() {
       let context = {
         left: "The quick brown fox ",
@@ -48,6 +67,27 @@ describe('Tokenization functions', function() {
       assert.deepEqual(tokenization, expectedResult);
     });
 
+    it('tokenizes English using defaults, post-whitespace caret, partial context', function() {
+      let context = {
+        left: "quick brown fox ",
+        right: "jumped over the lazy",
+        startOfBuffer: false,
+        endOfBuffer: false
+      };
+
+      let tokenization = models.tokenize(wordBreakers.default, context);
+
+      // Technically, we're editing the start of the first token on the right
+      // when in this context.
+      let expectedResult = {
+        left: ['quick', 'brown', 'fox', ''],
+        right: ['jumped', 'over', 'the', 'lazy'],
+        caretSplitsToken: true
+      };
+
+      assert.deepEqual(tokenization, expectedResult);
+    });
+
     it('tokenizes English using defaults, splitting caret', function() {
       let context = {
         left: "The quick brown fox jum",
@@ -67,6 +107,55 @@ describe('Tokenization functions', function() {
       assert.deepEqual(tokenization, expectedResult);
     });
 
+    it('empty context case', function() {
+      // Wordbreaking on a empty space => no word.
+      let context = {
+        left: '', startOfBuffer: true,
+        right: '', endOfBuffer: true
+      };
+  
+      let tokenization = models.tokenize(wordBreakers.default, context);
+
+      let expectedResult = {
+        left: [],
+        right: [],
+        caretSplitsToken: false
+      };
+  
+      assert.deepEqual(tokenization, expectedResult);
+    });
+
+    it('nil context case', function() {
+      // Wordbreaking on a empty space => no word.
+      let tokenization = models.tokenize(wordBreakers.default, null);
+
+      let expectedResult = {
+        left: [],
+        right: [],
+        caretSplitsToken: false
+      };
+  
+      assert.deepEqual(tokenization, expectedResult);
+    });
+
+    it('near-empty context:  one space before caret', function() {
+      // Wordbreaking on a empty space => no word.
+      let context = {
+        left: ' ', startOfBuffer: true,
+        right: '', endOfBuffer: true
+      };
+  
+      let tokenization = models.tokenize(wordBreakers.default, context);
+
+      let expectedResult = {
+        left: [''],
+        right: [],
+        caretSplitsToken: false
+      };
+  
+      assert.deepEqual(tokenization, expectedResult);
+    });
+
     // For the next few tests:  a mocked wordbreaker for Khmer, a language
     // without whitespace between words.
     let mockedKhmerBreaker = function(text) {

common/predictive-text/unit_tests/headless/default-word-breaker.js

@@ -1,130 +0,0 @@
-/**
- * Smoke-test the default word breaker.
- */
-var assert = require('chai').assert;
-var TrieModel = require('../../build/intermediate').models.TrieModel;
-
-var breakWords = require('../../build/intermediate').wordBreakers['default'];
-const SHY = '\u00AD';
-
-// The following are **integration tests** testing the interaction between the
-// Trie model "template" and the default word breaker.
-//
-// They exercise whether the default word breaker produces the correct word
-// breaks, and whether the Trie model can consume the word breaks properly.
-describe('The default word breaker', function () {
-  it('recognizes a word at end of complete lefthand context', function () {
-    var model = new TrieModel(jsonFixture('tries/english-1000'), {
-      wordBreaker: breakWords // wordBreakers['default'] when fully integrated.
-    });
-
-    // Standard case - wordbreaking at the end of a word.
-    var context = {
-      left: 'The quick brown fox jumped', startOfBuffer: true,
-      right: ' over the lazy dog.', endOfBuffer: true
-    };
-
-    var broken = model.wordbreak(context);
-
-    assert.strictEqual(broken, 'jumped');
-  });
-
-  // Same test as before, but we want to be sure the start/end of buffer flags
-  // don't affect our results.
-  it('recognizes a word at end of incomplete lefthand context', function () {
-    var model = new TrieModel(jsonFixture('tries/english-1000'), {
-      wordBreaker: breakWords
-    });
-
-    // Standard case - wordbreaking at the end of a word.
-    var context = {
-      left: 'The quick brown fox jumped', startOfBuffer: false,
-      right: ' over the lazy dog.', endOfBuffer: false
-    };
-
-    var broken = model.wordbreak(context);
-
-    assert.strictEqual(broken, 'jumped');
-  });
-
-  it('returns text for a word in-progress', function() {
-    var model = new TrieModel(jsonFixture('tries/english-1000'), {
-      wordBreaker: breakWords
-    });
-
-    // Standard case - midword (xylophone) call
-    var context = {
-      left: 'xyl', startOfBuffer: true,
-      right: '', endOfBuffer: true
-    };
-
-    var broken = model.wordbreak(context);
-
-    assert.strictEqual(broken, 'xyl');
-  });
-
-  it('returns empty string when called without word text', function() {
-    var model = new TrieModel(jsonFixture('tries/english-1000'), {
-      wordBreaker: breakWords
-    });
-
-    // Wordbreaking on a empty space => no word.
-    context = {
-      left: 'The quick brown fox jumped ', startOfBuffer: true,
-      right: 'over the lazy dog.', endOfBuffer: true
-    };
-
-    broken = model.wordbreak(context);
-
-    assert.strictEqual(broken, '');
-  });
-
-  it('returns empty string when called with empty context', function() {
-    var model = new TrieModel(jsonFixture('tries/english-1000'), {
-      wordBreaker: breakWords
-    });
-
-    // Wordbreaking on a empty space => no word.
-    context = {
-      left: '', startOfBuffer: true,
-      right: '', endOfBuffer: true
-    };
-
-    broken = model.wordbreak(context);
-
-    assert.strictEqual(broken, '');
-  });
-
-  it('returns empty string when called with nil context', function() {
-    var model = new TrieModel(jsonFixture('tries/english-1000'), {
-      wordBreaker: breakWords
-    });
-
-    // Wordbreaking on a empty space => no word.
-    context = {
-      left: '', startOfBuffer: false,
-      right: '', endOfBuffer: false
-    };
-
-    broken = model.wordbreak(context);
-
-    assert.strictEqual(broken, '');
-  });
-
-  it.skip('correctly breaks a word when the caret is placed within it', function() {
-    var model = new TrieModel(jsonFixture('tries/english-1000'), {
-      wordBreaker: breakWords
-    });
-
-    // A limitation of the current implementation; we should fix this before release.
-    // Then again, when typing this is probably fine; just not when not typing.
-    context = {
-      left: 'The quick brown fox jum', startOfBuffer: true,
-      right: 'ped over the lazy dog.', endOfBuffer: true
-    };
-
-    broken = model.wordbreak(context);
-
-    assert.strictEqual(broken, 'jumped');  // Current result:  'jum'.
-  });
-});