From 2e5855090a397eaa0c21fd785508d96c0b9d0a22 Mon Sep 17 00:00:00 2001
From: Eddie Antonio Santos <easantos@ualberta.ca>
Date: Thu, 25 Jul 2019 11:22:07 -0600
Subject: [PATCH 01/21] Fix tsconfig...

---
 developer/js/tests/test-punctuation.ts | 30 ++++++++++++++++++++++++++
 developer/js/tests/tsconfig.json       | 10 +++++++++
 developer/js/tsconfig.json             |  5 +----
 3 files changed, 41 insertions(+), 4 deletions(-)
 create mode 100644 developer/js/tests/test-punctuation.ts
 create mode 100644 developer/js/tests/tsconfig.json

diff --git a/developer/js/tests/test-punctuation.ts b/developer/js/tests/test-punctuation.ts
new file mode 100644
index 0000000000..e242375cc2
--- /dev/null
+++ b/developer/js/tests/test-punctuation.ts
@@ -0,0 +1,30 @@
+import LexicalModelCompiler from '../';
+import {assert} from 'chai';
+import 'mocha';
+
+const path = require('path');
+
+
+describe('LexicalModelCompiler', function () {
+  describe('#generateLexicalModelCode', function () {
+    const MODEL_ID = 'example.qaa.trivial';
+    const PATH = path.join(__dirname, 'fixtures', MODEL_ID)
+    it('should compile a trivial word list', function () {
+      let compiler = new LexicalModelCompiler;
+      let code = compiler.generateLexicalModelCode(MODEL_ID, {
+        format: 'trie-1.0',
+        sources: ['wordlist.tsv']
+      }, PATH) as string;
+
+      assert.doesNotThrow(function evalModelCode() {
+        eval(code);
+      }, SyntaxError);
+      // TODO: Mock LMLayerWorker.loadModel()
+  
+      // Sanity check: the word list has three total unweighted words, with a
+      // total weight of 3!
+      assert.match(code, /\btotalWeight\b["']?:\s*3\b/);
+    });
+  })
+});
+
diff --git a/developer/js/tests/tsconfig.json b/developer/js/tests/tsconfig.json
new file mode 100644
index 0000000000..655a3fcfe1
--- /dev/null
+++ b/developer/js/tests/tsconfig.json
@@ -0,0 +1,10 @@
+{
+    "compilerOptions": {
+        "module": "commonjs",
+        "noImplicitAny": true,
+        "sourceMap": true
+    },
+    "include": [
+        "**/test-*.ts"
+    ]
+}
\ No newline at end of file
diff --git a/developer/js/tsconfig.json b/developer/js/tsconfig.json
index 901da41ac0..0a05d8ae77 100644
--- a/developer/js/tsconfig.json
+++ b/developer/js/tsconfig.json
@@ -1,13 +1,10 @@
 {
   "compilerOptions": {
-    "target": "es6",
+    "target": "es2017",
     "module": "commonjs",
     "outDir": "dist",
     "sourceMap": true,
     "declaration": true,
-    "typeRoots": [
-      "./node_modules/@types"
-    ]
   },
   "include": [
     "lexical-model-compiler/**/*",

From 9589778597f660d3f1fa78c347186c6d0931dd9a Mon Sep 17 00:00:00 2001
From: Eddie Antonio Santos <easantos@ualberta.ca>
Date: Thu, 25 Jul 2019 11:26:39 -0600
Subject: [PATCH 02/21] Test that the compiler adds the punctuation.

---
 developer/js/tests/test-punctuation.ts | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/developer/js/tests/test-punctuation.ts b/developer/js/tests/test-punctuation.ts
index e242375cc2..7ad86798d8 100644
--- a/developer/js/tests/test-punctuation.ts
+++ b/developer/js/tests/test-punctuation.ts
@@ -6,24 +6,25 @@ const path = require('path');
 
 
 describe('LexicalModelCompiler', function () {
-  describe('#generateLexicalModelCode', function () {
-    const MODEL_ID = 'example.qaa.trivial';
+  describe('spefifying punctuation', function () {
+    const MODEL_ID = 'example.qaa.punctuation';
     const PATH = path.join(__dirname, 'fixtures', MODEL_ID)
+
     it('should compile a trivial word list', function () {
       let compiler = new LexicalModelCompiler;
       let code = compiler.generateLexicalModelCode(MODEL_ID, {
         format: 'trie-1.0',
-        sources: ['wordlist.tsv']
+        sources: ['wordlist.tsv'],
+        punctuation: {
+          quotesForKeepSuggestion: [`«`, `»`],
+          insertAfterWord: " " ,
+        }
       }, PATH) as string;
 
-      assert.doesNotThrow(function evalModelCode() {
-        eval(code);
-      }, SyntaxError);
-      // TODO: Mock LMLayerWorker.loadModel()
-  
-      // Sanity check: the word list has three total unweighted words, with a
-      // total weight of 3!
-      assert.match(code, /\btotalWeight\b["']?:\s*3\b/);
+      // Check that the punctuation actually made into the code:
+      assert.match(code, /«/);
+      assert.match(code, /»/);
+      // TODO: more robust assertions?
     });
   })
 });

From 3d2d94e93e6a85d91f00f61a73bd5d10cb74ad4d Mon Sep 17 00:00:00 2001
From: Eddie Antonio Santos <easantos@ualberta.ca>
Date: Thu, 25 Jul 2019 11:36:02 -0600
Subject: [PATCH 03/21] Write interface for specifying punctuation.

---
 .../lexical-model-compiler/lexical-model.ts   | 45 +++++++++++++++++++
 developer/js/tests/test-punctuation.ts        |  4 +-
 2 files changed, 47 insertions(+), 2 deletions(-)

diff --git a/developer/js/lexical-model-compiler/lexical-model.ts b/developer/js/lexical-model-compiler/lexical-model.ts
index 115cdaf792..f1afa193d6 100644
--- a/developer/js/lexical-model-compiler/lexical-model.ts
+++ b/developer/js/lexical-model-compiler/lexical-model.ts
@@ -35,6 +35,13 @@ interface LexicalModelSource extends LexicalModel {
    * This often involves removing accents, lowercasing, etc.
    */
   readonly searchTermToKey?: (term: string) => string;
+
+  /**
+   * Punctuation and spacing suggested by the model.
+   *
+   * @see LexicalModelPunctuation
+   */
+  readonly punctuation?: LexicalModelPunctuation;
 }
 
 interface LexicalModelCompiled extends LexicalModel {
@@ -45,6 +52,44 @@ interface LexicalModelCompiledTrie extends LexicalModelCompiled {
   trie: string;
 }
 
+/**
+ * Options for various punctuation to use in suggestions.
+ */
+interface LexicalModelPunctuation {
+  /**
+   * The quotes that appear in "keep" suggestions, e.g., keep what the user
+   * typed verbatim.
+   * 
+   * The keep suggestion is often the leftmost one, when suggested.
+   * 
+   * [ “Hrllo” ] [ Hello ] [ Heck ]
+   */
+  readonly quotesForKeepSuggestion: {
+    /**
+     * What will appear on the opening side of the quote.
+     * (left side for LTR scripts; right side for RTL scripts)
+     *
+     * Default: `“`
+     */
+    readonly open: string;
+    /**
+     * What will appear on the closing side of the quote.
+     * (right side for LTR scripts; left side for RTL scripts)
+     *
+     * Default: `”`
+     */
+    readonly close: string;
+  };
+  /**
+   * What punctuation or spacing to insert after every complete word
+   * prediction. This can be set to the empty string when the script does not
+   * use spaces to separate words.
+   * 
+   * Default: ` `
+   */
+  readonly insertAfterWord?: string;
+}
+
 interface LexicalModelCompiledFst extends LexicalModelCompiled {
   fst: string;
 }
diff --git a/developer/js/tests/test-punctuation.ts b/developer/js/tests/test-punctuation.ts
index 7ad86798d8..067367732a 100644
--- a/developer/js/tests/test-punctuation.ts
+++ b/developer/js/tests/test-punctuation.ts
@@ -7,7 +7,7 @@ const path = require('path');
 
 describe('LexicalModelCompiler', function () {
   describe('spefifying punctuation', function () {
-    const MODEL_ID = 'example.qaa.punctuation';
+    const MODEL_ID = 'example.qaa.trivial';
     const PATH = path.join(__dirname, 'fixtures', MODEL_ID)
 
     it('should compile a trivial word list', function () {
@@ -16,7 +16,7 @@ describe('LexicalModelCompiler', function () {
         format: 'trie-1.0',
         sources: ['wordlist.tsv'],
         punctuation: {
-          quotesForKeepSuggestion: [`«`, `»`],
+          quotesForKeepSuggestion: { open: `«`, close: `»`},
           insertAfterWord: " " ,
         }
       }, PATH) as string;

From 795b862ce6a7b299735a14fab7ddfe17d5f49e07 Mon Sep 17 00:00:00 2001
From: Eddie Antonio Santos <easantos@ualberta.ca>
Date: Thu, 25 Jul 2019 12:38:24 -0600
Subject: [PATCH 04/21] Add punctuation to trie model source code.

---
 developer/js/index.ts                              | 14 ++++++++++++++
 .../js/lexical-model-compiler/lexical-model.ts     |  2 +-
 developer/js/tests/test-punctuation.ts             |  3 ++-
 3 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/developer/js/index.ts b/developer/js/index.ts
index e53ffbe838..5ebd778c6a 100644
--- a/developer/js/index.ts
+++ b/developer/js/index.ts
@@ -15,6 +15,14 @@ import { createTrieDataStructure } from "./lexical-model-compiler/build-trie";
 //                         author           .bcp47            .uniq
 const MODEL_ID_PATTERN = /^[a-z_][a-z0-9_]*\.[a-z_][a-z0-9_]*\.[a-z_][a-z0-9_]*$/;
 
+/**
+ * The default punctuation and spacing produced by the model.
+ */
+const DEFAULT_PUNCTUATION = {
+  quotesForKeepSuggestion: { open: `“`, close: `”`},
+  insertAfterWord: " " ,
+}
+
 export default class LexicalModelCompiler {
   compile(modelSource: LexicalModelSource) {
     //
@@ -221,6 +229,11 @@ export default class LexicalModelCompiler {
       }
     }
 
+    // Figure out the punctuation used in the model.
+    let punctuation: LexicalModelPunctuation = Object.assign(
+      {}, DEFAULT_PUNCTUATION, modelSource.punctuation
+    );
+
     //
     // Emit the model as code and data
     //
@@ -245,6 +258,7 @@ export default class LexicalModelCompiler {
         if (modelSource.searchTermToKey) {
           func += `  searchTermToKey: ${modelSource.searchTermToKey.toString()},\n`;
         }
+        func += `  punctuation: ${JSON.stringify(punctuation)},\n`;
         func += `}));\n`;
         break;
       default:
diff --git a/developer/js/lexical-model-compiler/lexical-model.ts b/developer/js/lexical-model-compiler/lexical-model.ts
index f1afa193d6..ebc1b247a8 100644
--- a/developer/js/lexical-model-compiler/lexical-model.ts
+++ b/developer/js/lexical-model-compiler/lexical-model.ts
@@ -64,7 +64,7 @@ interface LexicalModelPunctuation {
    * 
    * [ “Hrllo” ] [ Hello ] [ Heck ]
    */
-  readonly quotesForKeepSuggestion: {
+  readonly quotesForKeepSuggestion?: {
     /**
      * What will appear on the opening side of the quote.
      * (left side for LTR scripts; right side for RTL scripts)
diff --git a/developer/js/tests/test-punctuation.ts b/developer/js/tests/test-punctuation.ts
index 067367732a..8b0d49d142 100644
--- a/developer/js/tests/test-punctuation.ts
+++ b/developer/js/tests/test-punctuation.ts
@@ -17,7 +17,7 @@ describe('LexicalModelCompiler', function () {
         sources: ['wordlist.tsv'],
         punctuation: {
           quotesForKeepSuggestion: { open: `«`, close: `»`},
-          insertAfterWord: " " ,
+          insertAfterWord: " " , // OGHAM SPACE MARK
         }
       }, PATH) as string;
 
@@ -25,6 +25,7 @@ describe('LexicalModelCompiler', function () {
       assert.match(code, /«/);
       assert.match(code, /»/);
       // TODO: more robust assertions?
+      assert.match(code, /\u1680/);
     });
   })
 });

From d71cec5bf18377b6133e309ed7cd926d2e3aa12b Mon Sep 17 00:00:00 2001
From: Eddie Antonio Santos <easantos@ualberta.ca>
Date: Thu, 25 Jul 2019 13:00:35 -0600
Subject: [PATCH 05/21] Write test to detect if the model compositor
 understands punctuation.

---
 .../headless/worker-custom-punctuation.js     | 61 +++++++++++++++++++
 1 file changed, 61 insertions(+)
 create mode 100644 common/predictive-text/unit_tests/headless/worker-custom-punctuation.js

diff --git a/common/predictive-text/unit_tests/headless/worker-custom-punctuation.js b/common/predictive-text/unit_tests/headless/worker-custom-punctuation.js
new file mode 100644
index 0000000000..95310f33ab
--- /dev/null
+++ b/common/predictive-text/unit_tests/headless/worker-custom-punctuation.js
@@ -0,0 +1,61 @@
+/*
+ * Unit tests for the Dummy prediction model.
+ */
+
+var assert = require('chai').assert;
+var DummyModel = require('../../build/intermediate').models.DummyModel;
+var ModelCompositor = require('../../build/intermediate').ModelCompositor;
+
+describe('Custom Punctuation', function() {
+  it('appears in the keep suggestion', function () {
+    let dummySuggestions = [
+      {
+        transform: {
+          insert: 'Hrllo',
+          deleteLeft: 0,
+        },
+        tag: 'keep',
+        displayAs: 'Hrllo',
+      },
+      {
+        transform: {
+          insert: 'Hello',
+          deleteLeft: 0,
+        },
+        displayAs: 'Hello',
+      },
+      {
+        transform: {
+          insert: 'Jello',
+          deleteLeft: 0,
+        },
+        displayAs: 'Jello',
+      }
+    ];
+
+    var model = new DummyModel({futureSuggestions: [dummySuggestions] }, {
+      punctuation: {
+        quotesForKeepSuggestion: {
+          open: "«", close: "»"
+        }
+      }
+    });
+    // The model compositor is responsible for adding this to the display as
+    // string.
+    var composite = new ModelCompositor(model);
+    var suggestions =  composite.predict([{ sample: { insert: 'o', deleteLeft: 0 }, p: 1.00 }], {
+      left: 'Hrll', startOfBuffer: false, endOfBuffer: true
+    });
+    assert.lengthOf(suggestions, 3);
+
+    // We only care about the "keep" suggestion.
+    var keepSuggestions = suggestions.filter(suggestion => suggestion.tag === 'keep');
+    assert.lengthOf(keepSuggestions, 1,
+      `Expected exactly one "keep" suggestion, but found ${keepSuggestions.length}`
+    );
+
+    // The moment of truth: has our punctuation been applied?
+    var suggestion = keepSuggestions[0];
+    assert.equal(suggestion.displayAs, `«Hrllo»`);
+  });
+});
\ No newline at end of file

From 87089a197d7ac440125fd559937cbc4ef4c8b95e Mon Sep 17 00:00:00 2001
From: Eddie Antonio Santos <easantos@ualberta.ca>
Date: Thu, 25 Jul 2019 15:11:40 -0600
Subject: [PATCH 06/21] Implement alternate open/close quote.

---
 .../headless/worker-custom-punctuation.js     |  4 +-
 .../worker/model-compositor.ts                | 18 ++++++--
 .../worker/models/dummy-model.ts              |  5 ++
 .../worker/worker-interfaces.ts               | 46 +++++++++++++++++++
 developer/js/tests/test-punctuation.ts        |  3 +-
 5 files changed, 68 insertions(+), 8 deletions(-)

diff --git a/common/predictive-text/unit_tests/headless/worker-custom-punctuation.js b/common/predictive-text/unit_tests/headless/worker-custom-punctuation.js
index 95310f33ab..57de2f5016 100644
--- a/common/predictive-text/unit_tests/headless/worker-custom-punctuation.js
+++ b/common/predictive-text/unit_tests/headless/worker-custom-punctuation.js
@@ -33,13 +33,15 @@ describe('Custom Punctuation', function() {
       }
     ];
 
-    var model = new DummyModel({futureSuggestions: [dummySuggestions] }, {
+    var model = new DummyModel({
+      futureSuggestions: [dummySuggestions],
       punctuation: {
         quotesForKeepSuggestion: {
           open: "«", close: "»"
         }
       }
     });
+
     // The model compositor is responsible for adding this to the display as
     // string.
     var composite = new ModelCompositor(model);
diff --git a/common/predictive-text/worker/model-compositor.ts b/common/predictive-text/worker/model-compositor.ts
index 00dfc66997..265de955be 100644
--- a/common/predictive-text/worker/model-compositor.ts
+++ b/common/predictive-text/worker/model-compositor.ts
@@ -118,10 +118,10 @@ class ModelCompositor {
       };
     }
 
-    // TODO:  Customizable modeling for this formatting.  Different languages
-    //        use different quotation styles.  See https://github.com/keymanapp/keyman/issues/1883.
-    if(keepOption) {
-      keepOption.displayAs = '"' + keepOption.displayAs + '"';
+    if (keepOption) {
+      let punctuation = this.lexicalModel.punctuation || DEFAULT_PUNCTUATION;
+      let { open, close } = punctuation.quotesForKeepSuggestion;
+      keepOption.displayAs = open + keepOption.displayAs + close;
     }
 
     // Now that we've calculated a unique set of probability masses, time to make them into a proper
@@ -145,4 +145,12 @@ class ModelCompositor {
 
     return suggestions;
   }
-}
\ No newline at end of file
+}
+
+/**
+ * The default punctuation and spacing produced by the model.
+ */
+const DEFAULT_PUNCTUATION = {
+  quotesForKeepSuggestion: { open: `“`, close: `”`},
+  insertAfterWord: " " ,
+}
diff --git a/common/predictive-text/worker/models/dummy-model.ts b/common/predictive-text/worker/models/dummy-model.ts
index 4052fc5482..06809b3fa5 100644
--- a/common/predictive-text/worker/models/dummy-model.ts
+++ b/common/predictive-text/worker/models/dummy-model.ts
@@ -33,6 +33,7 @@ namespace models {
    */
   export class DummyModel implements WorkerInternalModel {
     configuration: Configuration;
+    punctuation?: LexicalModelPunctuation;
     private _futureSuggestions: Suggestion[][];
 
     constructor(options?: any) {
@@ -41,6 +42,10 @@ namespace models {
       // this class mutates the array.
       this._futureSuggestions = options.futureSuggestions
         ? options.futureSuggestions.slice() : [];
+
+      if (options.punctuation) {
+        this.punctuation = options.punctuation;
+      }
     }
 
     configure(capabilities: Capabilities): Configuration {
diff --git a/common/predictive-text/worker/worker-interfaces.ts b/common/predictive-text/worker/worker-interfaces.ts
index e813289306..514784aa4f 100644
--- a/common/predictive-text/worker/worker-interfaces.ts
+++ b/common/predictive-text/worker/worker-interfaces.ts
@@ -138,6 +138,14 @@ interface WorkerInternalModel {
   configure(capabilities: Capabilities): Configuration;
   predict(transform: Transform, context: Context): Distribution<Suggestion>;
   wordbreak(context: Context): USVString;
+
+  /**
+   * Punctuation and presentational settings that the underlying lexical model
+   * expects to be applied at higher levels. e.g., the ModelCompositor.
+   * 
+   * @see LexicalModelPunctuation
+   */
+  readonly punctuation?: LexicalModelPunctuation;
 }
 
 /**
@@ -189,3 +197,41 @@ interface Span {
   // UTF-16 code unit followed by a low surrogate UTF-16 code unit.
   readonly text: string;
 }
+
+/**
+ * Options for various punctuation to use in suggestions.
+ */
+interface LexicalModelPunctuation {
+  /**
+   * The quotes that appear in "keep" suggestions, e.g., keep what the user
+   * typed verbatim.
+   * 
+   * The keep suggestion is often the leftmost one, when suggested.
+   * 
+   * [ “Hrllo” ] [ Hello ] [ Heck ]
+   */
+  readonly quotesForKeepSuggestion: {
+    /**
+     * What will appear on the opening side of the quote.
+     * (left side for LTR scripts; right side for RTL scripts)
+     *
+     * Default: `“`
+     */
+    readonly open: string;
+    /**
+     * What will appear on the closing side of the quote.
+     * (right side for LTR scripts; left side for RTL scripts)
+     *
+     * Default: `”`
+     */
+    readonly close: string;
+  };
+  /**
+   * What punctuation or spacing to insert after every complete word
+   * prediction. This can be set to the empty string when the script does not
+   * use spaces to separate words.
+   * 
+   * Default: ` `
+   */
+  readonly insertAfterWord: string;
+}
\ No newline at end of file
diff --git a/developer/js/tests/test-punctuation.ts b/developer/js/tests/test-punctuation.ts
index 8b0d49d142..336a7ec1f3 100644
--- a/developer/js/tests/test-punctuation.ts
+++ b/developer/js/tests/test-punctuation.ts
@@ -10,7 +10,7 @@ describe('LexicalModelCompiler', function () {
     const MODEL_ID = 'example.qaa.trivial';
     const PATH = path.join(__dirname, 'fixtures', MODEL_ID)
 
-    it('should compile a trivial word list', function () {
+    it('should compile punctuation into the generated code', function () {
       let compiler = new LexicalModelCompiler;
       let code = compiler.generateLexicalModelCode(MODEL_ID, {
         format: 'trie-1.0',
@@ -29,4 +29,3 @@ describe('LexicalModelCompiler', function () {
     });
   })
 });
-

From 096ecad140c30413581d9fb7ca199ebc70ff81f8 Mon Sep 17 00:00:00 2001
From: Eddie Antonio Santos <easantos@ualberta.ca>
Date: Thu, 25 Jul 2019 15:29:59 -0600
Subject: [PATCH 07/21] Test that the proper space is appended.

---
 .../headless/worker-custom-punctuation.js     | 49 ++++++++++++++++++-
 1 file changed, 47 insertions(+), 2 deletions(-)

diff --git a/common/predictive-text/unit_tests/headless/worker-custom-punctuation.js b/common/predictive-text/unit_tests/headless/worker-custom-punctuation.js
index 57de2f5016..8ba6aa2e91 100644
--- a/common/predictive-text/unit_tests/headless/worker-custom-punctuation.js
+++ b/common/predictive-text/unit_tests/headless/worker-custom-punctuation.js
@@ -6,7 +6,7 @@ var assert = require('chai').assert;
 var DummyModel = require('../../build/intermediate').models.DummyModel;
 var ModelCompositor = require('../../build/intermediate').ModelCompositor;
 
-describe('Custom Punctuation', function() {
+describe('Custom Punctuation', function () {
   it('appears in the keep suggestion', function () {
     let dummySuggestions = [
       {
@@ -38,7 +38,8 @@ describe('Custom Punctuation', function() {
       punctuation: {
         quotesForKeepSuggestion: {
           open: "«", close: "»"
-        }
+        },
+        insertAfterWord: " "
       }
     });
 
@@ -60,4 +61,48 @@ describe('Custom Punctuation', function() {
     var suggestion = keepSuggestions[0];
     assert.equal(suggestion.displayAs, `«Hrllo»`);
   });
+
+  describe("insertAfterWord", function () {
+    it('appears after "word" suggestion', function () {
+      let dummySuggestions = [
+        {
+          transform: { insert: 'ᚈᚑᚋ', deleteLeft: 0, },
+          displayAs: 'ᚈᚑᚋ',
+        },
+        {
+          transform: { insert: 'ᚄ', deleteLeft: 0, },
+          displayAs: 'ᚄ',
+        },
+        {
+          transform: { insert: 'ᚉᚑᚈᚈ', deleteLeft: 0, },
+          displayAs: 'ᚉᚑᚈᚈ',
+        }
+      ];
+
+      var model = new DummyModel({
+        futureSuggestions: [dummySuggestions],
+        punctuation: {
+          // OGHAM SPACE: it's technically whitespace, but it don't look it!
+          insertAfterWord: " ",
+          quotesForKeepSuggestion: {
+            open: "“", close: "”"
+          },
+        }
+      });
+
+      // The model compositor is responsible for adding this to the display as
+      // string.
+      var composite = new ModelCompositor(model);
+      var suggestions =  composite.predict([{ sample: { insert: 'ᚋ', deleteLeft: 0 }, p: 1.00 }], {
+        left: '᚛ᚈᚑ', startOfBuffer: false, endOfBuffer: true
+      });
+      assert.lengthOf(suggestions, dummySuggestions.length);
+
+      // Check that it has been changed:
+      for (var i = 0; i < dummySuggestions.length; i++) {
+        assert.equal(suggestions[i].transform.insert,
+                     dummySuggestions[i].transform.insert + " ");
+      }
+    });
+  })
 });
\ No newline at end of file

From 529f0d3adf3c4e8f23e6e541bd7cfb00e3f7d8d5 Mon Sep 17 00:00:00 2001
From: Eddie Antonio Santos <easantos@ualberta.ca>
Date: Thu, 25 Jul 2019 15:47:54 -0600
Subject: [PATCH 08/21] Move space adding to the model compositor.

---
 .../headless/worker-custom-punctuation.js           |  6 +++---
 common/predictive-text/worker/model-compositor.ts   | 13 ++++++++++---
 common/predictive-text/worker/models/trie-model.ts  |  2 +-
 3 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/common/predictive-text/unit_tests/headless/worker-custom-punctuation.js b/common/predictive-text/unit_tests/headless/worker-custom-punctuation.js
index 8ba6aa2e91..a1bc0b39bb 100644
--- a/common/predictive-text/unit_tests/headless/worker-custom-punctuation.js
+++ b/common/predictive-text/unit_tests/headless/worker-custom-punctuation.js
@@ -82,7 +82,8 @@ describe('Custom Punctuation', function () {
       var model = new DummyModel({
         futureSuggestions: [dummySuggestions],
         punctuation: {
-          // OGHAM SPACE: it's technically whitespace, but it don't look it!
+          // U+1680 OGHAM SPACE MARK:
+          // it's technically whitespace, but it don't look it!
           insertAfterWord: " ",
           quotesForKeepSuggestion: {
             open: "“", close: "”"
@@ -100,8 +101,7 @@ describe('Custom Punctuation', function () {
 
       // Check that it has been changed:
       for (var i = 0; i < dummySuggestions.length; i++) {
-        assert.equal(suggestions[i].transform.insert,
-                     dummySuggestions[i].transform.insert + " ");
+        assert.isTrue(suggestions[i].transform.insert.endsWith(' '));
       }
     });
   })
diff --git a/common/predictive-text/worker/model-compositor.ts b/common/predictive-text/worker/model-compositor.ts
index 265de955be..57fd4d869b 100644
--- a/common/predictive-text/worker/model-compositor.ts
+++ b/common/predictive-text/worker/model-compositor.ts
@@ -49,6 +49,7 @@ class ModelCompositor {
     let postContext = models.applyTransform(inputTransform, context);
     let keepOptionText = this.lexicalModel.wordbreak(postContext);
     let keepOption: Suggestion = null;
+    let punctuation = this.lexicalModel.punctuation || DEFAULT_PUNCTUATION;
 
     for(let alt of transformDistribution) {
       let transform = alt.sample;
@@ -81,6 +82,12 @@ class ModelCompositor {
         if(preserveWhitespace) {
           models.prependTransform(pair.sample.transform, transform);
         }
+        
+        // The model is trying to add a word; thus, add some custom formatting
+        // to that word.
+        if (pair.sample.transform.insert.length > 0) {
+          pair.sample.transform.insert += punctuation.insertAfterWord;
+        }
 
         // Combine duplicate samples.
         let displayText = pair.sample.displayAs;
@@ -109,7 +116,7 @@ class ModelCompositor {
         transformId: inputTransform.id,
         // Replicate the original transform, modified for appropriate language insertion syntax.
         transform: {
-          insert: inputTransform.insert + ' ',
+          insert: inputTransform.insert + punctuation.insertAfterWord,
           deleteLeft: inputTransform.deleteLeft,
           deleteRight: inputTransform.deleteRight,
           id: inputTransform.id
@@ -118,8 +125,8 @@ class ModelCompositor {
       };
     }
 
+    // Add the surrounding quotes to the "keep" option's display string:
     if (keepOption) {
-      let punctuation = this.lexicalModel.punctuation || DEFAULT_PUNCTUATION;
       let { open, close } = punctuation.quotesForKeepSuggestion;
       keepOption.displayAs = open + keepOption.displayAs + close;
     }
@@ -153,4 +160,4 @@ class ModelCompositor {
 const DEFAULT_PUNCTUATION = {
   quotesForKeepSuggestion: { open: `“`, close: `”`},
   insertAfterWord: " " ,
-}
+};
diff --git a/common/predictive-text/worker/models/trie-model.ts b/common/predictive-text/worker/models/trie-model.ts
index 97b4df91b0..339df5ba43 100644
--- a/common/predictive-text/worker/models/trie-model.ts
+++ b/common/predictive-text/worker/models/trie-model.ts
@@ -94,7 +94,7 @@
       if (!transform.insert && context.startOfBuffer && context.endOfBuffer) {
         return makeDistribution(this._trie.firstN(MAX_SUGGESTIONS).map(({text, p}) => ({
           transform: {
-            insert: text + ' ', // TODO: do NOT add the space here!
+            insert: text,
             deleteLeft: 0
           },
           displayAs: text,

From ada6af46f69b995597874de1b935e7570af8d89e Mon Sep 17 00:00:00 2001
From: Eddie Antonio Santos <easantos@ualberta.ca>
Date: Thu, 25 Jul 2019 16:05:04 -0600
Subject: [PATCH 09/21] Fix old dummy model test case.

---
 .../future_suggestions/i_got_distracted_by_hazel.json |  2 +-
 .../in_browser/resources/models/simple-dummy.js       | 11 +++++++++--
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/common/predictive-text/unit_tests/in_browser/json/future_suggestions/i_got_distracted_by_hazel.json b/common/predictive-text/unit_tests/in_browser/json/future_suggestions/i_got_distracted_by_hazel.json
index 347b98d2e6..a9aa49753b 100644
--- a/common/predictive-text/unit_tests/in_browser/json/future_suggestions/i_got_distracted_by_hazel.json
+++ b/common/predictive-text/unit_tests/in_browser/json/future_suggestions/i_got_distracted_by_hazel.json
@@ -19,7 +19,7 @@
         "insert": "Oh ",
         "deleteLeft": 0
       },
-      "displayAs": "Oh "
+      "displayAs": "Oh"
     }
   ],
   [
diff --git a/common/predictive-text/unit_tests/in_browser/resources/models/simple-dummy.js b/common/predictive-text/unit_tests/in_browser/resources/models/simple-dummy.js
index 85d7b3057d..3f290d385e 100644
--- a/common/predictive-text/unit_tests/in_browser/resources/models/simple-dummy.js
+++ b/common/predictive-text/unit_tests/in_browser/resources/models/simple-dummy.js
@@ -7,6 +7,13 @@
     function Model() { // implements Model
     }
 
+    Model.punctuation = {
+      quotesForKeepSuggestion: { open: `“`, close: `”`},
+      // Important! Set this, or else the model compositor will
+      // insert something for us!
+      insertAfterWord: "",
+    };
+
     // A direct import/copy from i_got_distracted_by_hazel.json.
     Model.futureSuggestions = [
       [
@@ -29,7 +36,7 @@
             "insert": "Oh ",
             "deleteLeft": 0
           },
-          "displayAs": "Oh "
+          "displayAs": "Oh"
         }
       ],
       [
@@ -107,5 +114,5 @@
   }());
 
   // It's a 'dummy' model, so there's no need for extra methods and such within the Model's class definition.
-  LMLayerWorker.loadModel(new models.DummyModel({futureSuggestions: Model.futureSuggestions}));
+  LMLayerWorker.loadModel(new models.DummyModel({futureSuggestions: Model.futureSuggestions, punctuation: Model.punctuation}));
 })();
\ No newline at end of file

From 5e2528a78c32c187f971008c0041a3f64408a5ca Mon Sep 17 00:00:00 2001
From: Eddie Antonio Santos <easantos@ualberta.ca>
Date: Mon, 29 Jul 2019 15:29:36 -0600
Subject: [PATCH 10/21] Extract method: determinePunctuationFromModel().

---
 common/predictive-text/worker/model-compositor.ts | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/common/predictive-text/worker/model-compositor.ts b/common/predictive-text/worker/model-compositor.ts
index 57fd4d869b..ea33be1c03 100644
--- a/common/predictive-text/worker/model-compositor.ts
+++ b/common/predictive-text/worker/model-compositor.ts
@@ -49,7 +49,7 @@ class ModelCompositor {
     let postContext = models.applyTransform(inputTransform, context);
     let keepOptionText = this.lexicalModel.wordbreak(postContext);
     let keepOption: Suggestion = null;
-    let punctuation = this.lexicalModel.punctuation || DEFAULT_PUNCTUATION;
+    let punctuation = this.determinePunctuationFromModel();
 
     for(let alt of transformDistribution) {
       let transform = alt.sample;
@@ -152,6 +152,10 @@ class ModelCompositor {
 
     return suggestions;
   }
+
+  private determinePunctuationFromModel() {
+    return this.lexicalModel.punctuation || DEFAULT_PUNCTUATION;
+  }
 }
 
 /**

From b20541ea8bbce3bbad107aa12db7a5db94cfc33f Mon Sep 17 00:00:00 2001
From: Eddie Antonio Santos <easantos@ualberta.ca>
Date: Mon, 29 Jul 2019 15:40:08 -0600
Subject: [PATCH 11/21] Allow for the partial specification of punctuation,
 with more robust defaults selector thing.

---
 .../headless/worker-custom-punctuation.js     |  4 ---
 .../worker/model-compositor.ts                | 28 +++++++++++++++++--
 2 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/common/predictive-text/unit_tests/headless/worker-custom-punctuation.js b/common/predictive-text/unit_tests/headless/worker-custom-punctuation.js
index a1bc0b39bb..3407ace41f 100644
--- a/common/predictive-text/unit_tests/headless/worker-custom-punctuation.js
+++ b/common/predictive-text/unit_tests/headless/worker-custom-punctuation.js
@@ -39,7 +39,6 @@ describe('Custom Punctuation', function () {
         quotesForKeepSuggestion: {
           open: "«", close: "»"
         },
-        insertAfterWord: " "
       }
     });
 
@@ -85,9 +84,6 @@ describe('Custom Punctuation', function () {
           // U+1680 OGHAM SPACE MARK:
           // it's technically whitespace, but it don't look it!
           insertAfterWord: " ",
-          quotesForKeepSuggestion: {
-            open: "“", close: "”"
-          },
         }
       });
 
diff --git a/common/predictive-text/worker/model-compositor.ts b/common/predictive-text/worker/model-compositor.ts
index ea33be1c03..60fb8bca5d 100644
--- a/common/predictive-text/worker/model-compositor.ts
+++ b/common/predictive-text/worker/model-compositor.ts
@@ -49,7 +49,7 @@ class ModelCompositor {
     let postContext = models.applyTransform(inputTransform, context);
     let keepOptionText = this.lexicalModel.wordbreak(postContext);
     let keepOption: Suggestion = null;
-    let punctuation = this.determinePunctuationFromModel();
+    let punctuation = this.determinePunctuationFromModel(this.lexicalModel);
 
     for(let alt of transformDistribution) {
       let transform = alt.sample;
@@ -153,8 +153,30 @@ class ModelCompositor {
     return suggestions;
   }
 
-  private determinePunctuationFromModel() {
-    return this.lexicalModel.punctuation || DEFAULT_PUNCTUATION;
+  /**
+   * Returns the punctuation used for this model, filling out unspecified fields
+   */
+  private determinePunctuationFromModel(model: WorkerInternalModel): LexicalModelPunctuation {
+    let defaults = DEFAULT_PUNCTUATION;
+
+    // Use the defaults of the model does not provide any punctuation at all.
+    if (!model.punctuation)
+      return defaults;
+
+    let specifiedPunctuation = this.lexicalModel.punctuation;
+    let insertAfterWord = specifiedPunctuation.insertAfterWord;
+    if (insertAfterWord !== '' && !insertAfterWord) {
+      insertAfterWord = defaults.insertAfterWord;
+    }
+
+    let quotesForKeepSuggestion = specifiedPunctuation.quotesForKeepSuggestion;
+    if (!quotesForKeepSuggestion) {
+      quotesForKeepSuggestion = defaults.quotesForKeepSuggestion;
+    }
+
+    return {
+      insertAfterWord, quotesForKeepSuggestion
+    }
   }
 }
 

From 32a8dcb0c2d8950af3748483ff063f607668ec74 Mon Sep 17 00:00:00 2001
From: Eddie Antonio Santos <easantos@ualberta.ca>
Date: Mon, 29 Jul 2019 15:44:23 -0600
Subject: [PATCH 12/21] Convert determinePunctuationFromModel() to a static
 method.

---
 common/predictive-text/worker/model-compositor.ts | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/common/predictive-text/worker/model-compositor.ts b/common/predictive-text/worker/model-compositor.ts
index 60fb8bca5d..4c2664f7f3 100644
--- a/common/predictive-text/worker/model-compositor.ts
+++ b/common/predictive-text/worker/model-compositor.ts
@@ -1,9 +1,11 @@
 class ModelCompositor {
   private lexicalModel: WorkerInternalModel;
   private static readonly MAX_SUGGESTIONS = 12;
+  private readonly punctuation: LexicalModelPunctuation;
 
   constructor(lexicalModel: WorkerInternalModel) {
     this.lexicalModel = lexicalModel;
+    this.punctuation = ModelCompositor.determinePunctuationFromModel(lexicalModel);
   }
 
   protected isWhitespace(transform: Transform): boolean {
@@ -28,6 +30,7 @@ class ModelCompositor {
 
   predict(transformDistribution: Transform | Distribution<Transform>, context: Context): Suggestion[] {
     let suggestionDistribution: Distribution<Suggestion> = [];
+    let punctuation = this.punctuation;
 
     // Assumption:  Duplicated 'displayAs' properties indicate duplicated Suggestions.
     // When true, we can use an 'associative array' to de-duplicate everything.
@@ -49,7 +52,6 @@ class ModelCompositor {
     let postContext = models.applyTransform(inputTransform, context);
     let keepOptionText = this.lexicalModel.wordbreak(postContext);
     let keepOption: Suggestion = null;
-    let punctuation = this.determinePunctuationFromModel(this.lexicalModel);
 
     for(let alt of transformDistribution) {
       let transform = alt.sample;
@@ -156,14 +158,14 @@ class ModelCompositor {
   /**
    * Returns the punctuation used for this model, filling out unspecified fields
    */
-  private determinePunctuationFromModel(model: WorkerInternalModel): LexicalModelPunctuation {
+  private static determinePunctuationFromModel(model: WorkerInternalModel): LexicalModelPunctuation {
     let defaults = DEFAULT_PUNCTUATION;
 
     // Use the defaults of the model does not provide any punctuation at all.
     if (!model.punctuation)
       return defaults;
 
-    let specifiedPunctuation = this.lexicalModel.punctuation;
+    let specifiedPunctuation = model.punctuation;
     let insertAfterWord = specifiedPunctuation.insertAfterWord;
     if (insertAfterWord !== '' && !insertAfterWord) {
       insertAfterWord = defaults.insertAfterWord;

From 31650eba4965469f4a0a2cd599827d912f9a1caa Mon Sep 17 00:00:00 2001
From: Eddie Antonio Santos <easantos@ualberta.ca>
Date: Mon, 29 Jul 2019 15:55:17 -0600
Subject: [PATCH 13/21] Allow for optional punctuation to be passed into
 options.

---
 developer/js/index.ts                  | 4 +++-
 developer/js/tests/test-punctuation.ts | 5 +++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/developer/js/index.ts b/developer/js/index.ts
index 5ebd778c6a..0b9c85796c 100644
--- a/developer/js/index.ts
+++ b/developer/js/index.ts
@@ -258,7 +258,9 @@ export default class LexicalModelCompiler {
         if (modelSource.searchTermToKey) {
           func += `  searchTermToKey: ${modelSource.searchTermToKey.toString()},\n`;
         }
-        func += `  punctuation: ${JSON.stringify(punctuation)},\n`;
+        if (modelSource.punctuation) {
+          func += `  punctuation: ${JSON.stringify(punctuation)},\n`;
+        }
         func += `}));\n`;
         break;
       default:
diff --git a/developer/js/tests/test-punctuation.ts b/developer/js/tests/test-punctuation.ts
index 336a7ec1f3..e8f24bdba3 100644
--- a/developer/js/tests/test-punctuation.ts
+++ b/developer/js/tests/test-punctuation.ts
@@ -2,7 +2,7 @@ import LexicalModelCompiler from '../';
 import {assert} from 'chai';
 import 'mocha';
 
-const path = require('path');
+import path = require('path');
 
 
 describe('LexicalModelCompiler', function () {
@@ -24,8 +24,9 @@ describe('LexicalModelCompiler', function () {
       // Check that the punctuation actually made into the code:
       assert.match(code, /«/);
       assert.match(code, /»/);
-      // TODO: more robust assertions?
+      // Ensure we inserted that OGHAM SPACE MARK!
       assert.match(code, /\u1680/);
+      // TODO: more robust assertions?
     });
   })
 });

From eb12da683b5268f70a33bc253b3709606fd9c94f Mon Sep 17 00:00:00 2001
From: Eddie Antonio Santos <easantos@ualberta.ca>
Date: Mon, 29 Jul 2019 16:04:41 -0600
Subject: [PATCH 14/21] Test that the trie model exposes punctuation.

---
 .../headless/worker-predict-trie.js           | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/common/predictive-text/unit_tests/headless/worker-predict-trie.js b/common/predictive-text/unit_tests/headless/worker-predict-trie.js
index a19994f803..cde71da62c 100644
--- a/common/predictive-text/unit_tests/headless/worker-predict-trie.js
+++ b/common/predictive-text/unit_tests/headless/worker-predict-trie.js
@@ -6,6 +6,27 @@ var assert = require('chai').assert;
 var TrieModel = require('../../build/intermediate').models.TrieModel;
 
 describe('LMLayerWorker trie model for word lists', function() {
+  describe('instantiation', function () {
+    it('should expose the punctuation object', function () {
+      var spaceMark = "👩🏻‍🚀";
+      var openQuote = "🌜";
+      var closeQuote = "🌛";
+
+      var model = new TrieModel(jsonFixture('tries/english-1000'), {
+        punctuation: {
+          insertAfterWord: spaceMark,
+          quotesForKeepSuggestion: {
+            open: openQuote, close: closeQuote
+          }
+        }
+      })
+      
+      assert.equal(model.punctuation.insertAfterWord, spaceMark);
+      assert.equal(model.punctuation.quotesForKeepSuggestion.open, openQuote);
+      assert.equal(model.punctuation.quotesForKeepSuggestion.close, closeQuote);
+    })
+  });
+
   describe('prediction', function () {
     var MIN_SUGGESTIONS = 3;
 

From d99337bf3971f36ca050e5760c3e13ecd9c322d0 Mon Sep 17 00:00:00 2001
From: Eddie Antonio Santos <easantos@ualberta.ca>
Date: Mon, 29 Jul 2019 16:06:43 -0600
Subject: [PATCH 15/21] Add punctuation to trie model.

---
 common/predictive-text/worker/models/trie-model.ts | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/common/predictive-text/worker/models/trie-model.ts b/common/predictive-text/worker/models/trie-model.ts
index 339df5ba43..57ca0ead14 100644
--- a/common/predictive-text/worker/models/trie-model.ts
+++ b/common/predictive-text/worker/models/trie-model.ts
@@ -48,6 +48,11 @@
      *  This should simplify a search term into a key.
      */
     searchTermToKey?: (searchTerm: string) => string;
+
+    /**
+     * Any punctuation to expose to the user.
+     */
+    punctuation?: LexicalModelPunctuation;
   }
 
   /**
@@ -72,6 +77,7 @@
     configuration: Configuration;
     private _trie: Trie;
     readonly breakWords: WordBreakingFunction;
+    readonly punctuation?: LexicalModelPunctuation;
 
     constructor(trieData: object, options: TrieModelOptions = {}) {
       this._trie = new Trie(
@@ -80,6 +86,7 @@
         options.searchTermToKey as Wordform2Key || defaultWordform2Key
       );
       this.breakWords = options.wordBreaker || wordBreakers.placeholder;
+      this.punctuation = options.punctuation;
     }
 
     configure(capabilities: Capabilities): Configuration {

From 13c5a4cdd52ed8560126030a69e7729cb3f03083 Mon Sep 17 00:00:00 2001
From: Eddie Antonio Santos <easantos@ualberta.ca>
Date: Tue, 30 Jul 2019 09:47:15 -0600
Subject: [PATCH 16/21] Trie to appease IE...

---
 .../unit_tests/in_browser/resources/models/simple-dummy.js      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/predictive-text/unit_tests/in_browser/resources/models/simple-dummy.js b/common/predictive-text/unit_tests/in_browser/resources/models/simple-dummy.js
index 3f290d385e..97180729bd 100644
--- a/common/predictive-text/unit_tests/in_browser/resources/models/simple-dummy.js
+++ b/common/predictive-text/unit_tests/in_browser/resources/models/simple-dummy.js
@@ -8,7 +8,7 @@
     }
 
     Model.punctuation = {
-      quotesForKeepSuggestion: { open: `“`, close: `”`},
+      quotesForKeepSuggestion: { open: '“', close: '”'},
       // Important! Set this, or else the model compositor will
       // insert something for us!
       insertAfterWord: "",

From 3017300f33182d4dbca57ec106c6343278679854 Mon Sep 17 00:00:00 2001
From: Eddie Antonio Santos <easantos@ualberta.ca>
Date: Tue, 30 Jul 2019 10:07:11 -0600
Subject: [PATCH 17/21] Factor out some interfaces required both by compiler
 and by worker to their own file.

---
 .../worker/worker-compiler-interfaces.d.ts    | 80 +++++++++++++++++++
 .../worker/worker-interfaces.ts               | 78 +-----------------
 .../lexical-model-compiler/lexical-model.ts   | 44 ++--------
 3 files changed, 87 insertions(+), 115 deletions(-)
 create mode 100644 common/predictive-text/worker/worker-compiler-interfaces.d.ts

diff --git a/common/predictive-text/worker/worker-compiler-interfaces.d.ts b/common/predictive-text/worker/worker-compiler-interfaces.d.ts
new file mode 100644
index 0000000000..c918e1cc47
--- /dev/null
+++ b/common/predictive-text/worker/worker-compiler-interfaces.d.ts
@@ -0,0 +1,80 @@
+/**
+ * Types and interfaces that must be know both by the lexical model compiler,
+ * and by the runtime.
+ */
+
+/**
+ * A simple word breaking function takes a phrase, and splits it into "words",
+ * for whatever definition of "word" is usable for the language model.
+ *
+ * For example:
+ *
+ *   getText(breakWordsEnglish("Hello, world!")) == ["Hello", "world"]
+ *   getText(breakWordsCree("ᑕᐻ ᒥᔪ ᑮᓯᑲᐤ ᐊᓄᐦᐨ᙮")) == ["ᑕᐻ", "ᒥᔪ ᑮᓯᑲᐤ""", "ᐊᓄᐦᐨ"]
+ *   getText(breakWordsJapanese("英語を話せますか？")) == ["英語", "を", "話せます", "か"]
+ *
+ * Not all language models take in a configurable word breaking function.
+ *
+ * @returns an array of spans from the phrase, in order as they appear in the
+ *          phrase, each span which representing a word.
+ */
+interface WordBreakingFunction {
+  // invariant: span[i].end <= span[i + 1].start
+  // invariant: for all span[i] and span[i + 1], there does not exist a span[k]
+  //            where span[i].end <= span[k].start AND span[k].end <= span[i + 1].start
+  (phrase: string): Span[];
+}
+
+/**
+ * A span of text in a phrase. This is usually meant to reprent words from a
+ * pharse.
+ */
+interface Span {
+  // invariant: start < end (empty spans not allowed)
+  readonly start: number;
+  // invariant: end > end (empty spans not allowed)
+  readonly end: number;
+  // invariant: length === end - start
+  readonly length: number;
+  // invariant: text.length === length
+  // invariant: each character is BMP UTF-16 code unit, or is a high surrogate
+  // UTF-16 code unit followed by a low surrogate UTF-16 code unit.
+  readonly text: string;
+}
+/**
+ * Options for various punctuation to use in suggestions.
+ */
+interface LexicalModelPunctuation {
+  /**
+   * The quotes that appear in "keep" suggestions, e.g., keep what the user
+   * typed verbatim.
+   *
+   * The keep suggestion is often the leftmost one, when suggested.
+   *
+   * [ “Hrllo” ] [ Hello ] [ Heck ]
+   */
+  readonly quotesForKeepSuggestion: {
+    /**
+     * What will appear on the opening side of the quote.
+     * (left side for LTR scripts; right side for RTL scripts)
+     *
+     * Default: `“`
+     */
+    readonly open: string;
+    /**
+     * What will appear on the closing side of the quote.
+     * (right side for LTR scripts; left side for RTL scripts)
+     *
+     * Default: `”`
+     */
+    readonly close: string;
+  };
+  /**
+   * What punctuation or spacing to insert after every complete word
+   * prediction. This can be set to the empty string when the script does not
+   * use spaces to separate words.
+   *
+   * Default: ` `
+   */
+  readonly insertAfterWord: string;
+}
diff --git a/common/predictive-text/worker/worker-interfaces.ts b/common/predictive-text/worker/worker-interfaces.ts
index a555db671e..eabef62841 100644
--- a/common/predictive-text/worker/worker-interfaces.ts
+++ b/common/predictive-text/worker/worker-interfaces.ts
@@ -27,6 +27,7 @@
  */
 
 /// <reference path="../message.d.ts" />
+/// <reference path="./worker-compiler-interfaces.d.ts" />
 
 /**
  * The signature of self.postMessage(), so that unit tests can mock it.
@@ -204,80 +205,3 @@ interface WorkerInternalModelConstructor {
    */
   new(...modelParameters: any[]): WorkerInternalModel;
 }
-
-/**
- * A simple word breaking function takes a phrase, and splits it into "words",
- * for whatever definition of "word" is usable for the language model.
- *
- * For example:
- *
- *   getText(breakWordsEnglish("Hello, world!")) == ["Hello", "world"]
- *   getText(breakWordsCree("ᑕᐻ ᒥᔪ ᑮᓯᑲᐤ ᐊᓄᐦᐨ᙮")) == ["ᑕᐻ", "ᒥᔪ ᑮᓯᑲᐤ""", "ᐊᓄᐦᐨ"]
- *   getText(breakWordsJapanese("英語を話せますか？")) == ["英語", "を", "話せます", "か"]
- *
- * Not all language models take in a configurable word breaking function.
- *
- * @returns an array of spans from the phrase, in order as they appear in the
- *          phrase, each span which representing a word.
- */
-interface WordBreakingFunction {
-  // invariant: span[i].end <= span[i + 1].start
-  // invariant: for all span[i] and span[i + 1], there does not exist a span[k]
-  //            where span[i].end <= span[k].start AND span[k].end <= span[i + 1].start
-  (phrase: USVString): Span[];
-}
-
-/**
- * A span of text in a phrase. This is usually meant to reprent words from a
- * pharse.
- */
-interface Span {
-  // invariant: start < end (empty spans not allowed)
-  readonly start: number;
-  // invariant: end > end (empty spans not allowed)
-  readonly end: number;
-  // invariant: length === end - start
-  readonly length: number;
-  // invariant: text.length === length
-  // invariant: each character is BMP UTF-16 code unit, or is a high surrogate
-  // UTF-16 code unit followed by a low surrogate UTF-16 code unit.
-  readonly text: string;
-}
-
-/**
- * Options for various punctuation to use in suggestions.
- */
-interface LexicalModelPunctuation {
-  /**
-   * The quotes that appear in "keep" suggestions, e.g., keep what the user
-   * typed verbatim.
-   * 
-   * The keep suggestion is often the leftmost one, when suggested.
-   * 
-   * [ “Hrllo” ] [ Hello ] [ Heck ]
-   */
-  readonly quotesForKeepSuggestion: {
-    /**
-     * What will appear on the opening side of the quote.
-     * (left side for LTR scripts; right side for RTL scripts)
-     *
-     * Default: `“`
-     */
-    readonly open: string;
-    /**
-     * What will appear on the closing side of the quote.
-     * (right side for LTR scripts; left side for RTL scripts)
-     *
-     * Default: `”`
-     */
-    readonly close: string;
-  };
-  /**
-   * What punctuation or spacing to insert after every complete word
-   * prediction. This can be set to the empty string when the script does not
-   * use spaces to separate words.
-   * 
-   * Default: ` `
-   */
-  readonly insertAfterWord: string;
-}
\ No newline at end of file
diff --git a/developer/js/lexical-model-compiler/lexical-model.ts b/developer/js/lexical-model-compiler/lexical-model.ts
index ebc1b247a8..4a557ec975 100644
--- a/developer/js/lexical-model-compiler/lexical-model.ts
+++ b/developer/js/lexical-model-compiler/lexical-model.ts
@@ -1,3 +1,9 @@
+/**
+ * Interfaces and constants used by the lexical model compiler. These target
+ * the LMLayer's internal worker code, so we provide those definitions too.
+ */
+/// <reference path="../../../common/predictive-text/worker/worker-compiler-interfaces.d.ts" />
+
 interface ClassBasedWordBreaker {
   allowedCharacters?: { initials?: string, medials?: string, finals?: string } | string,
   defaultBreakCharacter?: string
@@ -52,44 +58,6 @@ interface LexicalModelCompiledTrie extends LexicalModelCompiled {
   trie: string;
 }
 
-/**
- * Options for various punctuation to use in suggestions.
- */
-interface LexicalModelPunctuation {
-  /**
-   * The quotes that appear in "keep" suggestions, e.g., keep what the user
-   * typed verbatim.
-   * 
-   * The keep suggestion is often the leftmost one, when suggested.
-   * 
-   * [ “Hrllo” ] [ Hello ] [ Heck ]
-   */
-  readonly quotesForKeepSuggestion?: {
-    /**
-     * What will appear on the opening side of the quote.
-     * (left side for LTR scripts; right side for RTL scripts)
-     *
-     * Default: `“`
-     */
-    readonly open: string;
-    /**
-     * What will appear on the closing side of the quote.
-     * (right side for LTR scripts; left side for RTL scripts)
-     *
-     * Default: `”`
-     */
-    readonly close: string;
-  };
-  /**
-   * What punctuation or spacing to insert after every complete word
-   * prediction. This can be set to the empty string when the script does not
-   * use spaces to separate words.
-   * 
-   * Default: ` `
-   */
-  readonly insertAfterWord?: string;
-}
-
 interface LexicalModelCompiledFst extends LexicalModelCompiled {
   fst: string;
 }

From 5513503dd26e4ae471fe9eb781eeaf77eb1b7e4a Mon Sep 17 00:00:00 2001
From: Eddie Antonio Santos <easantos@ualberta.ca>
Date: Tue, 30 Jul 2019 10:12:37 -0600
Subject: [PATCH 18/21] Clean-up unused interfaces in lexical model compiler.

---
 developer/js/index.ts                         |  6 ++----
 .../lexical-model-compiler/lexical-model.ts   | 20 +------------------
 2 files changed, 3 insertions(+), 23 deletions(-)

diff --git a/developer/js/index.ts b/developer/js/index.ts
index 0b9c85796c..cd50a67b7c 100644
--- a/developer/js/index.ts
+++ b/developer/js/index.ts
@@ -244,11 +244,9 @@ export default class LexicalModelCompiler {
         func += `LMLayerWorker.loadModel(new ${modelSource.rootClass}());\n`;
         break;
       case "fst-foma-1.0":
-        (oc as LexicalModelCompiledFst).fst = Buffer.from(sources.join('')).toString('base64');
-        this.logError('Unimplemented model format '+modelSource.format);
+        this.logError('Unimplemented model format ' + modelSource.format);
         return false;
-      case "trie-1.0": // TODO: in lexical-models, rollback all trie-2.0 models to use trie-1.0!
-      case 'trie-2.0':
+      case "trie-1.0":
         func += `LMLayerWorker.loadModel(new models.TrieModel(${
           createTrieDataStructure(sources, modelSource.searchTermToKey)
         }, {\n`;
diff --git a/developer/js/lexical-model-compiler/lexical-model.ts b/developer/js/lexical-model-compiler/lexical-model.ts
index 4a557ec975..92aa4c7261 100644
--- a/developer/js/lexical-model-compiler/lexical-model.ts
+++ b/developer/js/lexical-model-compiler/lexical-model.ts
@@ -15,17 +15,10 @@ interface ClassBasedWordBreaker {
 }
 
 interface LexicalModel {
-  // TODO: remove trie-2.0 when https://github.com/keymanapp/lexical-models/pull/39 is merged.
-  readonly format: 'trie-1.0'|'trie-2.0'|'fst-foma-1.0'|'custom-1.0',
+  readonly format: 'trie-1.0'|'fst-foma-1.0'|'custom-1.0',
   //... metadata ...
 }
 
-interface LexicalModelPrediction {
-  display?: string;
-  transform: string;
-  delete: number;
-}
-
 interface LexicalModelSource extends LexicalModel {
   readonly sources: Array<string>;
   /**
@@ -53,14 +46,3 @@ interface LexicalModelSource extends LexicalModel {
 interface LexicalModelCompiled extends LexicalModel {
   readonly id: string;
 }
-
-interface LexicalModelCompiledTrie extends LexicalModelCompiled {
-  trie: string;
-}
-
-interface LexicalModelCompiledFst extends LexicalModelCompiled {
-  fst: string;
-}
-
-interface LexicalModelCompiledCustom extends LexicalModelCompiled {
-}

From 6c72c2acb3aacf5ab27c71c74b61e229fc0e9b85 Mon Sep 17 00:00:00 2001
From: Eddie Antonio Santos <easantos@ualberta.ca>
Date: Tue, 30 Jul 2019 10:16:40 -0600
Subject: [PATCH 19/21] Default punctuation is no longer specified in compiler.

---
 developer/js/index.ts | 15 +--------------
 1 file changed, 1 insertion(+), 14 deletions(-)

diff --git a/developer/js/index.ts b/developer/js/index.ts
index cd50a67b7c..5ef9ea5c5f 100644
--- a/developer/js/index.ts
+++ b/developer/js/index.ts
@@ -15,14 +15,6 @@ import { createTrieDataStructure } from "./lexical-model-compiler/build-trie";
 //                         author           .bcp47            .uniq
 const MODEL_ID_PATTERN = /^[a-z_][a-z0-9_]*\.[a-z_][a-z0-9_]*\.[a-z_][a-z0-9_]*$/;
 
-/**
- * The default punctuation and spacing produced by the model.
- */
-const DEFAULT_PUNCTUATION = {
-  quotesForKeepSuggestion: { open: `“`, close: `”`},
-  insertAfterWord: " " ,
-}
-
 export default class LexicalModelCompiler {
   compile(modelSource: LexicalModelSource) {
     //
@@ -229,11 +221,6 @@ export default class LexicalModelCompiler {
       }
     }
 
-    // Figure out the punctuation used in the model.
-    let punctuation: LexicalModelPunctuation = Object.assign(
-      {}, DEFAULT_PUNCTUATION, modelSource.punctuation
-    );
-
     //
     // Emit the model as code and data
     //
@@ -257,7 +244,7 @@ export default class LexicalModelCompiler {
           func += `  searchTermToKey: ${modelSource.searchTermToKey.toString()},\n`;
         }
         if (modelSource.punctuation) {
-          func += `  punctuation: ${JSON.stringify(punctuation)},\n`;
+          func += `  punctuation: ${JSON.stringify(modelSource.punctuation)},\n`;
         }
         func += `}));\n`;
         break;

From 27e63e16b19e1a898288cc3c034077761a64a962 Mon Sep 17 00:00:00 2001
From: Eddie Antonio Santos <easantos@ualberta.ca>
Date: Tue, 30 Jul 2019 10:20:36 -0600
Subject: [PATCH 20/21] Be explicit about the type of punctuation defaults.

---
 common/predictive-text/worker/model-compositor.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/predictive-text/worker/model-compositor.ts b/common/predictive-text/worker/model-compositor.ts
index 4c2664f7f3..8331336e9c 100644
--- a/common/predictive-text/worker/model-compositor.ts
+++ b/common/predictive-text/worker/model-compositor.ts
@@ -185,7 +185,7 @@ class ModelCompositor {
 /**
  * The default punctuation and spacing produced by the model.
  */
-const DEFAULT_PUNCTUATION = {
+const DEFAULT_PUNCTUATION: LexicalModelPunctuation = {
   quotesForKeepSuggestion: { open: `“`, close: `”`},
   insertAfterWord: " " ,
 };

From fd5173c88235fd22281afb1780d55569b4fc86ac Mon Sep 17 00:00:00 2001
From: Eddie Antonio Santos <easantos@ualberta.ca>
Date: Wed, 31 Jul 2019 09:40:09 -0600
Subject: [PATCH 21/21] Typo

Co-Authored-By: Marc Durdin <marc@durdin.net>
---
 common/predictive-text/worker/worker-compiler-interfaces.d.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/predictive-text/worker/worker-compiler-interfaces.d.ts b/common/predictive-text/worker/worker-compiler-interfaces.d.ts
index c918e1cc47..05d432c95a 100644
--- a/common/predictive-text/worker/worker-compiler-interfaces.d.ts
+++ b/common/predictive-text/worker/worker-compiler-interfaces.d.ts
@@ -26,7 +26,7 @@ interface WordBreakingFunction {
 }
 
 /**
- * A span of text in a phrase. This is usually meant to reprent words from a
+ * A span of text in a phrase. This is usually meant to represent words from a
  * pharse.
  */
 interface Span {