From b39dcd5588bac7cd2abe2ac2a53d33ecd0b35c2a Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 28 Feb 2019 11:52:03 +0700 Subject: [PATCH 01/22] Namespaces the existing model files in prep for move, to assist Git --- common/predictive-text/worker/dummy-model.ts | 56 ++++---- common/predictive-text/worker/index.ts | 3 + .../predictive-text/worker/wordlist-model.ts | 120 +++++++++--------- 3 files changed, 93 insertions(+), 86 deletions(-) diff --git a/common/predictive-text/worker/dummy-model.ts b/common/predictive-text/worker/dummy-model.ts index b6f89b582f..4ea134ab59 100644 --- a/common/predictive-text/worker/dummy-model.ts +++ b/common/predictive-text/worker/dummy-model.ts @@ -22,33 +22,35 @@ /// -/** - * @file dummy-model.ts - * - * Defines the Dummy model, which is used for testing the - * prediction API exclusively. - */ +namespace models { + /** + * @file dummy-model.ts + * + * Defines the Dummy model, which is used for testing the + * prediction API exclusively. + */ -/** - * The Dummy Model that returns nonsensical, but predictable results. - */ -LMLayerWorker.models.DummyModel = class DummyModel implements WorkerInternalModel { - configuration: Configuration; - private _futureSuggestions: Suggestion[][]; + /** + * The Dummy Model that returns nonsensical, but predictable results. + */ + LMLayerWorker.models.DummyModel = class DummyModel implements WorkerInternalModel { + configuration: Configuration; + private _futureSuggestions: Suggestion[][]; - constructor(capabilities: Capabilities, options?: any) { - options = options || {}; - this.configuration = options.configuration || {}; - // Create a shallow copy of the suggestions; - // this class mutates the array. - this._futureSuggestions = options.futureSuggestions - ? options.futureSuggestions.slice() : []; - } - - predict(transform: Transform, context: Context, injectedSuggestions?: Suggestion[]): Suggestion[] { - if (injectedSuggestions) { - return injectedSuggestions; + constructor(capabilities: Capabilities, options?: any) { + options = options || {}; + this.configuration = options.configuration || {}; + // Create a shallow copy of the suggestions; + // this class mutates the array. + this._futureSuggestions = options.futureSuggestions + ? options.futureSuggestions.slice() : []; } - return this._futureSuggestions.shift(); - } -}; + + predict(transform: Transform, context: Context, injectedSuggestions?: Suggestion[]): Suggestion[] { + if (injectedSuggestions) { + return injectedSuggestions; + } + return this._futureSuggestions.shift(); + } + }; +} \ No newline at end of file diff --git a/common/predictive-text/worker/index.ts b/common/predictive-text/worker/index.ts index fc4d00b830..f97d8c5c29 100644 --- a/common/predictive-text/worker/index.ts +++ b/common/predictive-text/worker/index.ts @@ -236,6 +236,9 @@ class LMLayerWorker { let worker = new LMLayerWorker({ postMessage: scope.postMessage }); scope.onmessage = worker.onMessage.bind(worker); + // Ensures that the worker instance is accessible for loaded model scripts. + scope['LMLayerWorker'] = worker; + return worker; } } diff --git a/common/predictive-text/worker/wordlist-model.ts b/common/predictive-text/worker/wordlist-model.ts index 518a304e58..f3b35bbd84 100644 --- a/common/predictive-text/worker/wordlist-model.ts +++ b/common/predictive-text/worker/wordlist-model.ts @@ -28,71 +28,73 @@ * Defines a simple word list (unigram) model. */ -/** - * @class WordListModel - * - * Defines the word list model, or the unigram model. - * Unigram models throw away all preceding words, and search - * for the next word exclusively. As such, they can perform simple - * prefix searches within words, however they are not very good - * at predicting the next word. - */ -LMLayerWorker.models.WordListModel = (function () { - /** Upper bound on the amount of suggestions to generate. */ - const MAX_SUGGESTIONS = 3; + namespace models { + /** + * @class WordListModel + * + * Defines the word list model, or the unigram model. + * Unigram models throw away all preceding words, and search + * for the next word exclusively. As such, they can perform simple + * prefix searches within words, however they are not very good + * at predicting the next word. + */ + LMLayerWorker.models.WordListModel = (function () { + /** Upper bound on the amount of suggestions to generate. */ + const MAX_SUGGESTIONS = 3; - return class WordListModel implements WorkerInternalModel { - private _wordlist: string[]; + return class WordListModel implements WorkerInternalModel { + private _wordlist: string[]; - constructor(_capabilities: Capabilities, wordlist: string[]) { - this._wordlist = wordlist; - } - - predict(transform: Transform, context: Context): Suggestion[] { - // EVERYTHING to the left of the cursor: - let fullLeftContext = context.left || ''; - // Stuff to the left of the cursor in the current word. - let leftContext = fullLeftContext.split(/\s+/).pop() || ''; - // All text to the left of the cursor INCLUDING anything that has - // just been typed. - let prefix = leftContext + (transform.insert || ''); - let suggestions: Suggestion[] = []; - - // Special-case the empty buffer/transform: return the top suggestions. - if (!transform.insert && context.startOfBuffer && context.endOfBuffer) { - return this._wordlist.slice(0, MAX_SUGGESTIONS).map(word => ({ - transform: { - insert: word + ' ', - deleteLeft: 0 - }, - displayAs: word - })); + constructor(_capabilities: Capabilities, wordlist: string[]) { + this._wordlist = wordlist; } - // Naïve O(n) exhaustive search through the entire word - // list, up to the suggestion limit. - for (let word of this._wordlist) { - let suggestionPrefix = word.substr(0, prefix.length); - if (prefix !== suggestionPrefix) { - continue; + predict(transform: Transform, context: Context): Suggestion[] { + // EVERYTHING to the left of the cursor: + let fullLeftContext = context.left || ''; + // Stuff to the left of the cursor in the current word. + let leftContext = fullLeftContext.split(/\s+/).pop() || ''; + // All text to the left of the cursor INCLUDING anything that has + // just been typed. + let prefix = leftContext + (transform.insert || ''); + let suggestions: Suggestion[] = []; + + // Special-case the empty buffer/transform: return the top suggestions. + if (!transform.insert && context.startOfBuffer && context.endOfBuffer) { + return this._wordlist.slice(0, MAX_SUGGESTIONS).map(word => ({ + transform: { + insert: word + ' ', + deleteLeft: 0 + }, + displayAs: word + })); } - suggestions.push({ - transform: { - // The left part of the word has already been entered. - insert: word.substr(leftContext.length) + ' ', - deleteLeft: 0, - }, - displayAs: word, - }); + // Naïve O(n) exhaustive search through the entire word + // list, up to the suggestion limit. + for (let word of this._wordlist) { + let suggestionPrefix = word.substr(0, prefix.length); + if (prefix !== suggestionPrefix) { + continue; + } - // Do not exceed the limit on suggestions. - if (suggestions.length >= MAX_SUGGESTIONS) { - break; + suggestions.push({ + transform: { + // The left part of the word has already been entered. + insert: word.substr(leftContext.length) + ' ', + deleteLeft: 0, + }, + displayAs: word, + }); + + // Do not exceed the limit on suggestions. + if (suggestions.length >= MAX_SUGGESTIONS) { + break; + } } + + return suggestions; } - - return suggestions; - } - }; -}()); + }; + }()); +} \ No newline at end of file From fb38399833a6f65258873c7c7ffc179e57f6264b Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 28 Feb 2019 12:21:18 +0700 Subject: [PATCH 02/22] Adjusted models and their direct tests to be namespace-compatible. --- .../headless/worker-predict-dummy.js | 1 + .../headless/worker-predict-wordlist.js | 1 + common/predictive-text/worker/index.ts | 17 +-- .../worker/{ => models}/dummy-model.ts | 4 +- .../worker/models/wordlist-model.ts | 97 +++++++++++++++++ .../predictive-text/worker/wordlist-model.ts | 100 ------------------ 6 files changed, 105 insertions(+), 115 deletions(-) rename common/predictive-text/worker/{ => models}/dummy-model.ts (94%) create mode 100644 common/predictive-text/worker/models/wordlist-model.ts delete mode 100644 common/predictive-text/worker/wordlist-model.ts diff --git a/common/predictive-text/unit_tests/headless/worker-predict-dummy.js b/common/predictive-text/unit_tests/headless/worker-predict-dummy.js index e42e3002ed..5650d6b893 100644 --- a/common/predictive-text/unit_tests/headless/worker-predict-dummy.js +++ b/common/predictive-text/unit_tests/headless/worker-predict-dummy.js @@ -4,6 +4,7 @@ var assert = require('chai').assert; +debugger var DummyModel = require('../../build/intermediate').models.DummyModel; describe('LMLayerWorker dummy model', function() { diff --git a/common/predictive-text/unit_tests/headless/worker-predict-wordlist.js b/common/predictive-text/unit_tests/headless/worker-predict-wordlist.js index 8e895ba07d..937b8f8dad 100644 --- a/common/predictive-text/unit_tests/headless/worker-predict-wordlist.js +++ b/common/predictive-text/unit_tests/headless/worker-predict-wordlist.js @@ -4,6 +4,7 @@ var assert = require('chai').assert; +debugger var WordListModel = require('../../build/intermediate').models.WordListModel; describe('LMLayerWorker word list model', function() { diff --git a/common/predictive-text/worker/index.ts b/common/predictive-text/worker/index.ts index f97d8c5c29..88ed9416dc 100644 --- a/common/predictive-text/worker/index.ts +++ b/common/predictive-text/worker/index.ts @@ -30,6 +30,8 @@ */ /// +/// +/// /** * Encapsulates all the state required for the LMLayer's worker thread. @@ -54,16 +56,6 @@ * as such, they are NOT direct properties of the LMLayerWorker. */ class LMLayerWorker { - /** - * All of the bundled model implementations will add themselves here: - * Note: the models will add themselves by including this - * file using a triple-slash directive: - * /// - * then adding the constructor to this static object: - * LMLayerWorker.models.MyModelImplementation = class {} - */ - static models: {[key: string]: WorkerInternalModelConstructor} = {}; - /** * State pattern. This object handles onMessage(). * handleMessage() can transition to a different state, if @@ -142,11 +134,11 @@ class LMLayerWorker { }; if (desc.type === 'dummy') { - model = new LMLayerWorker.models.DummyModel(capabilities, { + model = new models.DummyModel(capabilities, { futureSuggestions: desc.futureSuggestions }); } else if (desc.type === 'wordlist') { - model = new LMLayerWorker.models.WordListModel(capabilities, desc.wordlist); + model = new models.WordListModel(capabilities, desc.wordlist); } else { throw new Error('Invalid model'); } @@ -246,6 +238,7 @@ class LMLayerWorker { // Let LMLayerWorker be available both in the browser and in Node. if (typeof module !== 'undefined' && typeof module.exports !== 'undefined') { module.exports = LMLayerWorker; + module.exports['models'] = models; } else if (typeof self !== 'undefined' && 'postMessage' in self) { // Automatically install if we're in a Web Worker. LMLayerWorker.install(self as DedicatedWorkerGlobalScope); diff --git a/common/predictive-text/worker/dummy-model.ts b/common/predictive-text/worker/models/dummy-model.ts similarity index 94% rename from common/predictive-text/worker/dummy-model.ts rename to common/predictive-text/worker/models/dummy-model.ts index 4ea134ab59..9e90f174e2 100644 --- a/common/predictive-text/worker/dummy-model.ts +++ b/common/predictive-text/worker/models/dummy-model.ts @@ -20,8 +20,6 @@ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -/// - namespace models { /** * @file dummy-model.ts @@ -33,7 +31,7 @@ namespace models { /** * The Dummy Model that returns nonsensical, but predictable results. */ - LMLayerWorker.models.DummyModel = class DummyModel implements WorkerInternalModel { + export class DummyModel implements WorkerInternalModel { configuration: Configuration; private _futureSuggestions: Suggestion[][]; diff --git a/common/predictive-text/worker/models/wordlist-model.ts b/common/predictive-text/worker/models/wordlist-model.ts new file mode 100644 index 0000000000..e77ca1b7a8 --- /dev/null +++ b/common/predictive-text/worker/models/wordlist-model.ts @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2018 National Research Council Canada (author: Eddie A. Santos) + * Copyright (c) 2018 SIL International + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/** + * @file wordlist-model.ts + * + * Defines a simple word list (unigram) model. + */ + + namespace models { + /** + * @class WordListModel + * + * Defines the word list model, or the unigram model. + * Unigram models throw away all preceding words, and search + * for the next word exclusively. As such, they can perform simple + * prefix searches within words, however they are not very good + * at predicting the next word. + */ + + /** Upper bound on the amount of suggestions to generate. */ + const MAX_SUGGESTIONS = 3; + + export class WordListModel implements WorkerInternalModel { + private _wordlist: string[]; + + constructor(_capabilities: Capabilities, wordlist: string[]) { + this._wordlist = wordlist; + } + + predict(transform: Transform, context: Context): Suggestion[] { + // EVERYTHING to the left of the cursor: + let fullLeftContext = context.left || ''; + // Stuff to the left of the cursor in the current word. + let leftContext = fullLeftContext.split(/\s+/).pop() || ''; + // All text to the left of the cursor INCLUDING anything that has + // just been typed. + let prefix = leftContext + (transform.insert || ''); + let suggestions: Suggestion[] = []; + + // Special-case the empty buffer/transform: return the top suggestions. + if (!transform.insert && context.startOfBuffer && context.endOfBuffer) { + return this._wordlist.slice(0, MAX_SUGGESTIONS).map(word => ({ + transform: { + insert: word + ' ', + deleteLeft: 0 + }, + displayAs: word + })); + } + + // Naïve O(n) exhaustive search through the entire word + // list, up to the suggestion limit. + for (let word of this._wordlist) { + let suggestionPrefix = word.substr(0, prefix.length); + if (prefix !== suggestionPrefix) { + continue; + } + + suggestions.push({ + transform: { + // The left part of the word has already been entered. + insert: word.substr(leftContext.length) + ' ', + deleteLeft: 0, + }, + displayAs: word, + }); + + // Do not exceed the limit on suggestions. + if (suggestions.length >= MAX_SUGGESTIONS) { + break; + } + } + + return suggestions; + } + }; +} \ No newline at end of file diff --git a/common/predictive-text/worker/wordlist-model.ts b/common/predictive-text/worker/wordlist-model.ts deleted file mode 100644 index f3b35bbd84..0000000000 --- a/common/predictive-text/worker/wordlist-model.ts +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (c) 2018 National Research Council Canada (author: Eddie A. Santos) - * Copyright (c) 2018 SIL International - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of - * this software and associated documentation files (the "Software"), to deal in - * the Software without restriction, including without limitation the rights to - * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of - * the Software, and to permit persons to whom the Software is furnished to do so, - * subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER - * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -/// - -/** - * @file wordlist-model.ts - * - * Defines a simple word list (unigram) model. - */ - - namespace models { - /** - * @class WordListModel - * - * Defines the word list model, or the unigram model. - * Unigram models throw away all preceding words, and search - * for the next word exclusively. As such, they can perform simple - * prefix searches within words, however they are not very good - * at predicting the next word. - */ - LMLayerWorker.models.WordListModel = (function () { - /** Upper bound on the amount of suggestions to generate. */ - const MAX_SUGGESTIONS = 3; - - return class WordListModel implements WorkerInternalModel { - private _wordlist: string[]; - - constructor(_capabilities: Capabilities, wordlist: string[]) { - this._wordlist = wordlist; - } - - predict(transform: Transform, context: Context): Suggestion[] { - // EVERYTHING to the left of the cursor: - let fullLeftContext = context.left || ''; - // Stuff to the left of the cursor in the current word. - let leftContext = fullLeftContext.split(/\s+/).pop() || ''; - // All text to the left of the cursor INCLUDING anything that has - // just been typed. - let prefix = leftContext + (transform.insert || ''); - let suggestions: Suggestion[] = []; - - // Special-case the empty buffer/transform: return the top suggestions. - if (!transform.insert && context.startOfBuffer && context.endOfBuffer) { - return this._wordlist.slice(0, MAX_SUGGESTIONS).map(word => ({ - transform: { - insert: word + ' ', - deleteLeft: 0 - }, - displayAs: word - })); - } - - // Naïve O(n) exhaustive search through the entire word - // list, up to the suggestion limit. - for (let word of this._wordlist) { - let suggestionPrefix = word.substr(0, prefix.length); - if (prefix !== suggestionPrefix) { - continue; - } - - suggestions.push({ - transform: { - // The left part of the word has already been entered. - insert: word.substr(leftContext.length) + ' ', - deleteLeft: 0, - }, - displayAs: word, - }); - - // Do not exceed the limit on suggestions. - if (suggestions.length >= MAX_SUGGESTIONS) { - break; - } - } - - return suggestions; - } - }; - }()); -} \ No newline at end of file From 6218b808dd92d2d4724214b42251a53bf37cb416 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 28 Feb 2019 14:45:42 +0700 Subject: [PATCH 03/22] Retools how the LMLayer adds models, fixes most headless cases. --- .../headless/worker-initialization.js | 56 ++++++--- .../headless/worker-predict-dummy.js | 2 - .../headless/worker-predict-wordlist.js | 2 - .../unit_tests/headless/worker-predict.js | 2 +- common/predictive-text/unit_tests/helpers.js | 15 +++ .../resources/models/simple-dummy.js | 115 ++++++++++++++++++ common/predictive-text/worker/index.ts | 43 ++++--- .../worker/models/dummy-model.ts | 6 + .../worker/models/wordlist-model.ts | 6 + .../worker/worker-interfaces.ts | 10 +- 10 files changed, 205 insertions(+), 52 deletions(-) create mode 100644 common/predictive-text/unit_tests/in_browser/resources/models/simple-dummy.js diff --git a/common/predictive-text/unit_tests/headless/worker-initialization.js b/common/predictive-text/unit_tests/headless/worker-initialization.js index 958236c153..096fad0bb5 100644 --- a/common/predictive-text/unit_tests/headless/worker-initialization.js +++ b/common/predictive-text/unit_tests/headless/worker-initialization.js @@ -12,13 +12,18 @@ describe('LMLayerWorker', function() { describe('#constructor()', function() { it('should allow for the mocking of postMessage()', function () { var fakePostMessage = sinon.fake(); - var worker = new LMLayerWorker({ postMessage: fakePostMessage }); + var context = { + postMessage: fakePostMessage + }; + context.importScripts = importScriptsWith(context); + + var worker = LMLayerWorker.install(context); + //var worker = new LMLayerWorker({ postMessage: fakePostMessage }); // Sending it the initialize it should notify us that it's initialized! worker.onMessage(createMessageEventWithData({ message: 'initialize', - model: dummyModel(), - capabilities: defaultCapabilities() + model: "./unit_tests/in_browser/resources/models/simple-dummy.js" })); assert(fakePostMessage.calledOnce); }); @@ -26,9 +31,11 @@ describe('LMLayerWorker', function() { describe('#onMessage()', function() { it('should fail if not given the `message` attribute', function () { - var worker = new LMLayerWorker({ - postMessage: sinon.fake(), // required, but ignored in this test case - }); + var context = { + postMessage: sinon.fake() + }; + context.importScripts = importScriptsWith(context); + var worker = LMLayerWorker.install(context); // Every message is a discriminated union with the tag being `message`. // If it doesn't see 'message', something is deeply wrong, // and it should loudly let us know. @@ -46,6 +53,8 @@ describe('LMLayerWorker', function() { onmessage: undefined, postMessage: new sinon.fake(), }; + fakeWorkerGlobal.importScripts = importScriptsWith(fakeWorkerGlobal); + // Instantiate and install a worker on our global object. var worker = LMLayerWorker.install(fakeWorkerGlobal); assert.instanceOf(worker, LMLayerWorker); @@ -55,8 +64,7 @@ describe('LMLayerWorker', function() { // Send a message; we should get something back. worker.onMessage(createMessageEventWithData({ message: 'initialize', - model: dummyModel(), - capabilities: defaultCapabilities() + model: "./unit_tests/in_browser/resources/models/simple-dummy.js" })); // It called the postMessage() in its global scope. @@ -66,9 +74,12 @@ describe('LMLayerWorker', function() { describe('Message: initialize', function () { it('should disallow any other message', function () { - var worker = new LMLayerWorker({ - postMessage: sinon.fake(), // required, but ignored - }); + var context = { + postMessage: sinon.fake() + }; + context.importScripts = importScriptsWith(context); + + var worker = LMLayerWorker.install(context); // It should not respond to 'predict' assert.throws(function () { @@ -80,11 +91,15 @@ describe('LMLayerWorker', function() { it('should send back a "ready" message', function () { var fakePostMessage = sinon.fake(); - var worker = new LMLayerWorker({ postMessage: fakePostMessage }); + var context = { + postMessage: fakePostMessage + }; + context.importScripts = importScriptsWith(context); + + var worker = LMLayerWorker.install(context); worker.onMessage(createMessageEventWithData({ message: 'initialize', - model: dummyModel(), - capabilities: defaultCapabilities() + model: "./unit_tests/in_browser/resources/models/simple-dummy.js" })); assert(fakePostMessage.calledOnceWith(sinon.match({ @@ -94,14 +109,17 @@ describe('LMLayerWorker', function() { it('should send back configuration', function () { var fakePostMessage = sinon.fake(); - var worker = new LMLayerWorker({ postMessage: fakePostMessage }); + var context = { + postMessage: fakePostMessage + }; + context.importScripts = importScriptsWith(context); + + var worker = LMLayerWorker.install(context); + // simple-dummy.js is set with the following. var maxCodeUnits = 64; worker.onMessage(createMessageEventWithData({ message: 'initialize', - model: dummyModel(), - capabilities: { - maxLeftContextCodeUnits: maxCodeUnits, - } + model: "./unit_tests/in_browser/resources/models/simple-dummy.js" })); sinon.assert.calledWithMatch(fakePostMessage, { diff --git a/common/predictive-text/unit_tests/headless/worker-predict-dummy.js b/common/predictive-text/unit_tests/headless/worker-predict-dummy.js index 5650d6b893..b0ffd9f6c5 100644 --- a/common/predictive-text/unit_tests/headless/worker-predict-dummy.js +++ b/common/predictive-text/unit_tests/headless/worker-predict-dummy.js @@ -3,8 +3,6 @@ */ var assert = require('chai').assert; - -debugger var DummyModel = require('../../build/intermediate').models.DummyModel; describe('LMLayerWorker dummy model', function() { diff --git a/common/predictive-text/unit_tests/headless/worker-predict-wordlist.js b/common/predictive-text/unit_tests/headless/worker-predict-wordlist.js index 937b8f8dad..b49f7eaa12 100644 --- a/common/predictive-text/unit_tests/headless/worker-predict-wordlist.js +++ b/common/predictive-text/unit_tests/headless/worker-predict-wordlist.js @@ -3,8 +3,6 @@ */ var assert = require('chai').assert; - -debugger var WordListModel = require('../../build/intermediate').models.WordListModel; describe('LMLayerWorker word list model', function() { diff --git a/common/predictive-text/unit_tests/headless/worker-predict.js b/common/predictive-text/unit_tests/headless/worker-predict.js index 83d3f937f1..722e09f3d6 100644 --- a/common/predictive-text/unit_tests/headless/worker-predict.js +++ b/common/predictive-text/unit_tests/headless/worker-predict.js @@ -5,7 +5,7 @@ let LMLayerWorker = require('../../build/intermediate'); describe('LMLayerWorker', function () { - describe('#predict()', function () { + describe.skip('#predict()', function () { it('should send back suggestions', function () { var suggestion = { transform: { diff --git a/common/predictive-text/unit_tests/helpers.js b/common/predictive-text/unit_tests/helpers.js index c38ce879dc..8afb087b0a 100644 --- a/common/predictive-text/unit_tests/helpers.js +++ b/common/predictive-text/unit_tests/helpers.js @@ -97,4 +97,19 @@ if (typeof require === 'function') { // └── ... return require('./in_browser/json/' + name); } + + var fs = require("fs"); + var vm = require("vm"); + + // This worker-global function does not exist by default in Node! + _.importScriptsWith = function(context) { + return function() { // the constructed context's importScripts method. + debugger + for(var i=0; i < arguments.length; i++) { + context = vm.createContext(context); + var script = new vm.Script(fs.readFileSync(arguments[i])); + script.runInContext(context); + } + } + } } diff --git a/common/predictive-text/unit_tests/in_browser/resources/models/simple-dummy.js b/common/predictive-text/unit_tests/in_browser/resources/models/simple-dummy.js new file mode 100644 index 0000000000..deebc126a5 --- /dev/null +++ b/common/predictive-text/unit_tests/in_browser/resources/models/simple-dummy.js @@ -0,0 +1,115 @@ +/** + * While handwritten, this class is designed to mirror the potential results of TypeScript compilation of + * model TS source files. + */ +(function(){ + var Model = /** @class */ (function() { + function Model() { // implements Model + } + + Model.capabilities = { + maxLeftContextCodeUnits: 64 + } + + // A direct import/copy from i_got_distracted_by_hazel.json. + Model.futureSuggestions = [ + [ + { + "transform": { + "insert": "I ", + "deleteLeft": 0 + }, + "displayAs": "I" + }, + { + "transform": { + "insert": "I'm ", + "deleteLeft": 0 + }, + "displayAs": "I'm" + }, + { + "transform": { + "insert": "Oh ", + "deleteLeft": 0 + }, + "displayAs": "Oh " + } + ], + [ + { + "transform": { + "insert": "love ", + "deleteLeft": 0 + }, + "displayAs": "love" + }, + { + "transform": { + "insert": "am ", + "deleteLeft": 0 + }, + "displayAs": "am" + }, + { + "transform": { + "insert": "got ", + "deleteLeft": 0 + }, + "displayAs": "got" + } + ], + [ + { + "transform": { + "insert": "distracted ", + "deleteLeft": 0 + }, + "displayAs": "distracted by" + }, + { + "transform": { + "insert": "distracted ", + "deleteLeft": 0 + }, + "displayAs": "distracted" + }, + { + "transform": { + "insert": "a ", + "deleteLeft": 0 + }, + "displayAs": "a" + } + ], + [ + { + "transform": { + "insert": "Hazel ", + "deleteLeft": 0 + }, + "displayAs": "Hazel" + }, + { + "transform": { + "insert": "the ", + "deleteLeft": 0 + }, + "displayAs": "the" + }, + { + "transform": { + "insert": "a ", + "deleteLeft": 0 + }, + "displayAs": "a" + } + ] + ]; + + return Model; + }()); + + // It's a 'dummy' model, so there's no need for extra methods and such within the Model's class definition. + LMLayerWorker.loadModel(new models.DummyModel(Model.capabilities, Model.futureSuggestions)); +})(); \ No newline at end of file diff --git a/common/predictive-text/worker/index.ts b/common/predictive-text/worker/index.ts index 88ed9416dc..1fe5960adc 100644 --- a/common/predictive-text/worker/index.ts +++ b/common/predictive-text/worker/index.ts @@ -69,10 +69,18 @@ class LMLayerWorker { */ private _postMessage: PostMessage; + /** + * By default, it's self.importScripts(), but can be overridden + * so that this can be tested **outside of a Worker**. + */ + private _importScripts: ImportScripts; + constructor(options = { + importScripts: null, postMessage: null, }) { this._postMessage = options.postMessage || postMessage; + this._importScripts = options.importScripts || importScripts; this.setupInitialState(); } @@ -126,24 +134,13 @@ class LMLayerWorker { * @param desc Type of the model to instantiate and its parameters. * @param capabilities Capabilities on offer from the keyboard. */ - private loadModel(desc: ModelDescription, capabilities: Capabilities) { - let model: WorkerInternalModel; + public loadModel(model: WorkerInternalModel) { + let capabilities = model.getCapabilities(); let configuration: Configuration = { leftContextCodeUnits: 0, rightContextCodeUnits: 0 }; - if (desc.type === 'dummy') { - model = new models.DummyModel(capabilities, { - futureSuggestions: desc.futureSuggestions - }); - } else if (desc.type === 'wordlist') { - model = new models.WordListModel(capabilities, desc.wordlist); - } else { - throw new Error('Invalid model'); - } - // TODO: when model is object with kind 'wordlist' or 'fst' - // Set reasonable defaults for the configuration. if (!configuration.leftContextCodeUnits) { configuration.leftContextCodeUnits = capabilities.maxLeftContextCodeUnits; @@ -152,7 +149,14 @@ class LMLayerWorker { configuration.rightContextCodeUnits = capabilities.maxRightContextCodeUnits || 0; } - return {model, configuration}; + this.transitionToReadyState(model); + this.cast('ready', { configuration }); + } + + private loadModelFile(url: string) { + // The self/global WebWorker method, allowing us to directly import another script file into WebWorker scope. + // If built correctly, the model's script file will auto-register the model with loadModel() above. + this._importScripts(url); } /** @@ -171,13 +175,7 @@ class LMLayerWorker { } // TODO: validate configuration? - let {model, configuration} = this.loadModel( - // TODO: validate configuration, and provide valid configuration in tests. - payload.model, payload.capabilities - ); - - this.transitionToReadyState(model); - this.cast('ready', { configuration }); + this.loadModelFile(payload.model); } }; } @@ -225,11 +223,12 @@ class LMLayerWorker { * @param scope A global scope to install upon. */ static install(scope: DedicatedWorkerGlobalScope): LMLayerWorker { - let worker = new LMLayerWorker({ postMessage: scope.postMessage }); + let worker = new LMLayerWorker({ postMessage: scope.postMessage, importScripts: scope.importScripts }); scope.onmessage = worker.onMessage.bind(worker); // Ensures that the worker instance is accessible for loaded model scripts. scope['LMLayerWorker'] = worker; + scope['models'] = models; return worker; } diff --git a/common/predictive-text/worker/models/dummy-model.ts b/common/predictive-text/worker/models/dummy-model.ts index 9e90f174e2..e28cd7de29 100644 --- a/common/predictive-text/worker/models/dummy-model.ts +++ b/common/predictive-text/worker/models/dummy-model.ts @@ -33,9 +33,11 @@ namespace models { */ export class DummyModel implements WorkerInternalModel { configuration: Configuration; + capabilities: Capabilities; private _futureSuggestions: Suggestion[][]; constructor(capabilities: Capabilities, options?: any) { + this.capabilities = capabilities; options = options || {}; this.configuration = options.configuration || {}; // Create a shallow copy of the suggestions; @@ -44,6 +46,10 @@ namespace models { ? options.futureSuggestions.slice() : []; } + getCapabilities(): Capabilities { + return this.capabilities; + } + predict(transform: Transform, context: Context, injectedSuggestions?: Suggestion[]): Suggestion[] { if (injectedSuggestions) { return injectedSuggestions; diff --git a/common/predictive-text/worker/models/wordlist-model.ts b/common/predictive-text/worker/models/wordlist-model.ts index e77ca1b7a8..3199cdc9be 100644 --- a/common/predictive-text/worker/models/wordlist-model.ts +++ b/common/predictive-text/worker/models/wordlist-model.ts @@ -41,12 +41,18 @@ const MAX_SUGGESTIONS = 3; export class WordListModel implements WorkerInternalModel { + capabilities: Capabilities; private _wordlist: string[]; constructor(_capabilities: Capabilities, wordlist: string[]) { + this.capabilities = _capabilities; this._wordlist = wordlist; } + getCapabilities(): Capabilities { + return this.capabilities; + } + predict(transform: Transform, context: Context): Suggestion[] { // EVERYTHING to the left of the cursor: let fullLeftContext = context.left || ''; diff --git a/common/predictive-text/worker/worker-interfaces.ts b/common/predictive-text/worker/worker-interfaces.ts index 30ebf86ada..b56368d0d3 100644 --- a/common/predictive-text/worker/worker-interfaces.ts +++ b/common/predictive-text/worker/worker-interfaces.ts @@ -32,6 +32,7 @@ * The signature of self.postMessage(), so that unit tests can mock it. */ type PostMessage = typeof DedicatedWorkerGlobalScope.prototype.postMessage; +type ImportScripts = typeof DedicatedWorkerGlobalScope.prototype.importScripts; /** @@ -48,13 +49,9 @@ interface InitializeMessage { message: 'initialize'; /** - * The model type, and all of its parameters. + * The model's compiled JS file. */ - model: ModelDescription; - /** - * The configuration that the keyboard can offer to the model. - */ - capabilities: Capabilities; + model: string; } /** @@ -104,6 +101,7 @@ interface LMLayerWorkerState { * The model implementation, within the Worker. */ interface WorkerInternalModel { + getCapabilities(): Capabilities; predict(transform: Transform, context: Context): Suggestion[]; } From 2f5b48f625fd6e1f18dc90f039a98bd41ed77089 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 28 Feb 2019 15:04:36 +0700 Subject: [PATCH 04/22] Reinstates the basic headless prediction test. --- .../unit_tests/headless/worker-predict.js | 20 ++++++++++++------- common/predictive-text/unit_tests/helpers.js | 2 +- .../resources/models/simple-dummy.js | 2 +- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/common/predictive-text/unit_tests/headless/worker-predict.js b/common/predictive-text/unit_tests/headless/worker-predict.js index 722e09f3d6..fc15bb3564 100644 --- a/common/predictive-text/unit_tests/headless/worker-predict.js +++ b/common/predictive-text/unit_tests/headless/worker-predict.js @@ -5,7 +5,7 @@ let LMLayerWorker = require('../../build/intermediate'); describe('LMLayerWorker', function () { - describe.skip('#predict()', function () { + describe('#predict()', function () { it('should send back suggestions', function () { var suggestion = { transform: { @@ -17,13 +17,15 @@ describe('LMLayerWorker', function () { // Initialize the worker with a model that will produce one suggestion. var fakePostMessage = sinon.fake(); - var worker = new LMLayerWorker({ postMessage: fakePostMessage }); + var context = { + postMessage: fakePostMessage + }; + context.importScripts = importScriptsWith(context); + + var worker = LMLayerWorker.install(context); worker.onMessage(createMessageEventWithData({ message: 'initialize', - model: dummyModel([ - [suggestion] - ]), - capabilities: defaultCapabilities() + model: "./unit_tests/in_browser/resources/models/simple-dummy.js" })); sinon.assert.calledWithMatch(fakePostMessage.lastCall, { message: 'ready', @@ -38,10 +40,14 @@ describe('LMLayerWorker', function () { transform: zeroTransform(), context: emptyContext() })); + + // Retrieve the internal 'dummy' suggestions for comparison. + var hazel = iGotDistractedByHazel(); + sinon.assert.calledWithMatch(fakePostMessage.lastCall, { message: 'suggestions', token: token, - suggestions: sinon.match.array.deepEquals([suggestion]) + suggestions: hazel[0] }); }); diff --git a/common/predictive-text/unit_tests/helpers.js b/common/predictive-text/unit_tests/helpers.js index 8afb087b0a..bc53215e5c 100644 --- a/common/predictive-text/unit_tests/helpers.js +++ b/common/predictive-text/unit_tests/helpers.js @@ -104,7 +104,7 @@ if (typeof require === 'function') { // This worker-global function does not exist by default in Node! _.importScriptsWith = function(context) { return function() { // the constructed context's importScripts method. - debugger + for(var i=0; i < arguments.length; i++) { context = vm.createContext(context); var script = new vm.Script(fs.readFileSync(arguments[i])); diff --git a/common/predictive-text/unit_tests/in_browser/resources/models/simple-dummy.js b/common/predictive-text/unit_tests/in_browser/resources/models/simple-dummy.js index deebc126a5..03cb8b88de 100644 --- a/common/predictive-text/unit_tests/in_browser/resources/models/simple-dummy.js +++ b/common/predictive-text/unit_tests/in_browser/resources/models/simple-dummy.js @@ -111,5 +111,5 @@ }()); // It's a 'dummy' model, so there's no need for extra methods and such within the Model's class definition. - LMLayerWorker.loadModel(new models.DummyModel(Model.capabilities, Model.futureSuggestions)); + LMLayerWorker.loadModel(new models.DummyModel(Model.capabilities, {futureSuggestions: Model.futureSuggestions})); })(); \ No newline at end of file From 2bf999edee0997cf5832823828115999490f378f Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 28 Feb 2019 15:52:24 +0700 Subject: [PATCH 05/22] Patches up the integrated tests to work with the model loading change. --- common/predictive-text/index.ts | 6 ++--- .../unit_tests/in_browser/base.conf.js | 4 ++++ .../cases/worker-dummy-integration.js | 10 ++------- .../cases/worker-wordlist-integration.js | 13 +++++------ .../unit_tests/in_browser/cases/worker.js | 5 +++-- .../resources/models/simple-wordlist.js | 22 +++++++++++++++++++ common/predictive-text/worker/index.ts | 6 +++-- 7 files changed, 42 insertions(+), 24 deletions(-) create mode 100644 common/predictive-text/unit_tests/in_browser/resources/models/simple-wordlist.js diff --git a/common/predictive-text/index.ts b/common/predictive-text/index.ts index 1e55cb2cce..77686be1e5 100644 --- a/common/predictive-text/index.ts +++ b/common/predictive-text/index.ts @@ -69,14 +69,12 @@ namespace com.keyman.text.prediction { } /** - * Initializes the LMLayer worker with the keyboard/platform's capabilities, - * as well as a description of the model required. + * Initializes the LMLayer worker with a path to the desired model file. */ - initialize(capabilities: Capabilities, model: ModelDescription): Promise { + initialize(model: string): Promise { return new Promise((resolve, _reject) => { this._worker.postMessage({ message: 'initialize', - capabilities, model }); diff --git a/common/predictive-text/unit_tests/in_browser/base.conf.js b/common/predictive-text/unit_tests/in_browser/base.conf.js index 25de1b9c1b..ac61cf3053 100644 --- a/common/predictive-text/unit_tests/in_browser/base.conf.js +++ b/common/predictive-text/unit_tests/in_browser/base.conf.js @@ -59,6 +59,10 @@ module.exports = { variableName: '__json__' }, + proxies: { + "/resources/": "/base/resources/" + }, + // web server port port: 9876, diff --git a/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js b/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js index f68f736da8..5137e042ad 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js +++ b/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js @@ -13,19 +13,13 @@ describe('LMLayer using dummy model', function () { describe('Prediction', function () { it('will predict future suggestions', function () { var lmLayer = new LMLayer(); - var capabilities = { - maxLeftContextCodeUnits: 32 + ~~Math.random() * 32 - }; // We're testing many as asynchronous messages in a row. // this would be cleaner using async/await syntax, but // alas some of our browsers don't support it. return lmLayer.initialize( - capabilities, - { - type: 'dummy', - futureSuggestions: iGotDistractedByHazel() - } + // We need to provide an absolute path since the worker is based within a blob. + document.location.protocol + '//' + document.location.host + "/resources/models/simple-dummy.js" ).then(function (actualConfiguration) { return Promise.resolve(); }).then(function () { diff --git a/common/predictive-text/unit_tests/in_browser/cases/worker-wordlist-integration.js b/common/predictive-text/unit_tests/in_browser/cases/worker-wordlist-integration.js index fb6f1a6503..a5e14af879 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/worker-wordlist-integration.js +++ b/common/predictive-text/unit_tests/in_browser/cases/worker-wordlist-integration.js @@ -9,19 +9,16 @@ describe('LMLayer using the word list model', function () { var EXPECTED_SUGGESTIONS = 3; it('will predict an empty buffer', function () { var lmLayer = new LMLayer(); - var capabilities = { - maxLeftContextCodeUnits: 32 + ~~Math.random() * 32 - }; + // var capabilities = { + // maxLeftContextCodeUnits: 32 + ~~Math.random() * 32 + // }; // We're testing many as asynchronous messages in a row. // this would be cleaner using async/await syntax, but // alas some of our browsers don't support it. return lmLayer.initialize( - capabilities, - { - type: 'wordlist', - wordlist: __json__['wordlists/english-1000'] - } + // We need to provide an absolute path since the worker is based within a blob. + document.location.protocol + '//' + document.location.host + "/resources/models/simple-wordlist.js" ).then(function (_actualConfiguration) { return Promise.resolve(); }).then(function () { diff --git a/common/predictive-text/unit_tests/in_browser/cases/worker.js b/common/predictive-text/unit_tests/in_browser/cases/worker.js index 544c35d638..be81f4c5ef 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/worker.js +++ b/common/predictive-text/unit_tests/in_browser/cases/worker.js @@ -2,6 +2,7 @@ var assert = chai.assert; var LMLayer = com.keyman.text.prediction.LMLayer; describe('LMLayerWorker', function () { + this.timeout(5000); describe('LMLayerWorkerCode', function() { it('should exist!', function() { assert.isFunction(LMLayerWorkerCode, @@ -19,8 +20,8 @@ describe('LMLayerWorker', function () { }; worker.postMessage({ message: 'initialize', - model: { type: 'dummy' }, - capabilities: { maxLeftContextCodeUnits: 64 } + // Since the worker's based in a blob, it's not on the 'same domain'. We need to absolute-path the model file. + model: document.location.protocol + '//' + document.location.host + "/resources/models/simple-dummy.js" }); }); }); diff --git a/common/predictive-text/unit_tests/in_browser/resources/models/simple-wordlist.js b/common/predictive-text/unit_tests/in_browser/resources/models/simple-wordlist.js new file mode 100644 index 0000000000..337062cdcf --- /dev/null +++ b/common/predictive-text/unit_tests/in_browser/resources/models/simple-wordlist.js @@ -0,0 +1,22 @@ +/** + * While handwritten, this class is designed to mirror the potential results of TypeScript compilation of + * model TS source files. + */ +(function(){ + var Model = /** @class */ (function() { + function Model() { // implements Model + } + + Model.capabilities = { + maxLeftContextCodeUnits: 64 + } + + // A direct import/copy from i_got_distracted_by_hazel.json. + Model.wordlist = ["the", "of", "and", "to", "a", "in", "that", "is", "was", "he", "for", "it", "with", "as", "his", "on", "be", "at", "by", "i", "this", "had", "not", "are", "but", "from", "or", "have", "an", "they", "which", "one", "you", "were", "her", "all", "she", "there", "would", "their", "we", "him", "been", "has", "when", "who", "will", "more", "if", "no", "out", "so", "said", "what", "up", "its", "about", "into", "than", "them", "can", "only", "other", "new", "some", "could", "time", "these", "two", "may", "then", "do", "first", "any", "my", "now", "such", "like", "our", "over", "man", "me", "even", "most", "made", "after", "also", "did", "many", "before", "must", "af", "through", "back", "years", "much", "where", "your", "way", "well", "down", "should", "because", "each", "just", "those", "people", "too", "how", "little", "state", "good", "very", "make", "world", "still", "own", "see", "men", "work", "long", "here", "get", "between", "both", "life", "being", "under", "never", "day", "same", "another", "know", "while", "last", "us", "might", "great", "old", "year", "off", "come", "since", "against", "go", "came", "right", "used", "three", "take", "himself", "states", "few", "use", "house", "during", "without", "again", "place", "american", "around", "however", "home", "small", "found", "thought", "went", "say", "part", "once", "general", "high", "upon", "school", "every", "does", "got", "united", "left", "number", "course", "war", "until", "always", "away", "something", "fact", "water", "though", "public", "put", "less", "think", "almost", "hand", "enough", "took", "far", "head", "yet", "government", "system", "set", "better", "told", "nothing", "night", "end", "why", "called", "eyes", "find", "look", "going", "asked", "later", "point", "knew", "next", "program", "city", "business", "group", "give", "toward", "young", "let", "room", "days", "president", "side", "social", "given", "present", "several", "order", "national", "possible", "rather", "second", "face", "per", "among", "form", "often", "important", "things", "looked", "early", "white", "case", "john", "large", "four", "need", "big", "within", "become", "felt", "along", "children", "saw", "best", "church", "ever", "least", "power", "development", "thing", "seemed", "light", "family", "interest", "want", "mind", "members", "area", "country", "others", "although", "turned", "done", "open", "god", "service", "problem", "kind", "certain", "different", "thus", "began", "door", "sense", "help", "means", "whole", "matter", "perhaps", "itself", "york", "times", "human", "law", "line", "above", "name", "example", "action", "company", "hands", "local", "show", "whether", "history", "five", "gave", "either", "today", "act", "feet", "across", "quite", "taken", "past", "anything", "having", "seen", "death", "experience", "body", "really", "half", "week", "car", "field", "words", "word", "already", "themselves", "information", "tell", "shall", "together", "college", "money", "period", "keep", "held", "sure", "probably", "free", "seems", "behind", "real", "cannot", "political", "question", "air", "making", "office", "brought", "miss", "whose", "special", "problems", "major", "heard", "became", "moment", "study", "ago", "federal", "known", "available", "street", "result", "economic", "boy", "reason", "position", "south", "change", "board", "individual", "job", "society", "am", "areas", "west", "close", "turn", "love", "community", "true", "force", "full", "court", "seem", "cost", "wife", "age", "future", "wanted", "voice", "department", "center", "woman", "common", "control", "necessary", "policy", "front", "following", "sometimes", "girl", "six", "clear", "further", "land", "music", "feel", "mother", "able", "party", "provide", "university", "education", "level", "run", "students", "effect", "child", "stood", "town", "short", "military", "total", "morning", "outside", "rate", "figure", "art", "class", "century", "usually", "north", "washington", "leave", "therefore", "plan", "top", "evidence", "sound", "million", "black", "hard", "strong", "various", "tax", "believe", "type", "value", "says", "play", "surface", "mean", "soon", "modern", "near", "peace", "lines", "table", "book", "road", "red", "situation", "personal", "minutes", "process", "nor", "idea", "alone", "women", "english", "schools", "gone", "increase", "living", "america", "started", "longer", "cut", "finally", "private", "nature", "secretary", "third", "section", "months", "greater", "call", "fire", "needed", "expected", "view", "values", "kept", "ground", "everything", "pressure", "dark", "basis", "space", "father", "east", "spirit", "required", "union", "complete", "wrote", "except", "moved", "support", "return", "conditions", "recent", "particular", "attention", "late", "hope", "live", "else", "costs", "brown", "beyond", "stage", "taking", "material", "nations", "forces", "report", "inside", "dead", "read", "coming", "person", "hours", "heart", "instead", "low", "miles", "lost", "looking", "data", "added", "makes", "single", "followed", "amount", "pay", "feeling", "basic", "including", "simply", "move", "cold", "hundred", "research", "industry", "tried", "developed", "reached", "hold", "committee", "defense", "equipment", "island", "actually", "shown", "son", "central", "religious", "river", "getting", "beginning", "ten", "sort", "received", "rest", "terms", "doing", "trying", "indeed", "care", "friends", "medical", "picture", "especially", "administration", "subject", "fine", "difficult", "building", "higher", "simple", "wall", "walked", "meeting", "bring", "floor", "foreign", "passed", "similar", "paper", "range", "natural", "final", "property", "training", "police", "international", "cent", "county", "market", "growth", "england", "talk", "written", "start", "story", "suddenly", "hear", "issue", "hall", "answer", "needs", "congress", "working", "likely", "considered", "countries", "earth", "sat", "entire", "meet", "purpose", "labor", "happened", "results", "difference", "cases", "stand", "hair", "william", "production", "stock", "involved", "fall", "food", "whom", "earlier", "increased", "particularly", "thinking", "club", "letter", "knowledge", "below", "effort", "hour", "using", "paid", "sent", "christian", "yes", "points", "boys", "industrial", "bill", "deal", "ready", "ideas", "certainly", "square", "blue", "trade", "moral", "bad", "due", "methods", "girls", "method", "addition", "neither", "showed", "throughout", "directly", "nearly", "statement", "decided", "reading", "weeks", "according", "questions", "anyone", "try", "color", "kennedy", "programs", "nation", "services", "lay", "french", "remember", "physical", "size", "member", "understand", "western", "strength", "record", "comes", "southern", "normal", "population", "appeared", "merely", "concerned", "district", "volume", "temperature", "direction", "trial", "trouble", "summer", "aid", "ran", "friend", "sales", "literature", "list", "maybe", "continued", "evening", "association", "generally", "influence", "led", "provided", "met", "army", "science", "changes", "husband", "step", "chance", "student", "opened", "former", "average", "month", "series", "hot", "works", "cause", "lead", "direct", "systems", "planning", "stopped", "myself", "theory", "piece", "wrong", "effective", "george", "soviet", "freedom", "worked", "ask", "movement", "organization", "clearly", "ways", "treatment", "fear", "beautiful", "meaning", "consider", "bed", "spring", "somewhat", "press", "lot", "forms", "efforts", "note", "truth", "placed", "hotel", "carried", "numbers", "respect", "plant", "herself", "apparently", "groups", "degree", "wide", "easy", "manner", "reaction", "farm", "lower", "recently", "approach", "immediately", "game", "larger", "running", "daily", "charge", "couple", "eye", "oh", "performance", "feed", "de", "persons", "c", "understanding", "arms", "blood", "opportunity", "march", "progress", "additional", "described", "technical", "stop", "fiscal", "radio", "religion", "test", "reported", "main", "based", "steps", "image", "chief", "served", "window", "decision", "determined", "responsibility", "character", "middle", "british", "aj", "europe", "gun", "writing", "horse", "appear", "learned", "account", "serious", "ones", "types", "activity", "green", "length", "letters", "nuclear", "specific", "slowly", "forward", "returned", "lived", "activities", "corner", "obtained", "audience", "justice", "doubt", "latter", "hit", "plane", "gives", "moving", "obviously", "straight", "design", "quality", "saying", "function", "choice", "staff", "include", "pattern", "figures", "born", "parts", "stay", "seven", "poor", "plans", "operation", "shot", "whatever", "cars", "sun", "faith", "pool", "extent", "speak", "heavy", "completely", "lack", "waiting", "standard", "hospital", "ball", "wish", "corps", "visit", "language", "ahead", "deep", "income", "principle", "democratic", "firm", "importance", "growing", "analysis", "designed", "effects", "price", "none", "distance", "indicated", "established", "products", "expect", "division", "continue", "leaders", "existence", "determine", "serve", "stress", "attitude", "pretty", "elements", "easily", "cities", "negro", "remained", "factors", "hardly", "applied", "limited", "closed", "agreement", "scene", "write", "afternoon", "b", "suggested", "health", "professional", "attack", "reach", "drive", "season", "interested", "station", "rhode", "married", "despite", "covered", "role", "becomes", "eight", "current", "played", "spent", "reasons", "council", "unit", "built", "commission", "date", "mouth", "exactly", "original", "studies", "race", "machine"]; + + return Model; + }()); + + // It's a 'dummy' model, so there's no need for extra methods and such within the Model's class definition. + LMLayerWorker.loadModel(new models.WordListModel(Model.capabilities, Model.wordlist)); +})(); \ No newline at end of file diff --git a/common/predictive-text/worker/index.ts b/common/predictive-text/worker/index.ts index 1fe5960adc..7aa85f1a60 100644 --- a/common/predictive-text/worker/index.ts +++ b/common/predictive-text/worker/index.ts @@ -75,9 +75,11 @@ class LMLayerWorker { */ private _importScripts: ImportScripts; + private _hostURL: string; + constructor(options = { importScripts: null, - postMessage: null, + postMessage: null }) { this._postMessage = options.postMessage || postMessage; this._importScripts = options.importScripts || importScripts; @@ -223,7 +225,7 @@ class LMLayerWorker { * @param scope A global scope to install upon. */ static install(scope: DedicatedWorkerGlobalScope): LMLayerWorker { - let worker = new LMLayerWorker({ postMessage: scope.postMessage, importScripts: scope.importScripts }); + let worker = new LMLayerWorker({ postMessage: scope.postMessage, importScripts: scope.importScripts.bind(scope) }); scope.onmessage = worker.onMessage.bind(worker); // Ensures that the worker instance is accessible for loaded model scripts. From 3c086a8ef6e85ce12d345053475d8427497e3d81 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 4 Mar 2019 08:28:17 +0700 Subject: [PATCH 06/22] Fixes LMLayer outer-layer init tests. --- common/predictive-text/index.ts | 2 +- .../unit_tests/headless/top-level-lmlayer.js | 25 +++---------------- 2 files changed, 5 insertions(+), 22 deletions(-) diff --git a/common/predictive-text/index.ts b/common/predictive-text/index.ts index 77686be1e5..0c43d6bd4b 100644 --- a/common/predictive-text/index.ts +++ b/common/predictive-text/index.ts @@ -75,7 +75,7 @@ namespace com.keyman.text.prediction { return new Promise((resolve, _reject) => { this._worker.postMessage({ message: 'initialize', - model + model: model }); // Sets up so the promise is resolved in the onMessage() callback, when it receives diff --git a/common/predictive-text/unit_tests/headless/top-level-lmlayer.js b/common/predictive-text/unit_tests/headless/top-level-lmlayer.js index 7ec7e33b48..fdf8cfb0de 100644 --- a/common/predictive-text/unit_tests/headless/top-level-lmlayer.js +++ b/common/predictive-text/unit_tests/headless/top-level-lmlayer.js @@ -17,15 +17,7 @@ describe('LMLayer', function() { let fakeWorker = createFakeWorker(); let lmLayer = new LMLayer(fakeWorker); - lmLayer.initialize( - { - maxLeftContextCodeUnits: 32, - }, - { - kind: 'wordlist', - words: ['foo', 'bar', 'baz', 'quux'] - } - ); + lmLayer.initialize("./unit_tests/in_browser/resources/models/simple-dummy.js"); assert.isFunction(fakeWorker.onmessage, 'LMLayer failed to set a callback!'); }); @@ -33,23 +25,14 @@ describe('LMLayer', function() { it('should send the `initialize` message to the LMLayer', async function () { let fakeWorker = createFakeWorker(fakePostMessage); let lmLayer = new LMLayer(fakeWorker); - let configuration = await lmLayer.initialize( - { - maxLeftContextCodeUnits: 32, - }, - { - kind: 'wordlist', - words: ['foo', 'bar', 'baz', 'quux'] - } - ); + let configuration = await lmLayer.initialize("./unit_tests/in_browser/resources/models/simple-dummy.js"); assert.propertyVal(fakeWorker.postMessage, 'callCount', 1); // In the "Worker", assert the message looks right and // ASYNCHRONOUSLY reply with ready message. function fakePostMessage(data) { - assert.propertyVal(data, 'message', 'initialize') - assert.isObject(data.capabilities); - assert.isObject(data.model); + assert.propertyVal(data, 'message', 'initialize'); + assert.isString(data.model); callAsynchronously(() => fakeWorker.onmessage({ data: { From 320ac0901d200ee618c20dcc08f3dd2c8bddee30 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 4 Mar 2019 09:01:34 +0700 Subject: [PATCH 07/22] Implements a shutdown() function on LMLayer, killing the worker. --- common/predictive-text/index.ts | 8 ++++++++ .../unit_tests/in_browser/cases/top-level-lmlayer.js | 2 ++ .../in_browser/cases/worker-dummy-integration.js | 1 + .../in_browser/cases/worker-wordlist-integration.js | 1 + .../unit_tests/in_browser/cases/worker.js | 1 + common/predictive-text/worker/index.ts | 11 +++++++++++ 6 files changed, 24 insertions(+) diff --git a/common/predictive-text/index.ts b/common/predictive-text/index.ts index 0c43d6bd4b..88ea5c1de1 100644 --- a/common/predictive-text/index.ts +++ b/common/predictive-text/index.ts @@ -113,6 +113,14 @@ namespace com.keyman.text.prediction { } } + /** + * Clears out any computational resources in use by the LMLayer, including shutting + * down any internal WebWorkers. + */ + public shutdown() { + this._worker.terminate(); + } + /** * Given a function, this utility returns the source code within it, as a string. * This is intended to unwrap the "wrapped" source code created in the LMLayerWorker diff --git a/common/predictive-text/unit_tests/in_browser/cases/top-level-lmlayer.js b/common/predictive-text/unit_tests/in_browser/cases/top-level-lmlayer.js index 2083e2e29c..a97e84b20d 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/top-level-lmlayer.js +++ b/common/predictive-text/unit_tests/in_browser/cases/top-level-lmlayer.js @@ -6,6 +6,7 @@ describe('LMLayer', function () { it('should construct with zero arguments', function () { let lmLayer = new LMLayer(); assert.instanceOf(lmLayer, LMLayer); + lmLayer.shutdown(); }); }); @@ -25,6 +26,7 @@ describe('LMLayer', function () { let worker = new Worker(uri); worker.onmessage = function thisShouldBeCalled(event) { assert.propertyVal(event, 'data', 'fhqwhgads'); + worker.terminate(); done(); }; }) diff --git a/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js b/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js index 5137e042ad..360699bf84 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js +++ b/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js @@ -35,6 +35,7 @@ describe('LMLayer using dummy model', function () { return lmLayer.predict(zeroTransform(), emptyContext()); }).then(function (suggestions) { assert.deepEqual(suggestions, iGotDistractedByHazel()[3]); + lmLayer.shutdown(); return Promise.resolve(); }); }); diff --git a/common/predictive-text/unit_tests/in_browser/cases/worker-wordlist-integration.js b/common/predictive-text/unit_tests/in_browser/cases/worker-wordlist-integration.js index a5e14af879..281c652e9f 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/worker-wordlist-integration.js +++ b/common/predictive-text/unit_tests/in_browser/cases/worker-wordlist-integration.js @@ -35,6 +35,7 @@ describe('LMLayer using the word list model', function () { return lmLayer.predict(type('q'), atEndOfBuffer('the ')); }).then(function (suggestions) { assert.isAtLeast(suggestions.length, EXPECTED_SUGGESTIONS); + lmLayer.shutdown(); return Promise.resolve(); }); }); diff --git a/common/predictive-text/unit_tests/in_browser/cases/worker.js b/common/predictive-text/unit_tests/in_browser/cases/worker.js index be81f4c5ef..693967323b 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/worker.js +++ b/common/predictive-text/unit_tests/in_browser/cases/worker.js @@ -17,6 +17,7 @@ describe('LMLayerWorker', function () { let worker = new Worker(uri); worker.onmessage = function thisShouldBeCalled(message) { done(); + worker.terminate(); }; worker.postMessage({ message: 'initialize', diff --git a/common/predictive-text/worker/index.ts b/common/predictive-text/worker/index.ts index 7aa85f1a60..6621cc818c 100644 --- a/common/predictive-text/worker/index.ts +++ b/common/predictive-text/worker/index.ts @@ -72,6 +72,9 @@ class LMLayerWorker { /** * By default, it's self.importScripts(), but can be overridden * so that this can be tested **outside of a Worker**. + * + * To function properly, self.importScripts() must be bound to self + * before being stored here, else it will fail. */ private _importScripts: ImportScripts; @@ -161,6 +164,13 @@ class LMLayerWorker { this._importScripts(url); } + public unloadModel() { + // Right now, this seems sufficient to clear out the old model. + // The only existing reference to a loaded model is held by + // transitionToReadyState's `handleMessage` closure. (The `model` var) + this.setupInitialState(); + } + /** * Sets the initial state, i.e., `uninitialized`. * This state only handles `initialized` messages, and will @@ -229,6 +239,7 @@ class LMLayerWorker { scope.onmessage = worker.onMessage.bind(worker); // Ensures that the worker instance is accessible for loaded model scripts. + // Assists unit-testing. scope['LMLayerWorker'] = worker; scope['models'] = models; From 393dc5b2706e555f140c188eefd07abb6e6bc312 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 4 Mar 2019 10:25:22 +0700 Subject: [PATCH 08/22] LMLayer.initialize -> .activateModel, adds .deactivateModel. --- common/predictive-text/index.d.ts | 7 ++++++- common/predictive-text/index.ts | 15 +++++++++++-- .../unit_tests/headless/top-level-lmlayer.js | 10 ++++----- .../cases/worker-dummy-integration.js | 2 +- .../cases/worker-wordlist-integration.js | 2 +- common/predictive-text/worker/index.ts | 21 ++++++++++++------- .../worker/worker-interfaces.ts | 7 +++++-- 7 files changed, 44 insertions(+), 20 deletions(-) diff --git a/common/predictive-text/index.d.ts b/common/predictive-text/index.d.ts index 25e9414f4b..a6f133e393 100644 --- a/common/predictive-text/index.d.ts +++ b/common/predictive-text/index.d.ts @@ -24,7 +24,12 @@ declare namespace com.keyman.text.prediction { * Initializes the LMLayer worker with the keyboard/platform's capabilities, * as well as a description of the model required. */ - initialize(capabilities: Capabilities, model: ModelDescription): Promise; + activateModel(model: string): Promise; + + /** + * Prepares the LMLayer for reinitialization with a different model/capability set. + */ + deactivateModel(); predict(transform: Transform, context: Context): Promise; diff --git a/common/predictive-text/index.ts b/common/predictive-text/index.ts index 88ea5c1de1..9da01af96b 100644 --- a/common/predictive-text/index.ts +++ b/common/predictive-text/index.ts @@ -34,9 +34,10 @@ * Since the Worker runs in a different thread, the public methods of this class are * asynchronous. Methods of note include: * - * - #initialize() -- initialize the LMLayer with a configuration and language model + * - #activateModel() -- initialize the LMLayer with a configuration and language model * - #predict() -- ask the LMLayer to offer suggestions (predictions or corrections) for * the input event + * - #deactivateModel() -- de-initializes the LMLayer, preparing it for re-initialization * * The top-level LMLayer will automatically starts up its own Web Worker. */ @@ -71,7 +72,7 @@ namespace com.keyman.text.prediction { /** * Initializes the LMLayer worker with a path to the desired model file. */ - initialize(model: string): Promise { + activateModel(model: string): Promise { return new Promise((resolve, _reject) => { this._worker.postMessage({ message: 'initialize', @@ -84,6 +85,16 @@ namespace com.keyman.text.prediction { }); } + /** + * Unloads the previously-active model from memory, resetting the LMLayer to prep + * for transition to use of a new model. + */ + public deactivateModel() { + this._worker.postMessage({ + message: 'unload' + }); + } + predict(transform: Transform, context: Context): Promise { let token = this._nextToken++; return new Promise((resolve, reject) => { diff --git a/common/predictive-text/unit_tests/headless/top-level-lmlayer.js b/common/predictive-text/unit_tests/headless/top-level-lmlayer.js index fdf8cfb0de..15785bd525 100644 --- a/common/predictive-text/unit_tests/headless/top-level-lmlayer.js +++ b/common/predictive-text/unit_tests/headless/top-level-lmlayer.js @@ -12,12 +12,12 @@ describe('LMLayer', function() { }); }); - describe('#initialize()', function () { + describe('#activateModel()', function () { it('should accept capabilities and model description', function () { let fakeWorker = createFakeWorker(); let lmLayer = new LMLayer(fakeWorker); - lmLayer.initialize("./unit_tests/in_browser/resources/models/simple-dummy.js"); + lmLayer.activateModel("./unit_tests/in_browser/resources/models/simple-dummy.js"); assert.isFunction(fakeWorker.onmessage, 'LMLayer failed to set a callback!'); }); @@ -25,7 +25,7 @@ describe('LMLayer', function() { it('should send the `initialize` message to the LMLayer', async function () { let fakeWorker = createFakeWorker(fakePostMessage); let lmLayer = new LMLayer(fakeWorker); - let configuration = await lmLayer.initialize("./unit_tests/in_browser/resources/models/simple-dummy.js"); + let configuration = await lmLayer.activateModel("./unit_tests/in_browser/resources/models/simple-dummy.js"); assert.propertyVal(fakeWorker.postMessage, 'callCount', 1); // In the "Worker", assert the message looks right and @@ -59,7 +59,7 @@ describe('LMLayer', function() { }); let lmLayer = new LMLayer(fakeWorker); - let actualConfiguration = await lmLayer.initialize( + let actualConfiguration = await lmLayer.activateModel( { maxLeftContextCodeUnits: 32, }, @@ -69,7 +69,7 @@ describe('LMLayer', function() { } ); - // This SHOULD be called by initialize(). + // This SHOULD be called by activateModel(). assert.deepEqual(actualConfiguration, expectedConfiguration); }) }); diff --git a/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js b/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js index 360699bf84..39e088ede1 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js +++ b/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js @@ -17,7 +17,7 @@ describe('LMLayer using dummy model', function () { // We're testing many as asynchronous messages in a row. // this would be cleaner using async/await syntax, but // alas some of our browsers don't support it. - return lmLayer.initialize( + return lmLayer.activateModel( // We need to provide an absolute path since the worker is based within a blob. document.location.protocol + '//' + document.location.host + "/resources/models/simple-dummy.js" ).then(function (actualConfiguration) { diff --git a/common/predictive-text/unit_tests/in_browser/cases/worker-wordlist-integration.js b/common/predictive-text/unit_tests/in_browser/cases/worker-wordlist-integration.js index 281c652e9f..124677ce1d 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/worker-wordlist-integration.js +++ b/common/predictive-text/unit_tests/in_browser/cases/worker-wordlist-integration.js @@ -16,7 +16,7 @@ describe('LMLayer using the word list model', function () { // We're testing many as asynchronous messages in a row. // this would be cleaner using async/await syntax, but // alas some of our browsers don't support it. - return lmLayer.initialize( + return lmLayer.activateModel( // We need to provide an absolute path since the worker is based within a blob. document.location.protocol + '//' + document.location.host + "/resources/models/simple-wordlist.js" ).then(function (_actualConfiguration) { diff --git a/common/predictive-text/worker/index.ts b/common/predictive-text/worker/index.ts index 6621cc818c..bcd317d3f1 100644 --- a/common/predictive-text/worker/index.ts +++ b/common/predictive-text/worker/index.ts @@ -203,15 +203,20 @@ class LMLayerWorker { this.state = { name: 'ready', handleMessage: (payload) => { - if (payload.message !== 'predict') { - throw new Error(`invalid message; expected 'predict' but got ${payload.message}`); + switch(payload.message) { + case 'predict': + let {transform, context} = payload; + this.cast('suggestions', { + token: payload.token, + suggestions: model.predict(transform, context) + }); + break; + case 'unload': + this.unloadModel(); + break; + default: + throw new Error(`invalid message; expected one of {'predict'} but got ${payload.message}`); } - - let {transform, context} = payload; - this.cast('suggestions', { - token: payload.token, - suggestions: model.predict(transform, context) - }); } }; } diff --git a/common/predictive-text/worker/worker-interfaces.ts b/common/predictive-text/worker/worker-interfaces.ts index b56368d0d3..500fef750a 100644 --- a/common/predictive-text/worker/worker-interfaces.ts +++ b/common/predictive-text/worker/worker-interfaces.ts @@ -38,8 +38,8 @@ type ImportScripts = typeof DedicatedWorkerGlobalScope.prototype.importScripts; /** * The valid incoming message kinds. */ -type IncomingMessageKind = 'initialize' | 'predict'; -type IncomingMessage = InitializeMessage | PredictMessage; +type IncomingMessageKind = 'initialize' | 'predict' | 'unload'; +type IncomingMessage = InitializeMessage | PredictMessage | UnloadMessage; /** * The structure of an initialization message. It should include the model (either in @@ -83,6 +83,9 @@ interface PredictMessage { context: Context; } +interface UnloadMessage { + message: 'unload' +} /** From aeae40ae213a37ac6d2cadfe3e6b724ab56c07c5 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Tue, 5 Mar 2019 08:12:54 +0700 Subject: [PATCH 09/22] Links KMW to the reworked LMLayer API. --- web/source/kmwbase.ts | 1 + web/source/text/prediction/modelManager.ts | 14 ++++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/web/source/kmwbase.ts b/web/source/kmwbase.ts index 6f3e02694d..eee76fe48e 100644 --- a/web/source/kmwbase.ts +++ b/web/source/kmwbase.ts @@ -159,6 +159,7 @@ namespace com.keyman { this.osk.shutdown(); this.util.shutdown(); this.keyboardManager.shutdown(); + this.modelManager.shutdown(); if(this.ui && this.ui.shutdown) { this.ui.shutdown(); diff --git a/web/source/text/prediction/modelManager.ts b/web/source/text/prediction/modelManager.ts index eee71ac101..34abcdd8c7 100644 --- a/web/source/text/prediction/modelManager.ts +++ b/web/source/text/prediction/modelManager.ts @@ -43,7 +43,7 @@ namespace com.keyman.text.prediction { } private deactivateModel() { - // TODO: Call a LMLayer method for model deactivation. + this.lmEngine.deactivateModel(); this.currentModel = null; } @@ -52,12 +52,8 @@ namespace com.keyman.text.prediction { throw new Error("Null reference not allowed."); } - // TODO: Activate this model within the LMLayer! let file = model.path; - - //this.lmEngine.initialize(file) // Currently unsupported. - console.log("Model detected!"); - + this.lmEngine.activateModel(file); this.currentModel = model; } @@ -102,5 +98,11 @@ namespace com.keyman.text.prediction { isRegistered(model: ModelSpec): boolean { return !! this.registeredModels[model.id]; } + + // TODO: actually calling this.lmEngine.predict. Will need its own method(s). + + public shutdown() { + this.lmEngine.shutdown(); + } } } \ No newline at end of file From 49bae3c774a6d5ac0dc03afd2092ef21cd9219e3 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Tue, 5 Mar 2019 16:16:42 +0700 Subject: [PATCH 10/22] Adds basic definitions for pending word-breaker implementation. --- common/predictive-text/index.d.ts | 2 +- common/predictive-text/worker/defaultWordBreaker.ts | 9 +++++++++ common/predictive-text/worker/index.ts | 4 ++++ common/predictive-text/worker/tsconfig.json | 1 + common/predictive-text/worker/worker-interfaces.ts | 4 ++++ 5 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 common/predictive-text/worker/defaultWordBreaker.ts diff --git a/common/predictive-text/index.d.ts b/common/predictive-text/index.d.ts index a6f133e393..c0df793eed 100644 --- a/common/predictive-text/index.d.ts +++ b/common/predictive-text/index.d.ts @@ -29,7 +29,7 @@ declare namespace com.keyman.text.prediction { /** * Prepares the LMLayer for reinitialization with a different model/capability set. */ - deactivateModel(); + deactivateModel(): void; predict(transform: Transform, context: Context): Promise; diff --git a/common/predictive-text/worker/defaultWordBreaker.ts b/common/predictive-text/worker/defaultWordBreaker.ts new file mode 100644 index 0000000000..64ca4081ef --- /dev/null +++ b/common/predictive-text/worker/defaultWordBreaker.ts @@ -0,0 +1,9 @@ +class DefaultWordBreaker implements WorkerInternalWordBreaker { + // TODO: Specify the data needed to implement a 'default' word breaker. + constructor(obj) { + } + + break(text: string): string[] { + throw "Not yet implemented."; + } +} \ No newline at end of file diff --git a/common/predictive-text/worker/index.ts b/common/predictive-text/worker/index.ts index bcd317d3f1..ae7e8cfa7a 100644 --- a/common/predictive-text/worker/index.ts +++ b/common/predictive-text/worker/index.ts @@ -170,6 +170,10 @@ class LMLayerWorker { // transitionToReadyState's `handleMessage` closure. (The `model` var) this.setupInitialState(); } + + public loadWordBreaker(breaker: WorkerInternalWordBreaker) { + // TODO: Actually store it somewhere for future use. Make sure we can forget it with `unloadModel` as well. + } /** * Sets the initial state, i.e., `uninitialized`. diff --git a/common/predictive-text/worker/tsconfig.json b/common/predictive-text/worker/tsconfig.json index a8625d5ab3..ff5d48e451 100644 --- a/common/predictive-text/worker/tsconfig.json +++ b/common/predictive-text/worker/tsconfig.json @@ -1,6 +1,7 @@ { "compilerOptions": { "allowJs": false, + "declaration": true, "module": "none", "outFile": "../build/intermediate/index.js", "inlineSources": true, diff --git a/common/predictive-text/worker/worker-interfaces.ts b/common/predictive-text/worker/worker-interfaces.ts index 500fef750a..9a2b7792b8 100644 --- a/common/predictive-text/worker/worker-interfaces.ts +++ b/common/predictive-text/worker/worker-interfaces.ts @@ -118,3 +118,7 @@ interface WorkerInternalModelConstructor { */ new(capabilities: Capabilities, ...modelParameters: any[]): WorkerInternalModel; } + +interface WorkerInternalWordBreaker { + break(text: string): string[]; // +} \ No newline at end of file From 263af9c721d3bc468eeadad97173165c2379ace3 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 6 Mar 2019 08:24:57 +0700 Subject: [PATCH 11/22] Addresses many of Eddie's PR review comments. --- common/predictive-text/index.ts | 6 +++--- .../unit_tests/headless/worker-initialization.js | 1 - common/predictive-text/unit_tests/helpers.js | 10 +++++++--- .../in_browser/cases/worker-wordlist-integration.js | 3 --- .../in_browser/resources/models/simple-wordlist.js | 4 ++-- common/predictive-text/worker/index.ts | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/common/predictive-text/index.ts b/common/predictive-text/index.ts index 9da01af96b..65535bb476 100644 --- a/common/predictive-text/index.ts +++ b/common/predictive-text/index.ts @@ -34,7 +34,7 @@ * Since the Worker runs in a different thread, the public methods of this class are * asynchronous. Methods of note include: * - * - #activateModel() -- initialize the LMLayer with a configuration and language model + * - #activateModel() -- initialize the LMLayer by loading a specified model file * - #predict() -- ask the LMLayer to offer suggestions (predictions or corrections) for * the input event * - #deactivateModel() -- de-initializes the LMLayer, preparing it for re-initialization @@ -72,11 +72,11 @@ namespace com.keyman.text.prediction { /** * Initializes the LMLayer worker with a path to the desired model file. */ - activateModel(model: string): Promise { + activateModel(modelFilePath: string): Promise { return new Promise((resolve, _reject) => { this._worker.postMessage({ message: 'initialize', - model: model + model: modelFilePath }); // Sets up so the promise is resolved in the onMessage() callback, when it receives diff --git a/common/predictive-text/unit_tests/headless/worker-initialization.js b/common/predictive-text/unit_tests/headless/worker-initialization.js index 096fad0bb5..b74425d234 100644 --- a/common/predictive-text/unit_tests/headless/worker-initialization.js +++ b/common/predictive-text/unit_tests/headless/worker-initialization.js @@ -18,7 +18,6 @@ describe('LMLayerWorker', function() { context.importScripts = importScriptsWith(context); var worker = LMLayerWorker.install(context); - //var worker = new LMLayerWorker({ postMessage: fakePostMessage }); // Sending it the initialize it should notify us that it's initialized! worker.onMessage(createMessageEventWithData({ diff --git a/common/predictive-text/unit_tests/helpers.js b/common/predictive-text/unit_tests/helpers.js index bc53215e5c..8fe7522f74 100644 --- a/common/predictive-text/unit_tests/helpers.js +++ b/common/predictive-text/unit_tests/helpers.js @@ -4,6 +4,9 @@ * Globally-defined helper functions for use in in Mocha tests. */ +var fs = require("fs"); +var vm = require("vm"); + // Choose the appropriate global object. Either `global` in // Node, or `window` in browsers. var _ = global || window; @@ -98,13 +101,14 @@ if (typeof require === 'function') { return require('./in_browser/json/' + name); } - var fs = require("fs"); - var vm = require("vm"); - // This worker-global function does not exist by default in Node! _.importScriptsWith = function(context) { return function() { // the constructed context's importScripts method. + /* Use of vm.createContext and script.runInContext allow us to avoid + * polluting the global scope with imports. When we throw away the + * context object, imported scripts will be automatically GC'd. + */ for(var i=0; i < arguments.length; i++) { context = vm.createContext(context); var script = new vm.Script(fs.readFileSync(arguments[i])); diff --git a/common/predictive-text/unit_tests/in_browser/cases/worker-wordlist-integration.js b/common/predictive-text/unit_tests/in_browser/cases/worker-wordlist-integration.js index 124677ce1d..eb558083e4 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/worker-wordlist-integration.js +++ b/common/predictive-text/unit_tests/in_browser/cases/worker-wordlist-integration.js @@ -9,9 +9,6 @@ describe('LMLayer using the word list model', function () { var EXPECTED_SUGGESTIONS = 3; it('will predict an empty buffer', function () { var lmLayer = new LMLayer(); - // var capabilities = { - // maxLeftContextCodeUnits: 32 + ~~Math.random() * 32 - // }; // We're testing many as asynchronous messages in a row. // this would be cleaner using async/await syntax, but diff --git a/common/predictive-text/unit_tests/in_browser/resources/models/simple-wordlist.js b/common/predictive-text/unit_tests/in_browser/resources/models/simple-wordlist.js index 337062cdcf..ec58dfd405 100644 --- a/common/predictive-text/unit_tests/in_browser/resources/models/simple-wordlist.js +++ b/common/predictive-text/unit_tests/in_browser/resources/models/simple-wordlist.js @@ -11,12 +11,12 @@ maxLeftContextCodeUnits: 64 } - // A direct import/copy from i_got_distracted_by_hazel.json. + // A direct import/copy from english-1000.json. Model.wordlist = ["the", "of", "and", "to", "a", "in", "that", "is", "was", "he", "for", "it", "with", "as", "his", "on", "be", "at", "by", "i", "this", "had", "not", "are", "but", "from", "or", "have", "an", "they", "which", "one", "you", "were", "her", "all", "she", "there", "would", "their", "we", "him", "been", "has", "when", "who", "will", "more", "if", "no", "out", "so", "said", "what", "up", "its", "about", "into", "than", "them", "can", "only", "other", "new", "some", "could", "time", "these", "two", "may", "then", "do", "first", "any", "my", "now", "such", "like", "our", "over", "man", "me", "even", "most", "made", "after", "also", "did", "many", "before", "must", "af", "through", "back", "years", "much", "where", "your", "way", "well", "down", "should", "because", "each", "just", "those", "people", "too", "how", "little", "state", "good", "very", "make", "world", "still", "own", "see", "men", "work", "long", "here", "get", "between", "both", "life", "being", "under", "never", "day", "same", "another", "know", "while", "last", "us", "might", "great", "old", "year", "off", "come", "since", "against", "go", "came", "right", "used", "three", "take", "himself", "states", "few", "use", "house", "during", "without", "again", "place", "american", "around", "however", "home", "small", "found", "thought", "went", "say", "part", "once", "general", "high", "upon", "school", "every", "does", "got", "united", "left", "number", "course", "war", "until", "always", "away", "something", "fact", "water", "though", "public", "put", "less", "think", "almost", "hand", "enough", "took", "far", "head", "yet", "government", "system", "set", "better", "told", "nothing", "night", "end", "why", "called", "eyes", "find", "look", "going", "asked", "later", "point", "knew", "next", "program", "city", "business", "group", "give", "toward", "young", "let", "room", "days", "president", "side", "social", "given", "present", "several", "order", "national", "possible", "rather", "second", "face", "per", "among", "form", "often", "important", "things", "looked", "early", "white", "case", "john", "large", "four", "need", "big", "within", "become", "felt", "along", "children", "saw", "best", "church", "ever", "least", "power", "development", "thing", "seemed", "light", "family", "interest", "want", "mind", "members", "area", "country", "others", "although", "turned", "done", "open", "god", "service", "problem", "kind", "certain", "different", "thus", "began", "door", "sense", "help", "means", "whole", "matter", "perhaps", "itself", "york", "times", "human", "law", "line", "above", "name", "example", "action", "company", "hands", "local", "show", "whether", "history", "five", "gave", "either", "today", "act", "feet", "across", "quite", "taken", "past", "anything", "having", "seen", "death", "experience", "body", "really", "half", "week", "car", "field", "words", "word", "already", "themselves", "information", "tell", "shall", "together", "college", "money", "period", "keep", "held", "sure", "probably", "free", "seems", "behind", "real", "cannot", "political", "question", "air", "making", "office", "brought", "miss", "whose", "special", "problems", "major", "heard", "became", "moment", "study", "ago", "federal", "known", "available", "street", "result", "economic", "boy", "reason", "position", "south", "change", "board", "individual", "job", "society", "am", "areas", "west", "close", "turn", "love", "community", "true", "force", "full", "court", "seem", "cost", "wife", "age", "future", "wanted", "voice", "department", "center", "woman", "common", "control", "necessary", "policy", "front", "following", "sometimes", "girl", "six", "clear", "further", "land", "music", "feel", "mother", "able", "party", "provide", "university", "education", "level", "run", "students", "effect", "child", "stood", "town", "short", "military", "total", "morning", "outside", "rate", "figure", "art", "class", "century", "usually", "north", "washington", "leave", "therefore", "plan", "top", "evidence", "sound", "million", "black", "hard", "strong", "various", "tax", "believe", "type", "value", "says", "play", "surface", "mean", "soon", "modern", "near", "peace", "lines", "table", "book", "road", "red", "situation", "personal", "minutes", "process", "nor", "idea", "alone", "women", "english", "schools", "gone", "increase", "living", "america", "started", "longer", "cut", "finally", "private", "nature", "secretary", "third", "section", "months", "greater", "call", "fire", "needed", "expected", "view", "values", "kept", "ground", "everything", "pressure", "dark", "basis", "space", "father", "east", "spirit", "required", "union", "complete", "wrote", "except", "moved", "support", "return", "conditions", "recent", "particular", "attention", "late", "hope", "live", "else", "costs", "brown", "beyond", "stage", "taking", "material", "nations", "forces", "report", "inside", "dead", "read", "coming", "person", "hours", "heart", "instead", "low", "miles", "lost", "looking", "data", "added", "makes", "single", "followed", "amount", "pay", "feeling", "basic", "including", "simply", "move", "cold", "hundred", "research", "industry", "tried", "developed", "reached", "hold", "committee", "defense", "equipment", "island", "actually", "shown", "son", "central", "religious", "river", "getting", "beginning", "ten", "sort", "received", "rest", "terms", "doing", "trying", "indeed", "care", "friends", "medical", "picture", "especially", "administration", "subject", "fine", "difficult", "building", "higher", "simple", "wall", "walked", "meeting", "bring", "floor", "foreign", "passed", "similar", "paper", "range", "natural", "final", "property", "training", "police", "international", "cent", "county", "market", "growth", "england", "talk", "written", "start", "story", "suddenly", "hear", "issue", "hall", "answer", "needs", "congress", "working", "likely", "considered", "countries", "earth", "sat", "entire", "meet", "purpose", "labor", "happened", "results", "difference", "cases", "stand", "hair", "william", "production", "stock", "involved", "fall", "food", "whom", "earlier", "increased", "particularly", "thinking", "club", "letter", "knowledge", "below", "effort", "hour", "using", "paid", "sent", "christian", "yes", "points", "boys", "industrial", "bill", "deal", "ready", "ideas", "certainly", "square", "blue", "trade", "moral", "bad", "due", "methods", "girls", "method", "addition", "neither", "showed", "throughout", "directly", "nearly", "statement", "decided", "reading", "weeks", "according", "questions", "anyone", "try", "color", "kennedy", "programs", "nation", "services", "lay", "french", "remember", "physical", "size", "member", "understand", "western", "strength", "record", "comes", "southern", "normal", "population", "appeared", "merely", "concerned", "district", "volume", "temperature", "direction", "trial", "trouble", "summer", "aid", "ran", "friend", "sales", "literature", "list", "maybe", "continued", "evening", "association", "generally", "influence", "led", "provided", "met", "army", "science", "changes", "husband", "step", "chance", "student", "opened", "former", "average", "month", "series", "hot", "works", "cause", "lead", "direct", "systems", "planning", "stopped", "myself", "theory", "piece", "wrong", "effective", "george", "soviet", "freedom", "worked", "ask", "movement", "organization", "clearly", "ways", "treatment", "fear", "beautiful", "meaning", "consider", "bed", "spring", "somewhat", "press", "lot", "forms", "efforts", "note", "truth", "placed", "hotel", "carried", "numbers", "respect", "plant", "herself", "apparently", "groups", "degree", "wide", "easy", "manner", "reaction", "farm", "lower", "recently", "approach", "immediately", "game", "larger", "running", "daily", "charge", "couple", "eye", "oh", "performance", "feed", "de", "persons", "c", "understanding", "arms", "blood", "opportunity", "march", "progress", "additional", "described", "technical", "stop", "fiscal", "radio", "religion", "test", "reported", "main", "based", "steps", "image", "chief", "served", "window", "decision", "determined", "responsibility", "character", "middle", "british", "aj", "europe", "gun", "writing", "horse", "appear", "learned", "account", "serious", "ones", "types", "activity", "green", "length", "letters", "nuclear", "specific", "slowly", "forward", "returned", "lived", "activities", "corner", "obtained", "audience", "justice", "doubt", "latter", "hit", "plane", "gives", "moving", "obviously", "straight", "design", "quality", "saying", "function", "choice", "staff", "include", "pattern", "figures", "born", "parts", "stay", "seven", "poor", "plans", "operation", "shot", "whatever", "cars", "sun", "faith", "pool", "extent", "speak", "heavy", "completely", "lack", "waiting", "standard", "hospital", "ball", "wish", "corps", "visit", "language", "ahead", "deep", "income", "principle", "democratic", "firm", "importance", "growing", "analysis", "designed", "effects", "price", "none", "distance", "indicated", "established", "products", "expect", "division", "continue", "leaders", "existence", "determine", "serve", "stress", "attitude", "pretty", "elements", "easily", "cities", "negro", "remained", "factors", "hardly", "applied", "limited", "closed", "agreement", "scene", "write", "afternoon", "b", "suggested", "health", "professional", "attack", "reach", "drive", "season", "interested", "station", "rhode", "married", "despite", "covered", "role", "becomes", "eight", "current", "played", "spent", "reasons", "council", "unit", "built", "commission", "date", "mouth", "exactly", "original", "studies", "race", "machine"]; return Model; }()); - // It's a 'dummy' model, so there's no need for extra methods and such within the Model's class definition. + // As a wordlist model, we can rely on our "model library" implementation, models.WordListModel. LMLayerWorker.loadModel(new models.WordListModel(Model.capabilities, Model.wordlist)); })(); \ No newline at end of file diff --git a/common/predictive-text/worker/index.ts b/common/predictive-text/worker/index.ts index bcd317d3f1..884e706885 100644 --- a/common/predictive-text/worker/index.ts +++ b/common/predictive-text/worker/index.ts @@ -215,7 +215,7 @@ class LMLayerWorker { this.unloadModel(); break; default: - throw new Error(`invalid message; expected one of {'predict'} but got ${payload.message}`); + throw new Error(`invalid message; expected one of {'predict', 'unload'} but got ${payload.message}`); } } }; From 473ed67b29e8d2817de456d09e1452bcb834425e Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 6 Mar 2019 09:35:46 +0700 Subject: [PATCH 12/22] Adds a 'config' state to the LMLayer worker. --- common/predictive-text/index.ts | 18 ++++++- .../unit_tests/headless/top-level-lmlayer.js | 28 ++++++++-- .../headless/worker-initialization.js | 44 ++++++++++++++++ .../unit_tests/headless/worker-predict.js | 2 + common/predictive-text/unit_tests/helpers.js | 20 ++++++++ .../unit_tests/in_browser/base.conf.js | 2 +- .../in_browser/cases/top-level-lmlayer.js | 4 +- .../cases/worker-dummy-integration.js | 2 +- .../cases/worker-wordlist-integration.js | 2 +- .../unit_tests/in_browser/cases/worker.js | 5 ++ .../unit_tests/in_browser/helpers.js | 8 +++ common/predictive-text/worker/index.ts | 51 ++++++++++++++----- .../worker/worker-interfaces.ts | 19 +++++-- web/source/text/prediction/modelManager.ts | 7 ++- 14 files changed, 185 insertions(+), 27 deletions(-) create mode 100644 common/predictive-text/unit_tests/in_browser/helpers.js diff --git a/common/predictive-text/index.ts b/common/predictive-text/index.ts index 65535bb476..d2352f1572 100644 --- a/common/predictive-text/index.ts +++ b/common/predictive-text/index.ts @@ -52,6 +52,7 @@ namespace com.keyman.text.prediction { private _declareLMLayerReady: (conf: Configuration) => void; private _promises: PromiseStore; private _nextToken: number; + private capabilities: Capabilities; /** * Construct the top-level LMLayer interface. This also starts the underlying Worker. @@ -60,13 +61,28 @@ namespace com.keyman.text.prediction { * @param uri URI of the underlying LMLayer worker code. This will usually be a blob: * or file: URI. If uri is not provided, this will start the default Worker. */ - constructor(worker?: Worker) { + constructor(capabilities: Capabilities, worker?: Worker) { // Either use the given worker, or instantiate the default worker. this._worker = worker || new Worker(LMLayer.asBlobURI(LMLayerWorkerCode)); this._worker.onmessage = this.onMessage.bind(this) this._declareLMLayerReady = null; this._promises = new PromiseStore; this._nextToken = Number.MIN_SAFE_INTEGER; + + this.sendConfig(capabilities); + } + + /** + * Initializes the LMLayer worker with the host platform's capability set. + * + * @param capabilities The host platform's capability spec - a model cannot assume access to more context + * than specified by this parameter. + */ + private sendConfig(capabilities: Capabilities) { + this._worker.postMessage({ + message: 'config', + capabilities: capabilities + }); } /** diff --git a/common/predictive-text/unit_tests/headless/top-level-lmlayer.js b/common/predictive-text/unit_tests/headless/top-level-lmlayer.js index 15785bd525..286e3de818 100644 --- a/common/predictive-text/unit_tests/headless/top-level-lmlayer.js +++ b/common/predictive-text/unit_tests/headless/top-level-lmlayer.js @@ -8,7 +8,20 @@ let LMLayer = require('../../build'); describe('LMLayer', function() { describe('[[constructor]]', function () { it('should accept a Worker to instantiate', function () { - new LMLayer(createFakeWorker()); + new LMLayer(capabilities(), createFakeWorker()); + }); + + it('should send the `config` message to the LMLayer', async function () { + let fakeWorker = createFakeWorker(fakePostMessage); + let lmLayer = new LMLayer(capabilities(), fakeWorker); + + assert.propertyVal(fakeWorker.postMessage, 'callCount', 1); + // In the "Worker", assert the message looks right and + // ASYNCHRONOUSLY reply with ready message. + function fakePostMessage(data) { + assert.propertyVal(data, 'message', 'config'); + assert.isObject(data.capabilities); + } }); }); @@ -16,7 +29,7 @@ describe('LMLayer', function() { it('should accept capabilities and model description', function () { let fakeWorker = createFakeWorker(); - let lmLayer = new LMLayer(fakeWorker); + let lmLayer = new LMLayer(capabilities(), fakeWorker); lmLayer.activateModel("./unit_tests/in_browser/resources/models/simple-dummy.js"); assert.isFunction(fakeWorker.onmessage, 'LMLayer failed to set a callback!'); @@ -24,13 +37,18 @@ describe('LMLayer', function() { it('should send the `initialize` message to the LMLayer', async function () { let fakeWorker = createFakeWorker(fakePostMessage); - let lmLayer = new LMLayer(fakeWorker); + let lmLayer = new LMLayer(capabilities(), fakeWorker); let configuration = await lmLayer.activateModel("./unit_tests/in_browser/resources/models/simple-dummy.js"); - assert.propertyVal(fakeWorker.postMessage, 'callCount', 1); + assert.propertyVal(fakeWorker.postMessage, 'callCount', 2); // In the "Worker", assert the message looks right and // ASYNCHRONOUSLY reply with ready message. function fakePostMessage(data) { + // Expected first call: config. Ignore it. + if(data.message == 'config') { + return; + } + assert.propertyVal(data, 'message', 'initialize'); assert.isString(data.model); @@ -58,7 +76,7 @@ describe('LMLayer', function() { })); }); - let lmLayer = new LMLayer(fakeWorker); + let lmLayer = new LMLayer(capabilities, fakeWorker); let actualConfiguration = await lmLayer.activateModel( { maxLeftContextCodeUnits: 32, diff --git a/common/predictive-text/unit_tests/headless/worker-initialization.js b/common/predictive-text/unit_tests/headless/worker-initialization.js index b74425d234..87f76ac0af 100644 --- a/common/predictive-text/unit_tests/headless/worker-initialization.js +++ b/common/predictive-text/unit_tests/headless/worker-initialization.js @@ -19,6 +19,9 @@ describe('LMLayerWorker', function() { var worker = LMLayerWorker.install(context); + // First the worker must receive config data... + configWorker(worker); + // Sending it the initialize it should notify us that it's initialized! worker.onMessage(createMessageEventWithData({ message: 'initialize', @@ -60,6 +63,9 @@ describe('LMLayerWorker', function() { // It should have installed a callback. assert.isFunction(fakeWorkerGlobal.onmessage); + // First the worker must receive config data... + configWorker(worker); + // Send a message; we should get something back. worker.onMessage(createMessageEventWithData({ message: 'initialize', @@ -71,6 +77,38 @@ describe('LMLayerWorker', function() { }); }); + describe('Message: config', function () { + it('should disallow any other message', function () { + var context = { + postMessage: sinon.fake() + }; + context.importScripts = importScriptsWith(context); + + var worker = LMLayerWorker.install(context); + + // It should not respond to 'predict' + assert.throws(function () { + worker.onMessage(createMessageEventWithData({ + message: 'predict', + })); + }, /invalid message/i); + }); + + it('accepts a capability set and transitions to the "initialize" state', function () { + var context = { + postMessage: sinon.fake() + }; + context.importScripts = importScriptsWith(context); + + var worker = LMLayerWorker.install(context); + + // Trigger the config message + configWorker(worker); + + assert.equal(worker.state.name, 'uninitialized'); + }); + }); + describe('Message: initialize', function () { it('should disallow any other message', function () { var context = { @@ -80,6 +118,8 @@ describe('LMLayerWorker', function() { var worker = LMLayerWorker.install(context); + configWorker(worker); + // It should not respond to 'predict' assert.throws(function () { worker.onMessage(createMessageEventWithData({ @@ -96,6 +136,8 @@ describe('LMLayerWorker', function() { context.importScripts = importScriptsWith(context); var worker = LMLayerWorker.install(context); + configWorker(worker); + worker.onMessage(createMessageEventWithData({ message: 'initialize', model: "./unit_tests/in_browser/resources/models/simple-dummy.js" @@ -114,6 +156,8 @@ describe('LMLayerWorker', function() { context.importScripts = importScriptsWith(context); var worker = LMLayerWorker.install(context); + configWorker(worker); + // simple-dummy.js is set with the following. var maxCodeUnits = 64; worker.onMessage(createMessageEventWithData({ diff --git a/common/predictive-text/unit_tests/headless/worker-predict.js b/common/predictive-text/unit_tests/headless/worker-predict.js index fc15bb3564..80dc7ab145 100644 --- a/common/predictive-text/unit_tests/headless/worker-predict.js +++ b/common/predictive-text/unit_tests/headless/worker-predict.js @@ -23,6 +23,8 @@ describe('LMLayerWorker', function () { context.importScripts = importScriptsWith(context); var worker = LMLayerWorker.install(context); + configWorker(worker); + worker.onMessage(createMessageEventWithData({ message: 'initialize', model: "./unit_tests/in_browser/resources/models/simple-dummy.js" diff --git a/common/predictive-text/unit_tests/helpers.js b/common/predictive-text/unit_tests/helpers.js index 8fe7522f74..0ac0425fe5 100644 --- a/common/predictive-text/unit_tests/helpers.js +++ b/common/predictive-text/unit_tests/helpers.js @@ -20,6 +20,26 @@ _.createMessageEventWithData = function createMessageEventWithData(data) { return { data }; } +/** + * Creates a simple, default capabilities object for standard-case LMLayer init. + */ +_.capabilities = function capabilities() { + return { + maxLeftContextCodeUnits: 64 + } +} + +/** + * Mimics a message from the outer LMLayer shell with a simple, default config object. + * Used for Worker tests. + */ +_.configWorker = function configWorker(worker) { + worker.onMessage(createMessageEventWithData({ + message: 'config', + capabilities: _.capabilities() + })); +} + /** * A valid model that suggests exactly what you want it to suggest. * diff --git a/common/predictive-text/unit_tests/in_browser/base.conf.js b/common/predictive-text/unit_tests/in_browser/base.conf.js index ac61cf3053..2a3adccd4b 100644 --- a/common/predictive-text/unit_tests/in_browser/base.conf.js +++ b/common/predictive-text/unit_tests/in_browser/base.conf.js @@ -33,7 +33,7 @@ module.exports = { files: [ // Include the generated worker code. Make sure it's linked before any of the test cases. '../../build/index.js', - + 'helpers.js', // Provides utility helpers and objects for tests. 'cases/**/*.js', // Where the tests actually reside. // We don't have anything in these locations... yet. But they'll be useful for test resources. diff --git a/common/predictive-text/unit_tests/in_browser/cases/top-level-lmlayer.js b/common/predictive-text/unit_tests/in_browser/cases/top-level-lmlayer.js index a97e84b20d..48ee845cf9 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/top-level-lmlayer.js +++ b/common/predictive-text/unit_tests/in_browser/cases/top-level-lmlayer.js @@ -3,8 +3,8 @@ var LMLayer = com.keyman.text.prediction.LMLayer; describe('LMLayer', function () { describe('[[constructor]]', function () { - it('should construct with zero arguments', function () { - let lmLayer = new LMLayer(); + it('should construct with a single argument', function () { + let lmLayer = new LMLayer(helpers.defaultCapabilities); assert.instanceOf(lmLayer, LMLayer); lmLayer.shutdown(); }); diff --git a/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js b/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js index 39e088ede1..806a062098 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js +++ b/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js @@ -12,7 +12,7 @@ var LMLayer = com.keyman.text.prediction.LMLayer; describe('LMLayer using dummy model', function () { describe('Prediction', function () { it('will predict future suggestions', function () { - var lmLayer = new LMLayer(); + var lmLayer = new LMLayer(helpers.defaultCapabilities); // We're testing many as asynchronous messages in a row. // this would be cleaner using async/await syntax, but diff --git a/common/predictive-text/unit_tests/in_browser/cases/worker-wordlist-integration.js b/common/predictive-text/unit_tests/in_browser/cases/worker-wordlist-integration.js index eb558083e4..ff60c79f2b 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/worker-wordlist-integration.js +++ b/common/predictive-text/unit_tests/in_browser/cases/worker-wordlist-integration.js @@ -8,7 +8,7 @@ describe('LMLayer using the word list model', function () { describe('Prediction', function () { var EXPECTED_SUGGESTIONS = 3; it('will predict an empty buffer', function () { - var lmLayer = new LMLayer(); + var lmLayer = new LMLayer(helpers.defaultCapabilities); // We're testing many as asynchronous messages in a row. // this would be cleaner using async/await syntax, but diff --git a/common/predictive-text/unit_tests/in_browser/cases/worker.js b/common/predictive-text/unit_tests/in_browser/cases/worker.js index 693967323b..e8b5c719db 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/worker.js +++ b/common/predictive-text/unit_tests/in_browser/cases/worker.js @@ -19,6 +19,11 @@ describe('LMLayerWorker', function () { done(); worker.terminate(); }; + // While the config message doesn't trigger a reply message, we have to send it a configuration message first. + worker.postMessage({ + message: 'config', + capabilities: helpers.defaultCapabilities + }) worker.postMessage({ message: 'initialize', // Since the worker's based in a blob, it's not on the 'same domain'. We need to absolute-path the model file. diff --git a/common/predictive-text/unit_tests/in_browser/helpers.js b/common/predictive-text/unit_tests/in_browser/helpers.js new file mode 100644 index 0000000000..a2d8ab127e --- /dev/null +++ b/common/predictive-text/unit_tests/in_browser/helpers.js @@ -0,0 +1,8 @@ +var helpers; + +// Establishes the equivalent of a TS namespace. +(function(helpers){ + helpers.defaultCapabilities = { + maxLeftContextCodeUnits: 64 + }; +})(helpers || (helpers = {})); \ No newline at end of file diff --git a/common/predictive-text/worker/index.ts b/common/predictive-text/worker/index.ts index 884e706885..05f15affc5 100644 --- a/common/predictive-text/worker/index.ts +++ b/common/predictive-text/worker/index.ts @@ -36,21 +36,22 @@ /** * Encapsulates all the state required for the LMLayer's worker thread. * - * Implements the state pattern. There are two states: + * Implements the state pattern. There are three states: * - * - `uninitialized` (initial state) - * - `ready` (accepting state) + * - `unconfigured` (initial state before configuration) + * - `uninitialized` (state without model loaded) + * - `ready` (state with model loaded, accepts prediction requests) * * Transitions are initiated by valid messages. Invalid * messages are errors, and do not lead to transitions. * - * +-----------------+ +---------+ - * | | initialize | | - * +------> uninitialized +----------->+ ready +---+ - * | | | | | - * +-----------------+ +----^----+ | predict - * | | - * +--------+ + * +-----------------+ initialize +---------+ + * config | |----------->| | + * +-------> uninitialized + + ready +---+ + * | |<-----------| | | + * +-----------------+ unload +----^----+ | predict + * | | + * +--------+ * * The model and the configuration are ONLY relevant in the `ready` state; * as such, they are NOT direct properties of the LMLayerWorker. @@ -78,6 +79,8 @@ class LMLayerWorker { */ private _importScripts: ImportScripts; + private _platformCapabilities: Capabilities; + private _hostURL: string; constructor(options = { @@ -86,7 +89,7 @@ class LMLayerWorker { }) { this._postMessage = options.postMessage || postMessage; this._importScripts = options.importScripts || importScripts; - this.setupInitialState(); + this.setupConfigState(); } /** @@ -140,6 +143,8 @@ class LMLayerWorker { * @param capabilities Capabilities on offer from the keyboard. */ public loadModel(model: WorkerInternalModel) { + // TODO: pass _platformConfig to model so that it can self-configure to the platform, + // returning a Configuration. let capabilities = model.getCapabilities(); let configuration: Configuration = { leftContextCodeUnits: 0, @@ -172,7 +177,29 @@ class LMLayerWorker { } /** - * Sets the initial state, i.e., `uninitialized`. + * Sets the initial state, i.e., `unconfigured`. + * This state only handles `config` messages, and will + * transition to the `uninitialized` state once it receives + * the config data from the host platform. + */ + private setupConfigState() { + this.state = { + name: 'unconfigured', + handleMessage: (payload) => { + // ... that message must have been 'config'! + if (payload.message !== 'config') { + throw new Error(`invalid message; expected 'config' but got ${payload.message}`); + } + + this._platformCapabilities = payload.capabilities; + + this.setupInitialState(); + } + } + } + + /** + * Sets the model-loading state, i.e., `uninitialized`. * This state only handles `initialized` messages, and will * transition to the `ready` state once it receives a model * description and capabilities. diff --git a/common/predictive-text/worker/worker-interfaces.ts b/common/predictive-text/worker/worker-interfaces.ts index 500fef750a..91b9ed919a 100644 --- a/common/predictive-text/worker/worker-interfaces.ts +++ b/common/predictive-text/worker/worker-interfaces.ts @@ -38,8 +38,21 @@ type ImportScripts = typeof DedicatedWorkerGlobalScope.prototype.importScripts; /** * The valid incoming message kinds. */ -type IncomingMessageKind = 'initialize' | 'predict' | 'unload'; -type IncomingMessage = InitializeMessage | PredictMessage | UnloadMessage; +type IncomingMessageKind = 'config' | 'initialize' | 'predict' | 'unload'; +type IncomingMessage = ConfigMessage | InitializeMessage | PredictMessage | UnloadMessage; + +/** + * The structure of a config message. It should include the platform's supported + * capabilities. + */ +interface ConfigMessage { + message: 'config'; + + /** + * The platform's supported capabilities. + */ + capabilities: Capabilities; +} /** * The structure of an initialization message. It should include the model (either in @@ -96,7 +109,7 @@ interface LMLayerWorkerState { * Informative property. Name of the state. Currently, the LMLayerWorker can only * be the following states: */ - name: 'uninitialized' | 'ready'; + name: 'unconfigured' | 'uninitialized' | 'ready'; handleMessage(payload: IncomingMessage): void; } diff --git a/web/source/text/prediction/modelManager.ts b/web/source/text/prediction/modelManager.ts index 34abcdd8c7..9739e8efda 100644 --- a/web/source/text/prediction/modelManager.ts +++ b/web/source/text/prediction/modelManager.ts @@ -36,7 +36,12 @@ namespace com.keyman.text.prediction { init() { let keyman = com.keyman.singleton; - this.lmEngine = new LMLayer(); + // Establishes KMW's platform 'capabilities', which limit the range of context a LMLayer + // model may expect. + let capabilities: Capabilities = { + maxLeftContextCodeUnits: 64 + } + this.lmEngine = new LMLayer(capabilities); // Registers this module for keyboard (and thus, language) change events. keyman['addEventListener']('keyboardchange', this.onKeyboardChange.bind(this)); From b6cc287cb132c0ce716819aefa0bad526297d9a8 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 6 Mar 2019 09:58:29 +0700 Subject: [PATCH 13/22] Renames the 'uninitialized' state to 'modelless', fixes (most?) docs. --- .../docs/worker-communication-protocol.md | 109 +++++++++++------- common/predictive-text/index.d.ts | 2 +- common/predictive-text/index.ts | 8 +- .../unit_tests/headless/top-level-lmlayer.js | 4 +- .../headless/worker-initialization.js | 16 +-- .../unit_tests/headless/worker-predict.js | 2 +- .../cases/worker-dummy-integration.js | 2 +- .../unit_tests/in_browser/cases/worker.js | 2 +- common/predictive-text/worker/index.ts | 38 +++--- .../worker/worker-interfaces.ts | 10 +- 10 files changed, 107 insertions(+), 86 deletions(-) diff --git a/common/predictive-text/docs/worker-communication-protocol.md b/common/predictive-text/docs/worker-communication-protocol.md index 3e2fd585fd..f5422830f6 100644 --- a/common/predictive-text/docs/worker-communication-protocol.md +++ b/common/predictive-text/docs/worker-communication-protocol.md @@ -117,69 +117,90 @@ Currently there are four message types: Message | Direction | Parameters | Expected reply | Uses token --------------|--------------------|---------------------|---------------------|--------------- -`initialize` | keyboard → LMLayer | capabilities, model | Yes — `ready` | No +`config` | LMLayer -> worker | capabilities | No | No +`load` | keyboard → LMLayer | capabilities, model | Yes — `ready` | No +`unload` | keyboard → LMLayer | none | No | No `ready` | LMLayer → keyboard | configuration | No | No `predict` | keyboard → LMLayer | transform, context | Yes — `suggestions` | Yes `suggestions` | LMLayer → keyboard | suggestions | No | Yes - -### Message: `initialize` +### Message: `config` Must be sent from the keyboard to the LMLayer so that the LMLayer -initializes a model. It will send `initialization` which is a plain +may properly configure loaded models. It will send `config`, a plain +JavaScript object specifying platform restrictions. + +The keyboard **MUST NOT** send any messages to the LMLayer prior to sending `config`. +After this, it is safe to assume the `config` was performed successfully and is ready to +`load` a model. + +The LMLayer needs to know the platform's abilities and restrictions (capabilities). + +```typescript +interface LoadMessage { + message: 'load'; + + /** + * The path to the model's compiled script file. + */ + capabilities: { + /** + * The maximum amount of UTF-16 code units that the keyboard will provide to + * the left of the cursor, as an integer. + */ + maxLeftContextCodeUnits: number, + + /** + * The maximum amount of code units that the keyboard will provide to the + * right of the cursor, as an integer. The value 0 or the absence of this + * rule implies that the right contexts are not supported. + */ + maxRightContextCodeUnits?: number, + + /** + * Whether the platform supports deleting to the right. The absence of this + * rule implies false. + */ + supportsDeleteRight?: false, + } +} +``` + +### Message: `load` + +Must be sent from the keyboard to the LMLayer so that the LMLayer +loads a model. It will send `load` which is a plain JavaScript object specify the path to the model, as well configurations and platform restrictions. -The keyboard **MUST NOT** send any messages to the LMLayer prior to -sending `initialize`. The keyboard **SHOULD NOT** send another message -to the keyboard until it receives `ready` message from the LMLayer +After a single `config` message, the keyboard **MUST NOT** send any messages +to the LMLayer prior to sending `load`. The keyboard **SHOULD NOT** send another +message to the keyboard until it receives `ready` message from the LMLayer before sending another message. The LMLayer needs to know the platform's abilities and restrictions (capabilities), as well as which concrete language model to instantiate. These properties are passed as `capabilities` and `model`, respectively. ```typescript -interface InitializeMessage { - message: 'initialize'; +interface LoadMessage { + message: 'load'; /** - * A ModelDescription that describes the language model and its parameters. - * The concrete documentation on is a valid ModelDescription - * can be found elsewhere. + * The path to the model's compiled script file. */ - model: { - /** - * What kind of model to instantiate. This is subject to availability, - * but common examples are 'wordlist', 'fst', and 'dummy'. - */ - type: string; - /** - * Each model type defines a set of configurable parameters. Please - * see the corresponding model's documentation for an extensive list. - */ - ...parameters: any; - }; + model: string +} +``` - capabilities: { - /** - * Whether the platform supports deleting to the right. - * The absence of this rule implies false. - */ - supportsDeleteRight?: false, +### Message: `unload` - /** - * The maximum amount of UTF-16 code units that the keyboard will - * provide to the left of the cursor, as an integer. - */ - maxLeftContextCodeUnits: number, +Must be sent from the keyboard to the LMLayer so that the LMLayer +resets itself in preparation for loading a new model. It will send `unload` +which is a plain message to trigger release of old model resources. - /** - * The maximum amount of code units that the keyboard will provide to - * the right of the cursor, as an integer. The absence of this rule - * implies the platform is incapable of supplying right contexts. - * See also, [[supportsRightContexts]]. - */ - maxRightContextCodeUnits?: number, - } + +```typescript +interface UnloadMessage { + message: 'unload'; } ``` @@ -187,7 +208,7 @@ interface InitializeMessage { ### Message: `ready` Must be sent from the LMLayer to the keyboard when the LMLayer's model -as a response to `initialize`. It will send `configuration`, which is +as a response to `load`. It will send `configuration`, which is a plain JavaScript object requesting configuration from the keyboard. There are only two options defined so far: diff --git a/common/predictive-text/index.d.ts b/common/predictive-text/index.d.ts index a6f133e393..659f2729ca 100644 --- a/common/predictive-text/index.d.ts +++ b/common/predictive-text/index.d.ts @@ -13,7 +13,7 @@ declare namespace com.keyman.text.prediction { /** * Construct the top-level LMLayer interface. This also starts the underlying Worker. - * Make sure to call .initialize() when using the default Worker. + * Make sure to call .load() when using the default Worker. * * @param uri URI of the underlying LMLayer worker code. This will usually be a blob: * or file: URI. If uri is not provided, this will start the default Worker. diff --git a/common/predictive-text/index.ts b/common/predictive-text/index.ts index d2352f1572..94115567b6 100644 --- a/common/predictive-text/index.ts +++ b/common/predictive-text/index.ts @@ -34,10 +34,11 @@ * Since the Worker runs in a different thread, the public methods of this class are * asynchronous. Methods of note include: * - * - #activateModel() -- initialize the LMLayer by loading a specified model file + * - #activateModel() -- loads a specified model file * - #predict() -- ask the LMLayer to offer suggestions (predictions or corrections) for * the input event - * - #deactivateModel() -- de-initializes the LMLayer, preparing it for re-initialization + * - #deactivateModel() -- unloads the LMLayer's currently loaded model, preparing it to + * receive (load) a new model * * The top-level LMLayer will automatically starts up its own Web Worker. */ @@ -56,7 +57,6 @@ namespace com.keyman.text.prediction { /** * Construct the top-level LMLayer interface. This also starts the underlying Worker. - * Make sure to call .initialize() when using the default Worker. * * @param uri URI of the underlying LMLayer worker code. This will usually be a blob: * or file: URI. If uri is not provided, this will start the default Worker. @@ -91,7 +91,7 @@ namespace com.keyman.text.prediction { activateModel(modelFilePath: string): Promise { return new Promise((resolve, _reject) => { this._worker.postMessage({ - message: 'initialize', + message: 'load', model: modelFilePath }); diff --git a/common/predictive-text/unit_tests/headless/top-level-lmlayer.js b/common/predictive-text/unit_tests/headless/top-level-lmlayer.js index 286e3de818..7ad903a070 100644 --- a/common/predictive-text/unit_tests/headless/top-level-lmlayer.js +++ b/common/predictive-text/unit_tests/headless/top-level-lmlayer.js @@ -35,7 +35,7 @@ describe('LMLayer', function() { assert.isFunction(fakeWorker.onmessage, 'LMLayer failed to set a callback!'); }); - it('should send the `initialize` message to the LMLayer', async function () { + it('should send the `load` message to the LMLayer', async function () { let fakeWorker = createFakeWorker(fakePostMessage); let lmLayer = new LMLayer(capabilities(), fakeWorker); let configuration = await lmLayer.activateModel("./unit_tests/in_browser/resources/models/simple-dummy.js"); @@ -49,7 +49,7 @@ describe('LMLayer', function() { return; } - assert.propertyVal(data, 'message', 'initialize'); + assert.propertyVal(data, 'message', 'load'); assert.isString(data.model); callAsynchronously(() => fakeWorker.onmessage({ diff --git a/common/predictive-text/unit_tests/headless/worker-initialization.js b/common/predictive-text/unit_tests/headless/worker-initialization.js index 87f76ac0af..1a17a0676f 100644 --- a/common/predictive-text/unit_tests/headless/worker-initialization.js +++ b/common/predictive-text/unit_tests/headless/worker-initialization.js @@ -22,9 +22,9 @@ describe('LMLayerWorker', function() { // First the worker must receive config data... configWorker(worker); - // Sending it the initialize it should notify us that it's initialized! + // Sending it the `load` message should notify us that it's loaded! worker.onMessage(createMessageEventWithData({ - message: 'initialize', + message: 'load', model: "./unit_tests/in_browser/resources/models/simple-dummy.js" })); assert(fakePostMessage.calledOnce); @@ -68,7 +68,7 @@ describe('LMLayerWorker', function() { // Send a message; we should get something back. worker.onMessage(createMessageEventWithData({ - message: 'initialize', + message: 'load', model: "./unit_tests/in_browser/resources/models/simple-dummy.js" })); @@ -94,7 +94,7 @@ describe('LMLayerWorker', function() { }, /invalid message/i); }); - it('accepts a capability set and transitions to the "initialize" state', function () { + it('accepts a capability set and transitions to the "modelless" state', function () { var context = { postMessage: sinon.fake() }; @@ -105,11 +105,11 @@ describe('LMLayerWorker', function() { // Trigger the config message configWorker(worker); - assert.equal(worker.state.name, 'uninitialized'); + assert.equal(worker.state.name, 'modelless'); }); }); - describe('Message: initialize', function () { + describe('Message: load', function () { it('should disallow any other message', function () { var context = { postMessage: sinon.fake() @@ -139,7 +139,7 @@ describe('LMLayerWorker', function() { configWorker(worker); worker.onMessage(createMessageEventWithData({ - message: 'initialize', + message: 'load', model: "./unit_tests/in_browser/resources/models/simple-dummy.js" })); @@ -161,7 +161,7 @@ describe('LMLayerWorker', function() { // simple-dummy.js is set with the following. var maxCodeUnits = 64; worker.onMessage(createMessageEventWithData({ - message: 'initialize', + message: 'load', model: "./unit_tests/in_browser/resources/models/simple-dummy.js" })); diff --git a/common/predictive-text/unit_tests/headless/worker-predict.js b/common/predictive-text/unit_tests/headless/worker-predict.js index 80dc7ab145..8f935c7804 100644 --- a/common/predictive-text/unit_tests/headless/worker-predict.js +++ b/common/predictive-text/unit_tests/headless/worker-predict.js @@ -26,7 +26,7 @@ describe('LMLayerWorker', function () { configWorker(worker); worker.onMessage(createMessageEventWithData({ - message: 'initialize', + message: 'load', model: "./unit_tests/in_browser/resources/models/simple-dummy.js" })); sinon.assert.calledWithMatch(fakePostMessage.lastCall, { diff --git a/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js b/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js index 806a062098..c042b308a7 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js +++ b/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js @@ -7,7 +7,7 @@ var LMLayer = com.keyman.text.prediction.LMLayer; * **injectable** suggestions: that is, you, as the tester, have * to provide the predictions. The dummy model does not create any * suggestions on its own. The dummy model can take in a series - * of suggestions when initialized and return them sequentially. + * of suggestions when loaded and return them sequentially. */ describe('LMLayer using dummy model', function () { describe('Prediction', function () { diff --git a/common/predictive-text/unit_tests/in_browser/cases/worker.js b/common/predictive-text/unit_tests/in_browser/cases/worker.js index e8b5c719db..13bd705ef0 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/worker.js +++ b/common/predictive-text/unit_tests/in_browser/cases/worker.js @@ -25,7 +25,7 @@ describe('LMLayerWorker', function () { capabilities: helpers.defaultCapabilities }) worker.postMessage({ - message: 'initialize', + message: 'load', // Since the worker's based in a blob, it's not on the 'same domain'. We need to absolute-path the model file. model: document.location.protocol + '//' + document.location.host + "/resources/models/simple-dummy.js" }); diff --git a/common/predictive-text/worker/index.ts b/common/predictive-text/worker/index.ts index 05f15affc5..d2889eb16d 100644 --- a/common/predictive-text/worker/index.ts +++ b/common/predictive-text/worker/index.ts @@ -39,19 +39,19 @@ * Implements the state pattern. There are three states: * * - `unconfigured` (initial state before configuration) - * - `uninitialized` (state without model loaded) + * - `modelless` (state without model loaded) * - `ready` (state with model loaded, accepts prediction requests) * * Transitions are initiated by valid messages. Invalid * messages are errors, and do not lead to transitions. * - * +-----------------+ initialize +---------+ - * config | |----------->| | - * +-------> uninitialized + + ready +---+ - * | |<-----------| | | - * +-----------------+ unload +----^----+ | predict - * | | - * +--------+ + * +-------------+ load +---------+ + * config | |----------->| | + * +-------> modelless + + ready +---+ + * | |<-----------| | | + * +-------------+ unload +----^----+ | predict + * | | + * +--------+ * * The model and the configuration are ONLY relevant in the `ready` state; * as such, they are NOT direct properties of the LMLayerWorker. @@ -173,13 +173,13 @@ class LMLayerWorker { // Right now, this seems sufficient to clear out the old model. // The only existing reference to a loaded model is held by // transitionToReadyState's `handleMessage` closure. (The `model` var) - this.setupInitialState(); + this.transitionToLoadingState(); } /** * Sets the initial state, i.e., `unconfigured`. * This state only handles `config` messages, and will - * transition to the `uninitialized` state once it receives + * transition to the `modelless` state once it receives * the config data from the host platform. */ private setupConfigState() { @@ -193,24 +193,24 @@ class LMLayerWorker { this._platformCapabilities = payload.capabilities; - this.setupInitialState(); + this.transitionToLoadingState(); } } } /** - * Sets the model-loading state, i.e., `uninitialized`. - * This state only handles `initialized` messages, and will + * Sets the model-loading state, i.e., `modelless`. + * This state only handles `load` messages, and will * transition to the `ready` state once it receives a model * description and capabilities. */ - private setupInitialState() { + private transitionToLoadingState() { this.state = { - name: 'uninitialized', + name: 'modelless', handleMessage: (payload) => { - // ...that message must have been 'initialize'! - if (payload.message !== 'initialize') { - throw new Error(`invalid message; expected 'initialize' but got ${payload.message}`); + // ...that message must have been 'load'! + if (payload.message !== 'load') { + throw new Error(`invalid message; expected 'load' but got ${payload.message}`); } // TODO: validate configuration? @@ -224,7 +224,7 @@ class LMLayerWorker { * fully-instantiated model. The `ready` state only responds * to `predict` message, and is an accepting state. * - * @param model The initialized language model. + * @param model The loaded language model. */ private transitionToReadyState(model: WorkerInternalModel) { this.state = { diff --git a/common/predictive-text/worker/worker-interfaces.ts b/common/predictive-text/worker/worker-interfaces.ts index 91b9ed919a..800b52fdd4 100644 --- a/common/predictive-text/worker/worker-interfaces.ts +++ b/common/predictive-text/worker/worker-interfaces.ts @@ -38,8 +38,8 @@ type ImportScripts = typeof DedicatedWorkerGlobalScope.prototype.importScripts; /** * The valid incoming message kinds. */ -type IncomingMessageKind = 'config' | 'initialize' | 'predict' | 'unload'; -type IncomingMessage = ConfigMessage | InitializeMessage | PredictMessage | UnloadMessage; +type IncomingMessageKind = 'config' | 'load' | 'predict' | 'unload'; +type IncomingMessage = ConfigMessage | LoadMessage | PredictMessage | UnloadMessage; /** * The structure of a config message. It should include the platform's supported @@ -58,8 +58,8 @@ interface ConfigMessage { * The structure of an initialization message. It should include the model (either in * source code or parameter form), as well as the keyboard's capabilities. */ -interface InitializeMessage { - message: 'initialize'; +interface LoadMessage { + message: 'load'; /** * The model's compiled JS file. @@ -109,7 +109,7 @@ interface LMLayerWorkerState { * Informative property. Name of the state. Currently, the LMLayerWorker can only * be the following states: */ - name: 'unconfigured' | 'uninitialized' | 'ready'; + name: 'unconfigured' | 'modelless' | 'ready'; handleMessage(payload: IncomingMessage): void; } From 3b1e715f4b3571ae90d65662fd861d1170ac5757 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 6 Mar 2019 10:15:14 +0700 Subject: [PATCH 14/22] Adjusts predictive model spec for 'configuring' based on 'capabilities' --- .../headless/worker-predict-dummy.js | 26 +++---------------- .../headless/worker-predict-wordlist.js | 6 +---- .../resources/models/simple-dummy.js | 6 +---- .../resources/models/simple-wordlist.js | 6 +---- common/predictive-text/worker/index.ts | 10 +++---- .../worker/models/dummy-model.ts | 14 +++++----- .../worker/models/wordlist-model.ts | 12 +++++---- .../worker/worker-interfaces.ts | 4 +-- 8 files changed, 27 insertions(+), 57 deletions(-) diff --git a/common/predictive-text/unit_tests/headless/worker-predict-dummy.js b/common/predictive-text/unit_tests/headless/worker-predict-dummy.js index b0ffd9f6c5..af31d1ab62 100644 --- a/common/predictive-text/unit_tests/headless/worker-predict-dummy.js +++ b/common/predictive-text/unit_tests/headless/worker-predict-dummy.js @@ -7,26 +7,10 @@ var DummyModel = require('../../build/intermediate').models.DummyModel; describe('LMLayerWorker dummy model', function() { describe('instantiation', function () { - it('can be instantiated with capabilities', function () { - var model = new DummyModel(defaultCapabilities); + it('can be instantiated with no arguments', function () { + var model = new DummyModel(); assert.isObject(model); }); - - it('supports dependency-injected configuration', function () { - let configuration = { - leftContextCodeUnits: 64, - rightContextCodeUnits: 0 - }; - - var model = new DummyModel({ - maxLeftContextCodeUnits: 64, - }, - { - configuration: configuration, - }); - - assert.deepEqual(model.configuration, configuration); - }); }); describe('prediction', function () { @@ -65,7 +49,7 @@ describe('LMLayerWorker dummy model', function() { }, ]; - var model = new DummyModel(defaultCapabilities()); + var model = new DummyModel(); // Type a 't' var suggestions = model.predict({ @@ -89,9 +73,7 @@ describe('LMLayerWorker dummy model', function() { assert.isDefined(futureSuggestions[2]); assert.isDefined(futureSuggestions[3]); - var model = new DummyModel(defaultCapabilities, { - futureSuggestions: futureSuggestions - }); + var model = new DummyModel({futureSuggestions: futureSuggestions}); // The dummy model should give suggestions in order, // regardless of the provided transform and context. diff --git a/common/predictive-text/unit_tests/headless/worker-predict-wordlist.js b/common/predictive-text/unit_tests/headless/worker-predict-wordlist.js index b49f7eaa12..1e72c7ed8f 100644 --- a/common/predictive-text/unit_tests/headless/worker-predict-wordlist.js +++ b/common/predictive-text/unit_tests/headless/worker-predict-wordlist.js @@ -8,7 +8,7 @@ var WordListModel = require('../../build/intermediate').models.WordListModel; describe('LMLayerWorker word list model', function() { describe('instantiation', function () { it('can be instantiated with an empty word list', function () { - var model = new WordListModel(defaultCapabilities(), []); + var model = new WordListModel([]); assert.isObject(model); }); @@ -25,7 +25,6 @@ describe('LMLayerWorker word list model', function() { // «t| » [Send] // [ to ] [ the ] [ this ] var model = new WordListModel( - defaultCapabilities(), jsonFixture('wordlists/english-1000') ); @@ -51,7 +50,6 @@ describe('LMLayerWorker word list model', function() { // «th| » [Send] // [ this ] [ the ] [ there ] var model = new WordListModel( - defaultCapabilities(), jsonFixture('wordlists/english-1000') ); @@ -92,7 +90,6 @@ describe('LMLayerWorker word list model', function() { // «| » [Send] // [ I'm ] [ I ] [ Hey ] var model = new WordListModel( - defaultCapabilities(), jsonFixture('wordlists/english-1000') ); @@ -115,7 +112,6 @@ describe('LMLayerWorker word list model', function() { // «I g| » [Send] // [ gave ] [ got ] [ got the ] var model = new WordListModel( - defaultCapabilities(), jsonFixture('wordlists/english-1000') ); diff --git a/common/predictive-text/unit_tests/in_browser/resources/models/simple-dummy.js b/common/predictive-text/unit_tests/in_browser/resources/models/simple-dummy.js index 03cb8b88de..85d7b3057d 100644 --- a/common/predictive-text/unit_tests/in_browser/resources/models/simple-dummy.js +++ b/common/predictive-text/unit_tests/in_browser/resources/models/simple-dummy.js @@ -7,10 +7,6 @@ function Model() { // implements Model } - Model.capabilities = { - maxLeftContextCodeUnits: 64 - } - // A direct import/copy from i_got_distracted_by_hazel.json. Model.futureSuggestions = [ [ @@ -111,5 +107,5 @@ }()); // It's a 'dummy' model, so there's no need for extra methods and such within the Model's class definition. - LMLayerWorker.loadModel(new models.DummyModel(Model.capabilities, {futureSuggestions: Model.futureSuggestions})); + LMLayerWorker.loadModel(new models.DummyModel({futureSuggestions: Model.futureSuggestions})); })(); \ No newline at end of file diff --git a/common/predictive-text/unit_tests/in_browser/resources/models/simple-wordlist.js b/common/predictive-text/unit_tests/in_browser/resources/models/simple-wordlist.js index ec58dfd405..df485e7626 100644 --- a/common/predictive-text/unit_tests/in_browser/resources/models/simple-wordlist.js +++ b/common/predictive-text/unit_tests/in_browser/resources/models/simple-wordlist.js @@ -7,10 +7,6 @@ function Model() { // implements Model } - Model.capabilities = { - maxLeftContextCodeUnits: 64 - } - // A direct import/copy from english-1000.json. Model.wordlist = ["the", "of", "and", "to", "a", "in", "that", "is", "was", "he", "for", "it", "with", "as", "his", "on", "be", "at", "by", "i", "this", "had", "not", "are", "but", "from", "or", "have", "an", "they", "which", "one", "you", "were", "her", "all", "she", "there", "would", "their", "we", "him", "been", "has", "when", "who", "will", "more", "if", "no", "out", "so", "said", "what", "up", "its", "about", "into", "than", "them", "can", "only", "other", "new", "some", "could", "time", "these", "two", "may", "then", "do", "first", "any", "my", "now", "such", "like", "our", "over", "man", "me", "even", "most", "made", "after", "also", "did", "many", "before", "must", "af", "through", "back", "years", "much", "where", "your", "way", "well", "down", "should", "because", "each", "just", "those", "people", "too", "how", "little", "state", "good", "very", "make", "world", "still", "own", "see", "men", "work", "long", "here", "get", "between", "both", "life", "being", "under", "never", "day", "same", "another", "know", "while", "last", "us", "might", "great", "old", "year", "off", "come", "since", "against", "go", "came", "right", "used", "three", "take", "himself", "states", "few", "use", "house", "during", "without", "again", "place", "american", "around", "however", "home", "small", "found", "thought", "went", "say", "part", "once", "general", "high", "upon", "school", "every", "does", "got", "united", "left", "number", "course", "war", "until", "always", "away", "something", "fact", "water", "though", "public", "put", "less", "think", "almost", "hand", "enough", "took", "far", "head", "yet", "government", "system", "set", "better", "told", "nothing", "night", "end", "why", "called", "eyes", "find", "look", "going", "asked", "later", "point", "knew", "next", "program", "city", "business", "group", "give", "toward", "young", "let", "room", "days", "president", "side", "social", "given", "present", "several", "order", "national", "possible", "rather", "second", "face", "per", "among", "form", "often", "important", "things", "looked", "early", "white", "case", "john", "large", "four", "need", "big", "within", "become", "felt", "along", "children", "saw", "best", "church", "ever", "least", "power", "development", "thing", "seemed", "light", "family", "interest", "want", "mind", "members", "area", "country", "others", "although", "turned", "done", "open", "god", "service", "problem", "kind", "certain", "different", "thus", "began", "door", "sense", "help", "means", "whole", "matter", "perhaps", "itself", "york", "times", "human", "law", "line", "above", "name", "example", "action", "company", "hands", "local", "show", "whether", "history", "five", "gave", "either", "today", "act", "feet", "across", "quite", "taken", "past", "anything", "having", "seen", "death", "experience", "body", "really", "half", "week", "car", "field", "words", "word", "already", "themselves", "information", "tell", "shall", "together", "college", "money", "period", "keep", "held", "sure", "probably", "free", "seems", "behind", "real", "cannot", "political", "question", "air", "making", "office", "brought", "miss", "whose", "special", "problems", "major", "heard", "became", "moment", "study", "ago", "federal", "known", "available", "street", "result", "economic", "boy", "reason", "position", "south", "change", "board", "individual", "job", "society", "am", "areas", "west", "close", "turn", "love", "community", "true", "force", "full", "court", "seem", "cost", "wife", "age", "future", "wanted", "voice", "department", "center", "woman", "common", "control", "necessary", "policy", "front", "following", "sometimes", "girl", "six", "clear", "further", "land", "music", "feel", "mother", "able", "party", "provide", "university", "education", "level", "run", "students", "effect", "child", "stood", "town", "short", "military", "total", "morning", "outside", "rate", "figure", "art", "class", "century", "usually", "north", "washington", "leave", "therefore", "plan", "top", "evidence", "sound", "million", "black", "hard", "strong", "various", "tax", "believe", "type", "value", "says", "play", "surface", "mean", "soon", "modern", "near", "peace", "lines", "table", "book", "road", "red", "situation", "personal", "minutes", "process", "nor", "idea", "alone", "women", "english", "schools", "gone", "increase", "living", "america", "started", "longer", "cut", "finally", "private", "nature", "secretary", "third", "section", "months", "greater", "call", "fire", "needed", "expected", "view", "values", "kept", "ground", "everything", "pressure", "dark", "basis", "space", "father", "east", "spirit", "required", "union", "complete", "wrote", "except", "moved", "support", "return", "conditions", "recent", "particular", "attention", "late", "hope", "live", "else", "costs", "brown", "beyond", "stage", "taking", "material", "nations", "forces", "report", "inside", "dead", "read", "coming", "person", "hours", "heart", "instead", "low", "miles", "lost", "looking", "data", "added", "makes", "single", "followed", "amount", "pay", "feeling", "basic", "including", "simply", "move", "cold", "hundred", "research", "industry", "tried", "developed", "reached", "hold", "committee", "defense", "equipment", "island", "actually", "shown", "son", "central", "religious", "river", "getting", "beginning", "ten", "sort", "received", "rest", "terms", "doing", "trying", "indeed", "care", "friends", "medical", "picture", "especially", "administration", "subject", "fine", "difficult", "building", "higher", "simple", "wall", "walked", "meeting", "bring", "floor", "foreign", "passed", "similar", "paper", "range", "natural", "final", "property", "training", "police", "international", "cent", "county", "market", "growth", "england", "talk", "written", "start", "story", "suddenly", "hear", "issue", "hall", "answer", "needs", "congress", "working", "likely", "considered", "countries", "earth", "sat", "entire", "meet", "purpose", "labor", "happened", "results", "difference", "cases", "stand", "hair", "william", "production", "stock", "involved", "fall", "food", "whom", "earlier", "increased", "particularly", "thinking", "club", "letter", "knowledge", "below", "effort", "hour", "using", "paid", "sent", "christian", "yes", "points", "boys", "industrial", "bill", "deal", "ready", "ideas", "certainly", "square", "blue", "trade", "moral", "bad", "due", "methods", "girls", "method", "addition", "neither", "showed", "throughout", "directly", "nearly", "statement", "decided", "reading", "weeks", "according", "questions", "anyone", "try", "color", "kennedy", "programs", "nation", "services", "lay", "french", "remember", "physical", "size", "member", "understand", "western", "strength", "record", "comes", "southern", "normal", "population", "appeared", "merely", "concerned", "district", "volume", "temperature", "direction", "trial", "trouble", "summer", "aid", "ran", "friend", "sales", "literature", "list", "maybe", "continued", "evening", "association", "generally", "influence", "led", "provided", "met", "army", "science", "changes", "husband", "step", "chance", "student", "opened", "former", "average", "month", "series", "hot", "works", "cause", "lead", "direct", "systems", "planning", "stopped", "myself", "theory", "piece", "wrong", "effective", "george", "soviet", "freedom", "worked", "ask", "movement", "organization", "clearly", "ways", "treatment", "fear", "beautiful", "meaning", "consider", "bed", "spring", "somewhat", "press", "lot", "forms", "efforts", "note", "truth", "placed", "hotel", "carried", "numbers", "respect", "plant", "herself", "apparently", "groups", "degree", "wide", "easy", "manner", "reaction", "farm", "lower", "recently", "approach", "immediately", "game", "larger", "running", "daily", "charge", "couple", "eye", "oh", "performance", "feed", "de", "persons", "c", "understanding", "arms", "blood", "opportunity", "march", "progress", "additional", "described", "technical", "stop", "fiscal", "radio", "religion", "test", "reported", "main", "based", "steps", "image", "chief", "served", "window", "decision", "determined", "responsibility", "character", "middle", "british", "aj", "europe", "gun", "writing", "horse", "appear", "learned", "account", "serious", "ones", "types", "activity", "green", "length", "letters", "nuclear", "specific", "slowly", "forward", "returned", "lived", "activities", "corner", "obtained", "audience", "justice", "doubt", "latter", "hit", "plane", "gives", "moving", "obviously", "straight", "design", "quality", "saying", "function", "choice", "staff", "include", "pattern", "figures", "born", "parts", "stay", "seven", "poor", "plans", "operation", "shot", "whatever", "cars", "sun", "faith", "pool", "extent", "speak", "heavy", "completely", "lack", "waiting", "standard", "hospital", "ball", "wish", "corps", "visit", "language", "ahead", "deep", "income", "principle", "democratic", "firm", "importance", "growing", "analysis", "designed", "effects", "price", "none", "distance", "indicated", "established", "products", "expect", "division", "continue", "leaders", "existence", "determine", "serve", "stress", "attitude", "pretty", "elements", "easily", "cities", "negro", "remained", "factors", "hardly", "applied", "limited", "closed", "agreement", "scene", "write", "afternoon", "b", "suggested", "health", "professional", "attack", "reach", "drive", "season", "interested", "station", "rhode", "married", "despite", "covered", "role", "becomes", "eight", "current", "played", "spent", "reasons", "council", "unit", "built", "commission", "date", "mouth", "exactly", "original", "studies", "race", "machine"]; @@ -18,5 +14,5 @@ }()); // As a wordlist model, we can rely on our "model library" implementation, models.WordListModel. - LMLayerWorker.loadModel(new models.WordListModel(Model.capabilities, Model.wordlist)); + LMLayerWorker.loadModel(new models.WordListModel(Model.wordlist)); })(); \ No newline at end of file diff --git a/common/predictive-text/worker/index.ts b/common/predictive-text/worker/index.ts index d2889eb16d..5df2d6d5a9 100644 --- a/common/predictive-text/worker/index.ts +++ b/common/predictive-text/worker/index.ts @@ -145,18 +145,14 @@ class LMLayerWorker { public loadModel(model: WorkerInternalModel) { // TODO: pass _platformConfig to model so that it can self-configure to the platform, // returning a Configuration. - let capabilities = model.getCapabilities(); - let configuration: Configuration = { - leftContextCodeUnits: 0, - rightContextCodeUnits: 0 - }; + let configuration = model.configure(this._platformCapabilities); // Set reasonable defaults for the configuration. if (!configuration.leftContextCodeUnits) { - configuration.leftContextCodeUnits = capabilities.maxLeftContextCodeUnits; + configuration.leftContextCodeUnits = this._platformCapabilities.maxLeftContextCodeUnits; } if (!configuration.rightContextCodeUnits) { - configuration.rightContextCodeUnits = capabilities.maxRightContextCodeUnits || 0; + configuration.rightContextCodeUnits = this._platformCapabilities.maxRightContextCodeUnits || 0; } this.transitionToReadyState(model); diff --git a/common/predictive-text/worker/models/dummy-model.ts b/common/predictive-text/worker/models/dummy-model.ts index e28cd7de29..ca78f490d9 100644 --- a/common/predictive-text/worker/models/dummy-model.ts +++ b/common/predictive-text/worker/models/dummy-model.ts @@ -33,21 +33,23 @@ namespace models { */ export class DummyModel implements WorkerInternalModel { configuration: Configuration; - capabilities: Capabilities; private _futureSuggestions: Suggestion[][]; - constructor(capabilities: Capabilities, options?: any) { - this.capabilities = capabilities; + constructor(options?: any) { options = options || {}; - this.configuration = options.configuration || {}; // Create a shallow copy of the suggestions; // this class mutates the array. this._futureSuggestions = options.futureSuggestions ? options.futureSuggestions.slice() : []; } - getCapabilities(): Capabilities { - return this.capabilities; + configure(capabilities: Capabilities): Configuration { + this.configuration = { + leftContextCodeUnits: capabilities.maxLeftContextCodeUnits, + rightContextCodeUnits: capabilities.maxRightContextCodeUnits + }; + + return this.configuration; } predict(transform: Transform, context: Context, injectedSuggestions?: Suggestion[]): Suggestion[] { diff --git a/common/predictive-text/worker/models/wordlist-model.ts b/common/predictive-text/worker/models/wordlist-model.ts index 3199cdc9be..2f3d564eb3 100644 --- a/common/predictive-text/worker/models/wordlist-model.ts +++ b/common/predictive-text/worker/models/wordlist-model.ts @@ -41,16 +41,18 @@ const MAX_SUGGESTIONS = 3; export class WordListModel implements WorkerInternalModel { - capabilities: Capabilities; + configuration: Configuration; private _wordlist: string[]; - constructor(_capabilities: Capabilities, wordlist: string[]) { - this.capabilities = _capabilities; + constructor(wordlist: string[]) { this._wordlist = wordlist; } - getCapabilities(): Capabilities { - return this.capabilities; + configure(capabilities: Capabilities): Configuration { + return this.configuration = { + leftContextCodeUnits: capabilities.maxLeftContextCodeUnits, + rightContextCodeUnits: capabilities.maxRightContextCodeUnits + }; } predict(transform: Transform, context: Context): Suggestion[] { diff --git a/common/predictive-text/worker/worker-interfaces.ts b/common/predictive-text/worker/worker-interfaces.ts index 800b52fdd4..fa1d148c82 100644 --- a/common/predictive-text/worker/worker-interfaces.ts +++ b/common/predictive-text/worker/worker-interfaces.ts @@ -117,7 +117,7 @@ interface LMLayerWorkerState { * The model implementation, within the Worker. */ interface WorkerInternalModel { - getCapabilities(): Capabilities; + configure(capabilities: Capabilities): Configuration; predict(transform: Transform, context: Context): Suggestion[]; } @@ -129,5 +129,5 @@ interface WorkerInternalModelConstructor { * WorkerInternalModel instances are all given the keyboard's * capabilities, plus any parameters they require. */ - new(capabilities: Capabilities, ...modelParameters: any[]): WorkerInternalModel; + new(...modelParameters: any[]): WorkerInternalModel; } From a873695c10623aecfece1275bd5b18f45266f4d0 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 6 Mar 2019 11:08:21 +0700 Subject: [PATCH 15/22] Updates and improves LMLayer definition files and their links. --- common/predictive-text/build.sh | 16 ++++++++++++++-- common/predictive-text/index.d.ts | 8 +++++++- web/source/includes/lmMsgs.d.ts | 2 +- web/source/includes/lmlayer.ts | 2 +- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/common/predictive-text/build.sh b/common/predictive-text/build.sh index ca5c4e77bf..4cd91ee4c7 100755 --- a/common/predictive-text/build.sh +++ b/common/predictive-text/build.sh @@ -9,12 +9,18 @@ LMLAYER_OUTPUT=build WORKER_OUTPUT=build/intermediate +INCLUDES_OUTPUT=build/includes NAKED_WORKER=$WORKER_OUTPUT/index.js EMBEDDED_WORKER=$WORKER_OUTPUT/embedded_worker.js # Build the worker and the main script. build ( ) { + # Ensure that the build-product destination for any generated include .d.ts files exists. + if ! [-d $INCLUDES_OUTPUT ]; then + mkdir -p "$INCLUDES_OUTPUT" + fi + # Build worker first; the main file depends on it. # Then wrap the worker; Then build the main file. @@ -24,8 +30,8 @@ build ( ) { # Builds the top-level JavaScript file (the second stage of compilation) build-main () { npm run tsc -- -p ./tsconfig.json || fail "Could not build top-level JavaScript file." - cp ./index.d.ts build/index.d.ts - cp ./message.d.ts build/message.d.ts + cp ./index.d.ts $INCLUDES_OUTPUT/LMLayer.d.ts + cp ./message.d.ts $INCLUDES_OUTPUT/message.d.ts } # Builds the inner JavaScript worker (the first stage of compilation). @@ -36,6 +42,12 @@ build-worker () { fi npm run tsc -- -p ./worker/tsconfig.json || fail "Could not build worker." + + # Tweak the output index.d.ts to have an updated reference to message.d.ts + sed -i 's/path="\.\.\/\.\.\/message\.d\.ts"/path="message\.d\.ts"/g' $WORKER_OUTPUT/index.d.ts \ + || fail "Could not update message.d.ts reference" + + mv $WORKER_OUTPUT/index.d.ts $INCLUDES_OUTPUT/LMLayerWorker.d.ts } # A nice, extensible method for -clean operations. Add to this as necessary. diff --git a/common/predictive-text/index.d.ts b/common/predictive-text/index.d.ts index 0e6508e868..1595f98944 100644 --- a/common/predictive-text/index.d.ts +++ b/common/predictive-text/index.d.ts @@ -18,7 +18,7 @@ declare namespace com.keyman.text.prediction { * @param uri URI of the underlying LMLayer worker code. This will usually be a blob: * or file: URI. If uri is not provided, this will start the default Worker. */ - constructor(worker?: Worker); + constructor(capabilities: Capabilities, worker?: Worker); /** * Initializes the LMLayer worker with the keyboard/platform's capabilities, @@ -62,5 +62,11 @@ declare namespace com.keyman.text.prediction { * })); */ static asBlobURI(fn: Function): string; + + /** + * Clears out any computational resources in use by the LMLayer, including shutting + * down any internal WebWorkers. + */ + public shutdown(): void; } } \ No newline at end of file diff --git a/web/source/includes/lmMsgs.d.ts b/web/source/includes/lmMsgs.d.ts index c91043a7ba..2ef1e5a80b 100644 --- a/web/source/includes/lmMsgs.d.ts +++ b/web/source/includes/lmMsgs.d.ts @@ -1 +1 @@ -/// \ No newline at end of file +/// \ No newline at end of file diff --git a/web/source/includes/lmlayer.ts b/web/source/includes/lmlayer.ts index dd0c9ef425..9a3034a944 100644 --- a/web/source/includes/lmlayer.ts +++ b/web/source/includes/lmlayer.ts @@ -4,4 +4,4 @@ // Defines the main interface of the Language Modeling Layer (LMLayer) and its original typing information. /// -/// +/// From 69f731941473dd8a1f10d0d18af1eaf107ef2491 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 6 Mar 2019 11:35:52 +0700 Subject: [PATCH 16/22] Right, KMW links to the actual LMLayer code, not the .d.ts file... --- common/predictive-text/build.sh | 4 ++-- web/source/includes/lmlayer.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common/predictive-text/build.sh b/common/predictive-text/build.sh index 4cd91ee4c7..b792514f8a 100755 --- a/common/predictive-text/build.sh +++ b/common/predictive-text/build.sh @@ -17,7 +17,7 @@ EMBEDDED_WORKER=$WORKER_OUTPUT/embedded_worker.js # Build the worker and the main script. build ( ) { # Ensure that the build-product destination for any generated include .d.ts files exists. - if ! [-d $INCLUDES_OUTPUT ]; then + if ! [ -d $INCLUDES_OUTPUT ]; then mkdir -p "$INCLUDES_OUTPUT" fi @@ -44,7 +44,7 @@ build-worker () { npm run tsc -- -p ./worker/tsconfig.json || fail "Could not build worker." # Tweak the output index.d.ts to have an updated reference to message.d.ts - sed -i 's/path="\.\.\/\.\.\/message\.d\.ts"/path="message\.d\.ts"/g' $WORKER_OUTPUT/index.d.ts \ + sed -i 's/path="\.\.\/\.\.\/message\.d\.ts"/path="message\.d\.ts"/g' "${WORKER_OUTPUT}/index.d.ts" \ || fail "Could not update message.d.ts reference" mv $WORKER_OUTPUT/index.d.ts $INCLUDES_OUTPUT/LMLayerWorker.d.ts diff --git a/web/source/includes/lmlayer.ts b/web/source/includes/lmlayer.ts index 9a3034a944..dd0c9ef425 100644 --- a/web/source/includes/lmlayer.ts +++ b/web/source/includes/lmlayer.ts @@ -4,4 +4,4 @@ // Defines the main interface of the Language Modeling Layer (LMLayer) and its original typing information. /// -/// +/// From 9a1a21d76720755873256aa2cc63c071d567e0f2 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 6 Mar 2019 11:53:49 +0700 Subject: [PATCH 17/22] Workaround for the mac sed issue, plus minor doc for one other point. --- common/predictive-text/build.sh | 10 +++++++++- web/source/includes/lmlayer.ts | 3 +++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/common/predictive-text/build.sh b/common/predictive-text/build.sh index b792514f8a..5513b0b768 100755 --- a/common/predictive-text/build.sh +++ b/common/predictive-text/build.sh @@ -43,8 +43,16 @@ build-worker () { npm run tsc -- -p ./worker/tsconfig.json || fail "Could not build worker." + get_builder_OS + + # macOS has a slightly different sed, which needs an extension to use for a backup file. Thanks, Apple. + BACKUP_EXT= + if [ os_id == 'mac' ]; then + BACKUP_EXT='.bak' + fi + # Tweak the output index.d.ts to have an updated reference to message.d.ts - sed -i 's/path="\.\.\/\.\.\/message\.d\.ts"/path="message\.d\.ts"/g' "${WORKER_OUTPUT}/index.d.ts" \ + sed -i $BACKUP_EXT 's/path="\.\.\/\.\.\/message\.d\.ts"/path="message\.d\.ts"/g' "${WORKER_OUTPUT}/index.d.ts" \ || fail "Could not update message.d.ts reference" mv $WORKER_OUTPUT/index.d.ts $INCLUDES_OUTPUT/LMLayerWorker.d.ts diff --git a/web/source/includes/lmlayer.ts b/web/source/includes/lmlayer.ts index dd0c9ef425..e34f8c6d21 100644 --- a/web/source/includes/lmlayer.ts +++ b/web/source/includes/lmlayer.ts @@ -5,3 +5,6 @@ // Defines the main interface of the Language Modeling Layer (LMLayer) and its original typing information. /// /// + +// We DO need the embedded_worker.d.ts file - since we're directly linking into the LMLayer's code, +// we need the typedef for the embedded worker to be linked. \ No newline at end of file From 810edb2fde26d8ec411255ed7af90cd90994b73d Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 6 Mar 2019 12:13:25 +0700 Subject: [PATCH 18/22] Forgot the $ symbol in the if-condition. --- common/predictive-text/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/predictive-text/build.sh b/common/predictive-text/build.sh index 5513b0b768..5876eeffe9 100755 --- a/common/predictive-text/build.sh +++ b/common/predictive-text/build.sh @@ -47,7 +47,7 @@ build-worker () { # macOS has a slightly different sed, which needs an extension to use for a backup file. Thanks, Apple. BACKUP_EXT= - if [ os_id == 'mac' ]; then + if [ $os_id == 'mac' ]; then BACKUP_EXT='.bak' fi From ab22076350378aea447186c4100c7b53826337a8 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Wed, 6 Mar 2019 13:04:18 -0700 Subject: [PATCH 19/22] Fix an inconsistent comment. --- .../predictive-text/unit_tests/headless/top-level-lmlayer.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/common/predictive-text/unit_tests/headless/top-level-lmlayer.js b/common/predictive-text/unit_tests/headless/top-level-lmlayer.js index 7ad903a070..bb9d9e0c58 100644 --- a/common/predictive-text/unit_tests/headless/top-level-lmlayer.js +++ b/common/predictive-text/unit_tests/headless/top-level-lmlayer.js @@ -16,8 +16,7 @@ describe('LMLayer', function() { let lmLayer = new LMLayer(capabilities(), fakeWorker); assert.propertyVal(fakeWorker.postMessage, 'callCount', 1); - // In the "Worker", assert the message looks right and - // ASYNCHRONOUSLY reply with ready message. + // In the "Worker", assert the message looks right function fakePostMessage(data) { assert.propertyVal(data, 'message', 'config'); assert.isObject(data.capabilities); From c53f493b9cc2cd5c0ef0e8fd9ad0faa04b805c28 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Wed, 6 Mar 2019 14:50:25 -0700 Subject: [PATCH 20/22] Add a state diagram of the LMLayer worker. --- common/predictive-text/docs/build.sh | 2 ++ common/predictive-text/docs/lmlayer-states.dot | 13 +++++++++++++ common/predictive-text/docs/lmlayer-states.png | Bin 0 -> 23087 bytes .../docs/worker-communication-protocol.md | 14 ++++++++++++++ 4 files changed, 29 insertions(+) create mode 100755 common/predictive-text/docs/build.sh create mode 100644 common/predictive-text/docs/lmlayer-states.dot create mode 100644 common/predictive-text/docs/lmlayer-states.png diff --git a/common/predictive-text/docs/build.sh b/common/predictive-text/docs/build.sh new file mode 100755 index 0000000000..ea6464429e --- /dev/null +++ b/common/predictive-text/docs/build.sh @@ -0,0 +1,2 @@ +#!/bin/sh +dot -Tpng lmlayer-states.dot -o lmlayer-states.png diff --git a/common/predictive-text/docs/lmlayer-states.dot b/common/predictive-text/docs/lmlayer-states.dot new file mode 100644 index 0000000000..6c0af8d210 --- /dev/null +++ b/common/predictive-text/docs/lmlayer-states.dot @@ -0,0 +1,13 @@ +digraph LMLayerFSM { + rankdir=LR; + + node [shape = point] 0; + edge [fontname="Courier"]; + + node [shape = circle]; + 0 -> unconfigured; + unconfigured -> modelless [label="config"]; + modelless -> ready [label="load"]; + ready -> ready [label="predict"]; + ready -> modelless [label="unload"]; +} diff --git a/common/predictive-text/docs/lmlayer-states.png b/common/predictive-text/docs/lmlayer-states.png new file mode 100644 index 0000000000000000000000000000000000000000..60cfd0a59598305944cd5af54389f4755ebf2eef GIT binary patch literal 23087 zcmY&=V|X6Xw{PqwZ)`Pa%qD4Un~iPTwr#s{W7|%f#&*)!#+{yX{^#ENC6i~8XJ*fy zwb%M#?TJv36aR#OhX4i!_DNDgL102Ael*EO=YNiSPf`JKu zNs0)nxPzbNz;L|6Z%#7m0v}SvRsF zimF0aR+k^x~#7(`{wC%)kpb!?Mb%nF!!3f!TsXG zg)-EiR38op;(re$Oi@%}8}Up~NZ`V~p#MDxY<(cX{J$rFKa-H?j~fW|cYpf-{&UjN z{NewM}FPwqU*3!HcRc2DNWb2{A9V|$IJby9ipvrC1xqX0)hp1?Eihf zV&s5DU9TyZ)1}|lwfzX^4?Q1UF9%t6*M{~3pEz2@i980oUd}r_7QfeDR<~?8Zu&gv zEO%r&s^`6Tyjak(<6SIQ{?U0d zp)=~RpzS2fVZBuA=;qRp(iL|0YYy)9MDN3K`zG(>b!3Wpgy8vU{lfRqG{Ypdl+TTu z^FLFZzXCVOzG`jTvhM2iv**3NH z?|Njy;Mmx6D3bti-9@+w1T-7n`(j9s^hT0VF4gySo( zH#0w1U#}+^ZTChKRw!*ZkqNn4Yjir=(dEAN(btcD&Tt(-X$Hd5Yi-r8=PXs%)1u|9 zc{Nm(zFTn%1_@`&bTYkQ;xN;SffpIUgs;z?)%(%@VJ}JkSiO4?O}e$Zd1d@Wp@qL5C``rsL^UsZshXPuiLJ&IvXVgMlba9=oGKRj>rQrk`8L+QgaeI%7c%W zo)0I%axLH;+`^zBY(gN?ZPkLV`wT~$*TVTfUK@PTTR<3+$&wJ(@HL`id+Bf}#3-1h zr=K|1e%Uns7V{v^h6!70xBKr^6i)(m-u+_x@Pq$NGNs=nO=rR9vJaNtv*ni%Vq2D7 zH)m0Cj?02ZEKW}x|NByP9 zU8lW|?N`J2l>rAgGyID_3I_^Xjv)>$UJvKzF&rEAPScz}jF3jQg!K}d7In2oUg6Ox zTY>1|aeJ8jIGtcN&GWgRtXJcQf=YrB7ceE##Srv6F8|@FING@ut;^7uhX4DbbvL$M zdxx-dQonPpSe0KZquiry`)#St=nWcMd+TA{O^=b4jS(g2-sovR?E#k1IX*3HG0LU> zc%%j9KRF}dN3N&T2*Geqk{s7q)ZMQr(J4LGWfLxjNYcS5!vzQ`#Yt!4VcKEF;j+Jf ze!OsBbpH9c96)Ke$>Z~J##h9617@*oVsL3^hX2D`r~7)G)@&tt51mme4SS-Hn8P;P zj9c;e_5I}{@2{3EasTuPkyn*1S)`Ww+>Y0xj@$g5*I83qE;en?$GaCJgOyJAFrL-X z{q4&zzTNG2J)b59Y_gc1^Y#;#JZ7WeMsto${#I03JWzn&`%PYj^%)SczJ1WxbNB** z>Ti9p#8n(2la$B4YK9Bl}?w8G6E{a_KC$Hvng5but z&pR$Qj)P%Ph^lt?_S5t>3<<|cVNyEI+bf5zFXPnrJJk_g^oARNrNMCH@%;BkS8I+- zjyjD?+5j@Q%m@G5?H^(E=a@t)A~v1x5su+#-%hF;JnJGo>)4yk@r!1#w5+$?mJGtM zCWeZBjBwxn64s)NyO#I;;Y#~bNG|~a;V76hDKdv4r%*YGJl9E&CA=@z|23DqPxg}; zIgDmY{=qllvvT@4q&0Qg z7V9sLTD50J!eBcx5u1Oa_3M+;m*X8|3gP=|woRj}A(`1_1Wx6+&-QG)?$pArz0TVX zEzv$TeC9*DC}=3g>Cl(O|L;2cl@!95z~~B5Xg0Jpwc4PQ#~#IZaMp0mvI`n8Sp4cLwmtWSI68PT^#N0H$cpFk0)|mw@sLZD4W|~_rkue2L z4ozjQzXe!G?LcAG^xJftpwG;Z$@;Y^aUC%`CQ%>(xA7z%d&1|8Sc?FU71!udj*M16fo_!CZk z#mi2r5Tb*4?I)mob*ku$)rQ5V&?7YV99L9+P2zHUx$MzEhTir>)DDPu?f)afvz=j< zTiy`j`9KRl5*8F`jAbg8KF8R8^jX$kQfRXZ<$vDUm>`Kd@;2+>Dpr^PUCcZP&3Xr{ zZU+;sIb;ze{rR_~Q=KB}Zp08dssR0l>-D{VsF(w}F1WS{ zT|OLld9MeqmyVH@z$D`bA4RDTU+r|(|6;@@5r0dn)+pSpNb1nr2%8A`=)%qvFJyA} znx+NKgw^3GWKMUCq4@6f{n|BJrIIsZ&m+=LBqhhA@ub3Q+hN2t0Y$2o-4bNng1)JRQQ8UVvi8IWo!P)S1js>iv&4%qh+dukcrp@ieR7_ z*5WCprVgU)U`o;m3KmsJ%?_1WTY86-J%l?7r!tt@McC9{JDO-iO5w<6ejP)V@gArC z(aL?^I-pnq=hhkV-%w6~3pM^xXdg8>y$uPBDl9;gvVp0TSYeWhNKrlm3bfWLnBO!u ztsRPzAc9$LT?kJZ;7Dnm7i`x+uy9GEPfvc6ihn;&`?GPWjonaw4R%>$5^3x_aI4Y0 zdX$}L#+Bng0alOdPZ`?kPW}A{!v=sDb%h7)zZBK>SB|y@-+@-xg$hR{D5FnPzU%&O zpZ;}v3ZSaxsfD)|>M*uesNyjcysx7%ILtyGh$Y@}s*vTM&j-ngoF_Rjaj@=nD{+OHMl9BhJ8t7Yph{-MKFzmK>1`fw{Qc*%0$*6wCIVh1pv zyTMFvHDQN_-7O=m@lIf|MCck7AQfFoRM<5>%MWsyz$Hh3CCT^GlDvdmk*I_T)CEnZ zk+>&whwm3`!Kn!b?$u#k7ueFOiO}#0AV-^^Qr!HJ=XYtHR$!2XLPu6wPaOy0+qM=I z?F<6gzozHRgZ8F>`QTF+SY5D|O;bvHSOgsVnr3D>__$nS0ct)0wpu8F7|bfQm>e1nO~W+(M+xL6-|zl`3tQ8QbcOX|UBhvMl!U_rQpvz2aF&F)Z#&^M#C@4jfnWjM zND5S;NV)1VGDrbp;{Hr2ibZ17tKg|T;oDvSQ5mzzvaXjjo^G(*CBY;mpcr;vz?lnI z4vxe4j$1N3S*Q*q$1YD2FvhFP?SVTYJ-@d*qrVVX-s7)b@6RWhwX@wmFY2+%6^hEp z(xfWGp5GANmQAq5ee}yLTYD+*gn>|zx%r1bO+%|2rn&A;q6wElWgze>_|{CgH20l| zFKcMhbOqZqU0x5aRzJI3L#l^c@XyWHY@n9q)0qTmSrlMXSDIQIWr5*~_#e|MEiA+k za4PXCJLEzpyH)<|ZXYIk-)HoTxKV&!W#6wQ`MO`7&TJGqy%8~GaGyPx=IUWKN1#B*&AttFzJoL+$CuOL>EGiDKXCwDfe$ZMnwQ}Q{P4#mJ01dc~ ze!$I_wT%YIuK4?>`*lH;c#gr`8nRgnOE+6(!%0v!U{~NX64>e{4}L>2vB8JHBqM>R z)K&j<%g1|MRs__Uves(ybE+$q_tGre>weK=vn(6-8P4%yt_YnbK3s@MfAvpqKsr06 zNNPO@51)^BWYq0vwe+=aAZQjO3_Ax$fL)^)Bm*HDCu&34f4V(e{cZt41h9|x^y8$c z*t@D+IFPs8H68-KkR52;~t`+Ee+D+=|9;D(U_PBoQWqhS2?Z@;nd< zA~p?E?Bn=S1p3v`j@E0-bFnoguX^+-3o;{tC*e!P?!ERDS%&Gam52t6=ioK?V?w`B z({Ui~M8I?f*D42Dw>>G?u^y~gX+yVU?^&*r>?v6cxUWaaTu9ZXaSqVOmL;P2YP=)(C# zaGlS`l@%C?_KpC&H}9IiThRJRq*K1iOgek8mQhwf^%E#|&pnU7ju*ag*OG!jB=2d1 z@t$gHf#A@Qo&RV`U+>k5)z9(`K`hpuT{I`!@BVbjNS8Q5f8v|dJfQ6<9I@$o(oddB z;hqB#8PZ36HGgEHfMeuEF|?2NWr&3&02F`2{o6hZ>)+C8x>S{1S-P@=1dHyMo`h+* zpCs_n@ttl!m{ID8` zJD_g$1&t<0k2&lPQ#;u0=EmH0qJq*nok~~THhof%cL4=qvAs$yuXR7g3!rT4x?X1` z6tCe8fm+jppfXqp;ZP)pM1d4)ag8WGg(oV6zGiY~&a#!Aif&WeeU=*bzkB~`Ht_H_ zjUp)Tg3T5Svf1!BoNxbaY_nTV znv~H>3%uo2xZckSapRge1^SCrZTI1h5X5s$X;JgU>Yf1n`=wvES(1#@ zfBzC#9&;xhE0&dPN{iJ$;l=%z+I-)hElP;$_d~2$w|V8%9!P3NrlKhl^1Z!2C{Lch zTG{1SkYVLSQaR8Xw&4`wFm&C{X8I>~FuclFy~5yIgWjv3TQrc#1i0KU_2~nB07|+X zMGc`F(+Xbz0r2}$WFceOOdN=Lfh|C?_+EiUNvvKf%S=bCDizKP0LWv@4bPKmLWe9M zbF9HBjKQMz-?yU7SNLd;t4|s%j(%~P;cW>m+td19ZBj~*Dv+3l776K$PFvGz7*TEG zQuJBCo-yb_oY<#W5+cTvXyIaygvok|Y5#&75F`kQ$=qvRSE>LmDg~OpX&b`QGB2MU z*J|MX6fg^?fQ7B${99Doj$i;wDS^UHhBVUX;d~vDCU*W9D3+q}NFh7_YEE#qD0`Et ztiOLUy%U*OwEaJs?plJak9wJU4nAOeAnb%Z$m{RpZro30#>|si+Umwcns5xF-YN0Y z(g;!zHUw3e`g4R|br_)LOcQz@6(A3YHs=!<3dU*4aMA@{!hgxmS4FJpem?%JwOnB; zOu5*&)9^?IG+K>fImI|%%3ckWc%kxhKDQkLm_QZHt705TDGnt5 z4(9JqhjB%JM{T6e9QHpZHv$H$T3bIg#QjmCn9@zU-bDW*HwOhRb=YXP$+vG_d~zFs zo+Nal`Eb{d$SAWowz}L5AMmf>o6@rd#mwi+DqyX7wC$%V>nfqC+;4c<$tqEcz9`l* zYZ<^{2!JWw3o-jWxjq3>#yT+CT|mJ+;Cne`rT8h2CTM6MiFEIFYz?Ib!Xr)w0$)@TWyAw~ID80-Y)7T5U4 zV$e8=PdLXrZV#Y}WGYIR(;p?s?1+LPb5(p-s2gtu?GACpR41AB`as9%)Z~BbAA)~i zCLV^z5QO6R8c8fhK^I3ALY1NNtq)&R9SRzUDkQ8nHXeQlF5<_&S3wBYtjlY`S$xW5 z^FOVk14xjpC%exE;|4^=ZPZjYqNwf+^^#nuOU6mE1SQir!2lvLgahRbk{gC#swo^J zx{u22i2DnoW#UshycA$ zgG`FH5*pk05~aoZ;Ve6Wto6M*Mutw2Wm~{xG61eaK96s_I=Up`Ls2ZDvZj5U(0wZ) zA@+A^Ofeay2HdFh24ih6#5=&drZp}3|AhOi$X)H!Tq|wy#Qd;3BtQjscyQ0SdYR7Q zezEI*qTJDK)@TC7FWEDueCC;{j38B@hpOo~=VCn9WmMy1oR5B0lBVAmUA`_X^N{>@ zqjXiK88+U?YBY?*82KX&ElZa!UY9aMw=DaW`bT!*$iX9^%uZPoAEGo69F#5iJ^#rg z2>JQ@D8`x%HXGv|F$ZjDDuRL<13OVj{WQ4&6dzHnq{9o4_3@O9rg7f|FmP>eW5$1a z$~hpxR(qidev(^8R)mD2inJd@6*q5Yveg}isPbqkY#gX}pW%Hl4@ba~?nE_a+D)y2 zr&r6)G%ISNXT)Z-*wx($!K*%5*h~lMFHi*+>7pqcas{%jj>5?RAgp>~=h$NJkNIP_ zA5a8zwY`0TYgs=!)uER$LMY)5MJbiF_p(gT?TW_|{PhDAsdm@n5NzE`V8SajQA*T< z(iDCyy%?nqRN;>LzMLKA`n`L6IoPSMPp;3b&65oW<8j$Bv^SjQuyRD#-IS7#kRqHr zpj?Q@?UwXG#o>0X`X}NC83fXq0x4P>%C`M4r3o%ccYOPX&&$2d5UaXv*8EB?j`(w! z+0AcpVwWRY&>15jA0;=f~dzOZ48MKp&M(iH7-G=hC1C2MVJ(@MctHuym2& zmV+d*|0M|PQJ%XsH1!9pOI94a?lBzkmmeTy|NUmbJyAhIGRLah3OSERQeff>z&lk7 zjvBG^)?MV701Qq1nTv0ObYVSz$0tF9&XsJ*F|3vQQ2p;tv8^nlzo<7&>cU*b!?`sY zFve2P);n6j6G2ejfVjBtAM{*=%fm55fVY$)ZQ_M%HA7vy~<*e9g-BnzfUE z<+EcalJ*}>KHu7_Ha&B#jQ_ld)-N?Cq3Tpm9op2iK$zUxiv? z*h+NmgE&Z8;Sk!~0GaUj1?q|4B%DHxbeYXpBf0}&jT?9oeE<&~_XjAWw8Tm}qHbwk z`EsNN?CBP1KFCm6%*DI{CJy&;zOqMgMljd3(BU3{aZ#)18kbG8RJ}dRTL==3Ns#tR zan!sKORU7TCpuNamDwb*A@cm*+whqUgOef&N^#hGOTq#XWj045DzK+7C|BGSvgyV6;EAK*BVK^VP>Gh5rt&o;QT_?E ztykjstI^^Q&9P-gity}0iEiP5s<|PYh@gHf=Yc|?&Zva!Y>>Z-to@?4Qw-U;%JF0A z;UNC2OdRI{P8h9Xg)ZA;hWf*5AQ*EQ1`A0;TGfUr_Aqt%@6%LuL-?l8Xl5EU#hRRZ zV#|Q@ll^-!*=x1f>t)?-Q-_uGhfGGIEs2uzJ$W@V83&`VkUF6xT21%T5!Ue?=u0^k2OxHHLXJo^gA^LjaOZs+%+rL3U)^<&=B zCv<|oB|8*3!P`8M0jd}y*T_+P;hC;sGO;G<aVVj(&lPrM;_$MaJdJfb|TLX6ClwUu^bUYNMkP*q7?^Xb;l=a)@e7Op8 zh9JZOU^r!H7ri&(ccA^Y9%29Re79BA(qsH&DS(j_)azGGrw zVnMmQUYeh!>KT=bH}1~YSGxf7FP5v3aw@79;9)=4QXyfHouwJOoNsc?foC~9-28ZV zE4ia?@}fx@h(Q0swH63mSPQg8^bFX!If^Td&_jFY6C|vHG}K#}EEe8S?3>vX&XY6_10Zfdn1MW7A+~3v>QwO#8TG{qDy54Xz4%A5{ zoq&hOb`jCMJxN9TH1;JgIl4zS$MNyBVTR@}n>OI30|WNw$$1I)qB#@#vR6EA7kk_|K_^?{1kS+_%ko-t3{&M zZVU&4E4$11HlQJ{1163x2m}DzbN1s-G-3IMZ8L-NR@xWA{z*@pmEV7k&Kqv6SV9#LRX*rye8MZ}Sr|Ej)Bh`H zNd(#s%k&3uuV}_le}H>_#?|fz07tBK{8A6PyX<1b_5!{&y&-)YmJ_?@1`T}{+7SKhUfQg8krs4-U!9Ul;vhNvSUvyvp zl7s@fQU_aIR7j`!C`iOK$3<=GT*o*;^w#e^p9ckpNx&%J(XXcMz>!Qm)|nh`F9`^0 zrMS5L*xmQ5I6r5Z<8aI(yTS2EVC+w>)hjIh^645acAMRnHk!g?0{#l|6O27PxH%R^ zjPBS9;K5_mGtW}A)q7ND8Cq|MR-3?7?&`X4L&p&A;g;^8I5R$_bG!fSg&(`;e}63M zW(BCu1^Yew#`>60mN~~4$R_*DD!Qi zN%d+Z;{4CLVZ3(jWbU_A8Ge31bvU-WpTH`ouAUfRcIp{=OYUk*!(n8UD#ZpI-V(Wo zl>B?F)nLD)Br0S273}>#ALnj8H@7jafS!F)>tVf#R(tl(DP3A=2|Wq(mnpn`M}sc+ z%?YM0ri*sw%saz=cGCp_yWW7zZEe6NTp)M;LHgYLMmpx| zyHso7@e*}H&UXDv80GJjj=8?nCy;BQs;Iybb7%7RO3SP#yd>P*G>tkv026hQyBM>h zj$<-nv30eS6pEX+L2aI}6fj&Gc1_S>KSYUd3u&NR8JM=I|8@4##WD=J^8II$`yA<2 zi&WLU7bnS8%x|B5V-xXziF{F``_0B<`4mP|N`UZD$~Pc%eL7^7V4+>M(y0z{_nIyN zV*o*w(fJNEJ-|o30(Gr?HOFzBI>Y1@6qU$um@|BD=rZ>p>&{QE$cbdceQ1Y97yv+s zb!iyC4ML@q_-gy44xSs8;|EsUC;4IFfoE36LqM8XBb=T*8j(It^0n!;#L~Dzr z(ii_Eu*=)+W6Dw}H#^XpFv}&?*(5;n39(LEsG^S{hE?UfuCEo`RL@5k;Z3kF zvdBje(O+a*(|^l_(BhtD)djhE@d=%(P=|r|H92$4Ibv)u3Bi-e9k#_b3;OAY-io}? zu+wQV(y&i*uNmqYRBQsm5&}Md7s^JhlUn|KzK<129?6JSf@S8)o$@M1D!f%VfwhCg zv22`lDq8n0R!7Olne9~t+c10*2zFJBgNUaZ-Y0sE>9n<`;NOsUMi$32Yt!>~e_l~p zGaEs7)xAxakmEy;S|D(cns)ZPNZfGS&~UlQYH{$Mv79i7Gcf%Iz&#zR*vcgYkr8VU zlUnN#PuQR}Z-D<<)prTw?H4%fPtk~5oOl>G8=#(BOJKH12K6(~ z^SC@SpTwd06C%okX6tl}JK9jLU6nv*T)zuhIaDRFMyJA8=CM6ovTejl{bqe;kxPO- zDR;9y2KoLEs-|e=7|Il$ta%=BG#X~2U`avORr-tQQ#Sbbn5+nh<9!5_XcOX8u^1V) zQNuAj3fh!OrFtD|$zaL};jynmrmW2uRRvWF=sqPtz9zG_$63*nELX1)}GoDXi)~K;(j4v4T zSC6&dBr}%c2vWJ$&W~pmgz`H-BcX~>FCIOpnct%&=*ZmE){-h`uYi)~Lhw9yA&G{X z;9w9YZ=LCIIaw%tgtRp}kFgGT0%HefNeOaEyrar)BI%(thC$Ke?7xZG{(+h@AeB4i zSWDeRuZtUvtkPpa;b_E`9qg}_vHU_R9O}NU73GW!wFuOk?Sy988!@5&3h*%~TUgRx z-&;5Pdm`Rb)ACK1<|~rWE(+Zf_OXAAMh!+;iN}e$)?vs$D2dO74WdU(++4?-wF?~7 z^IYn#3G3>fz5`h5{`&e3P<2}wPy&v2Snh>Tik8QW$#nr2X$28nQ&6mosD*>rZsUr3+S3Y2!0G^lm zl-u}zbN*>2WDt?|e%r6xU`@c-@hC(=cQLB)$OLUKq+vQqzxNK!E)<)FW?fA?G;^Mg zIgIO_&}B|hmol!MaP@lfMi)-f3ZM_UHRu$-r71o!UiI+ZhMXB7fr*vDP~NYLDNo7&>VN{s2QPKRX#1hjdKjgb zND|9je*XjvYRSdp$KBzW8?@TopS{!?F!t)ie=H!DJQ)yAAqlb=q}i2=DrcScMNS<#<~fm}9_ zg(x@X)b3o5B|0cvbWxhlAj%E=!bbAwdWRD}>o3PR8SY~NuYJY2Z{>f84X&TN#CL{J#kB*O9t@|?)>B;xW3{9bC6NF|(*ul`*(!kH&2_da~w#j4F^VmSG36Bs{;AhGk%9 z|Me#dbD%+q_RQ7c&?)jLu=gfydI$F|0bLCRz9-c!wtdX7emSk*cv(@NV_c2__@j!5 z&dPVc>s`l1I;juIu#BaxYpkA)lU!gP@H6U0xOVt=4V1DK_s+rW{<2@~nj}okVRya` zY2ZAz);+t!d*~OWPD@UweNBhXHNpH}9{qnq>oH*bg;ptCIK5-2+V&-y9n)WOxBTd+ zsq&Q~3ZcKO>DT?be2g`j7{h|VT5qzJ04bft+6Y1^Gt{Ar7Tmbably5}S%+L{I`IcIX&{U&Z+2z?h z@D}(GEs1flXV}mE%}blAnf?L?{!Brt+;COZN$1`P&$oz#rDmXI^l=zxz#S*pA;x7M zZCb0}xikLaX zjDv!5c0VWI#Pr#U9JTR+!Fwv^&zfGX-?Y>JS-=VH|=ORZ54I7 z4Cw1X{Vjviu%XaRiDV;YqvU!ezDeD-F^VZ2;^D2xNy=Dgt>>X=*)o+|@Pk+Kg`t72 z5n(Lt;1&3%0m$JdgM&z(`z532thZj0%e8IxlvC9%)5+Ao(g5Gr+8TLR!k^uhOIhNY z>58hC(rQ(*qEJ8mM#&RQIWGuCk!q%P%x<=}$V0p@LAoDrg~``< z7djHo_9I}zH+!N!5`XtBHw(Q=JyhS>Pz~5`6GIt8_KCW-X3=$$8Q!r@FlJ(#4F>t2 zx}>~|iH~ty<%YnNpyQ%)B%>oL9(-I1W6)MtOscMhJ(u z;-{EJ6+;BOn#VPB!noV(3GIGH2U`ynunwHs-~Nr^ql0rpj7B{8sobPYC{%wk%=-Ai z1fl>QW%v2W6DlB1$xvYZpWimq9E_6ez2G5M#Dfi}MozyEqjo2Z=iyN~#h~6*15m*> zIxcO=I78g}zHUDwX*B*qy+ST-%#w*@%aHA)!h2*$Qj@h$g3n7@`crt)&s_ON582qw z;jx%O4hxQUQYhuFD_ODizUP5yLUVjB+JY_|oO+V%ZtRx?>2+;tnts< zzwWYKvehJQacbdV_eYbWSPesU2SX}{UzW_2p$<8Rij;<22bjD8ENY3p&Sy=F%~Dbs z^evw-%Dg}0b?g?%1!5c34-tFh!-$P(uY-zd?{{@0AdQ_189LcrCb0Gmt$nOAu7;OI&27WnJX@`k! zyc1gixsPPLfkCZ3aYi)xFzN&(iu2f;+1@O;CAX{b30-@1GGj|=F$)}idunL=n) zA)R|324@X3e3qY5(uGxd6v`6mZu#LjAQ@NI(r_Rth>vhuEe7<_?~Mq@@l;t9wiJ+W zB=764s~H=S_R+l!mVfvm6@x5X(QI3pMln}7+&mO`8eY(n<<7YybSBqDKhn~<-%LYO z=(RUBb$10oQ%Fc?7D>h={x0}dZp`otuOUlA zJd+PUkKd>wwVb-_u>1+oSbJQK(t1rzm&&=^AuWJF-k?sF>I4lfG?^DielT5u+0lhl zjtL6vz(%=J^8B=)Ftj-I*zZniy{#X~F@&vfmMN6w!BS_+ctB$E)=Ff(;?Ysw=3An6VPaB z+x{y#<1F9cLDn*V#T`cHVGY(rWeMp}U6PzD9C1Vq@~PgJ=3o2OS1m*wNTHUfER-hYYqSw+oN)r$IbMS2Kt|v`#9w1g zbXhLsHC1WV7DlTtHx=5Gsh*9^>ith-h;2UO`ibc5hb&_oE=gpEZ9W9? zoo=I9Wh?5v4fwaMH_Y69!x_(Sd4~U1A1y)4oZAulxXV4~)&^xMD52Mz@RtXpaLpuN zzePDI)zCr;mutxhE@&4CzpPQOdD9OSZkOvqMN;HXBm<5YTwOm7DxH`0)MNNIxh*vF zI4fE7XRdCCJTmwB_)BdO4|bQZ&3BL+3QXeL&$`&Qant+ely}NuVk9SupN=@}70pj9 zjjIb6_C{O%1K2atx!9jpJZoOqUesH48&aB1UFP3qcc3!)SNQsJ$%{Z`k@}{{YJ8>w z60SY{TtqPBtMBq?H6PCD>G;r;ehAxHGZ&@JsMXsY%`S@t)QOpdfecE zcvwhLrFxQeprQ7wKd~Eo$xo(L#QiumaD#kU^le-f4%7%*b?gj{P~R}u-AClbnoMR! zhg;Ta8x}z{1k9#xr=N<1p}&%a3Sc{_gq2jZ(?gVuw>P%GZF`_xNzx&{ZDj}oI72ZP zafSBI+oAIk8E(e>o`nY*Zk+vstDs2Hjkzv_d_!{KhiA@G=poJ;>X@e5495I7cRg&a zAKZhNuvOKXYDm3m1(LV1AlF`({uP;M%i707qR)dZVRh7QJ`^8eFI{WQ)G|WMsrsTA z2avox6i62x7v1$EbcwWesfDlC+9RZ$b`YU}P#4p@4YiVdDg{!M7pT(Km2wbW@0?oA z@ws1?D*wz8eSIq^g5boLtF2u$`|Sf(6zQapWv;68Hh zZ6 z&HwJZ=O}mX3u?_|Gr}%~c6MR7W!7aSi!Chm{bcQ@*Kh^-I{vjD`(#(NdY56{ zBMTrIm)btA66d1d$63+swqXva-jm+B(bC|9w^=|-V_R&KE6zzVx$GX0>xSD2c5NY* zspKsYN0DfZJ((&__Lh5%Gc)Ur{AJ(%1_g=!D@H5D2n{kq6qS~U!IaKhFnvD3CZY+Z zQnzM;-#YCAADdWBaPnE5>KYykc^dnK3G34_!lG0y0;@7oYMv(HVk|le#N@T)LBD0Z z56XyB2z55&th zpc7@*%9q<2_6lh$7uZ1Fft^^D<2dqY`z;3%PRQV4gJ=A;I^@UL_*G03rkHm}^ZO>5MOy z;|SfbVsJ!~`U%@TU#t^+6?EI~w-Kon$MF4%Ob2)e?J6;I^h#)i&&*;-xm3kSYQ|Vb zDHJtKvR7_>G%Ik6)6eR7;JR}%`U)iVoL{9cEVRBtKp=Ix>-UrAqdsT<_cy@c*W+9DZ8hOoO-u8@An_?sk3xBv20fC$Dt&@`hhp&Gi#J2M#DS&3f8q8)jj z=R4i4ftt>gYjyQAnE)6cR)&z;^z#FgCJG7qApy?Pz?YTWD_haeN=GQTr%fPg!bLyJUIiIHh5qoYy*ES@qWSVf5)g;7 zM)6Pk>ur;Y5CM}gC1@wHS-;|6Y?AmUc5aw2%WD z?1xUtl7J!wlH%ZMOg-NKZAe8~5 zba-kL;Zgh0;40l{6@tUIOXu(iZCIxP_-O;Ax zxcJQ2NLWB(I^_?-J8jZ|AQ+K+D{W<8&H_)_*O81~;eQ)um*mBcoxDh~)w(+Y|5+ps zR-BHYF%c?%Vml0}pIykKbi%}5P51jZ#-}Oq`=zS%%XDER21=ix?0I))$4VK>#5xwl zd{9Y(P!=7#PFxqULk_sPNJZ6y>Vk*p3;iurb5D1-98(~WrR4js<1HpH-rruhNrex8 zbSp2##;3%GNTcaf2;DKE2?P9kXVqBh%hq&n|-y-->rePLNUuIw4MaFoePF%5knL+cKqqG}f%9`dyzD);#MNp{-4OHkz0KT_j z=^=E2%`_uaEIkL56sZC&wraWW((^Hy|4Vg-VR}WJ!fw;(9eFA(gJrxWDU129e54=r zCA;B_RgLbD=fcb=oEY7zY8n)OM5(guzoT>HRw~pHs{f-S;}%h9g81a=`6?t-(Nt0P z?~3gy_2GlJMHq)PSs6jYuaKV)fsRPyGG{|R?E!Xq2y~vjCo1cH$r2i^jHJ>7W)9AV z9AJ|#!-p9r&eym~fq!F_7Phimzrtrz24{Zbo0CLl>#ii2#ummh3cN@m=L!6KE)$ zqro}$4nx}83J~^~qPGT{(x?^@@>~P3^e9rLnX&=(YO=A=-= zM6UgKDJ(RU?OcxoTxH~_b=>9}atI|dWyqg5UCz?y*7)Q^;Ar3yh6A!#-+#Xh2K^Et zBzHhMSp~LB&o|*E^I`B~eCE5{Gx?$;a31F<2LDycSdFF)9P9>i33o>hP(pBkF9zVQ|N9oe+wG*QI#PmnB>PCDvVwwx zThUhj(bn*=s9ZT`XW*~D_FaxM9MgPkQma6~)Ffs8mb+$*bkk@&or4eAq$amBglmKb zCoQ<^*WaW`@1VC?=^hgjkP`7E^^jNlQN>|GMlzm`&7n!@AS5;<6BwGzZdI4<(ng$n zBPD{NU{eWf8_rmle2+)KY1fX%$k>04m(kNv8bSW>g z1A8bB)h;Snr@sj=N**|tayqbC_D55VWtbG6dPGz&*ic|8rA76hgbYtkK#fm`a5hv^ zZRZZkGgX-iMN(D01KV{U4Eyd1Z!KO7=Wh!^o{E>OBWBzS#BOU287B9;J#Z3T?lW(` z>n@8a!BFVd$K=N8kuM}Jhqhm|d7%^tf{zP4KWR!w2n$56TZR5mCqrB0I8pF`lUU2X zm@7|8OBbH9oKgitd}B;)i2MR)xIJ5?z!H*SEMX`iHn!tV#ibM>F24sfF+2eJX%P*t z5d1FgC@O{+i}YB1_fcV#Gp-+b!XOF}k2=57oZtiE#|^#n*LM2xOohcGPlu?Ne;c#o zhl7RD+{orEJ}6i)BXaRSJxYD;Crrj*cVLbup#IsWr!t63o&tRp{6Mp- z&k^FPl=Y7uz~X1MNqZ4a40{Fky>|YZEBid&w51)2Oqkh6ub1^tS2DGO6=mOKx6r2C z+sl-k=2etZDkqR4sh-9QK`+PPYu*5jM)|8y{K77Ux)e9 z&Wh6|Q7v>@)#;0&CKDgI=bth^F+u>$?_r|VVfV8j3!>H-v`mfwgcN+&w-c;Vbiv1cc)eIK%9Uy4ljeJhNuWG4wB zdqbOuqANuDo#*~u-TQm}{vYN%&pFTc`JOYM_nYgU+LB4aXPwE<6&q$Fk^E|za*mmU z>dIXj*K8W@Td0o6$G&GlRH4fmblBp?Q~T+Bwl?yf6QASt4UMs|2*l+`OVM7SKYi%=?{!L|nu9AKW6m!|fx< zANwv+ok_*y{S0UQnjHQ~u#3h0Vr_{i2Q`6Ag!3l*9k(ej-3zKE{LFVV5;5e_AUFn} z#R5CR&t?R98#&97EU3=j)5cn^%KpQ1=Uck#ylmjw4#-VB6t=N4ijv;uENl40^u5CD z{My=WpIv@znhW_mvb#LG*XkQ>v`hDXn(oDpVO!PgqiC5I*ooKr@Fw>i>|b~cp0LA7 zd7&W7TS%P9mV~$Bzdk)0PsiLntHs3eic6Sjoo<^AqNth%dQoLP6TOrnA{Car*OS!$&gErdx{hu zM*X{x82J4mxxYBBRE?={Q@>Alc$d#&xpXQ~1SP>Xp>+M?sOVYTSKnuZ3%W8azs@|} zkxC<9{t~@Qmv|}Dxsg)nLM$%k90@I%qARMEBYC|}E?FA$&}c_{tlX)(g8XPHg!YB& z*Ol&>=#I_$&X4tC2N7VTNELZpIV`FIaaQ>Mp*QQ_{996t1UN=llHJN^4HgCpI(OjjZcA@W)J>Dc)e zrrPdr+Cl#7X+xF|O= z`+DBn>CrD2H+ZA6-Jbl(!Mj2nPIj+d_Vvi>VC*`l>id|}N^(Y$A=Qyz zS{FUje~3O|+=*8)e*fSe?%ZaetPo#l3??yj{nB0jxBVoa-#S;`tKt(p+W8+7?I|d2-{>~bL)2DxT{5mpN)4NgtW=KKluI@SQlz; zm`RNUbcyUM{FC{_)0q%6ryZvY+7b%QWf=J^?&qaIfllMKQT{6k0*kIw6~jiB-9;TG z7!qrG_L`J$_C!|!4|7et>T`8u8_uRzkrL*arSY_ndLSv>IX5^i=>F4h48^59MIb0@ zan<6vzjKc`=9833=)spVXI)Kweo0_QdxD1d!Uz52Xz$B5y`db6 zhe+Op0Hgr_$XU)=w=0>owkhuVzBd;aE;+bXU3}Reb9BkA!2XAiX&nj%C)@3bTl+Xq zn)6cQV<%tj_9Ip`%5ootiz&SPb#<0^DN>&<_D*TLVJpoz10{Y_Luv$M%xxz*9}#D) zKdZ>cuto(kjB@1|C|PJcJKoh3_0Sq%W$7}pd~>MW;~uY(p?`}>ZMd9)*P*7#ifYqQ zz_RZ!)$*PrW)Rf$hE9<2TYh)%YJGm^-n!P(5DOQt5R1n40Q*gUz?YZ}bZ6Wu$I6!; zGYky;bs#cXf1Z5#2PoH-b)!j{d^bUM)*CoDI?~T+RcBZ5h&}xqjXmUi$EC^GL#fNz zFO3+z&~(otwGqj*pIaDZFebpJ@lE-&#(dv`#Ray<`QB+=%t~Wn zRr||fCTg-wk3B_rXbN-|aJW%knGInui^a1Fut+R`aY^G|HDhU37 zG#C1+U#kV^oTPjs3aOx1@}E#Cyk59X;ag1VZ(~{Qy7(xb)-e^Xbk}L=emgN@wM)%* zn5Nk-3#faDFha*5;nDXErc%hvWrlm-ciU%NhVylVzfad%0&uH=r2fhW9rb%g@b;@> zL=vp5GP*B|_)?RlDtoK)K$FNkA~a(}_xyR58aeq-vRc)~_s&Dk(rYOWVX4&QLF0(o zoZ`QCP(4;29B>3FSD}Gx+LL({XDhpO($A&pVlY4ta)2DLA(UTo=E9o|NH4dhVOAl7HFAoJS2ju$lyt^OQnNCAx`BsOi(p7fBqr`el_ZZG*%}Of1CZ{80V=PT>6*tzmhQyq}ZptAHh<4_3D|coNnfs~03p z!x0j3!CGGH6OV*lq|}Bf*Kp97Z;#IEvwr`sZ@sjT{zy94_f$hfmT5eWbCAN#)3k4< zv=u@69M!;(;oF>97i{3M<275NLa$aUuP=z}dgm#4iG1HrH1;T;e;CA`I-wnJ*nLbL z6vC_-SS=LB%S%x6JWeBIJYG04V1yCbC05mPPQ3#bfwa@)`MsUk?%>a~1-WtpgZgd& zAur;mQaRSMy$R<3K7zn}eE2Hj&YPPN-7#00b1!OAEVYAImS@s#tj85Gb8qdt=y%~` zd6HHqNE7!wMz^x8ct42`5`5`<24lLvPkv>;O67jeABI3qUz;MIy_NS7kdY1uiAbXq zvX2zlZP%({W4cVM+D^c#%2X3~Nua};Dm7nY7Pi+8;H-ACSC3pd z6&BU;+L(!b{&!*E%H-+eb{zM+rc$0(Vp@yuA~2VjUD6{8R-XHbqiUT%Q=vK6C#$0fgX04?XJ78EPxd{koeMd)qUZpCREBT`8xP_h z7^v0RNQWWyv{xXQ*U-`+uVi)u$_AMf88>igq&u299fEvcmLX$21u>hkmv7eM7QH#5 zZO(x<4A1|F122pZ{>$n&`VQp_#_Q_ZUI14h%hJTeN~~vgdjgO_gUl__qji4p%$+%V zt_GHXtBduQ;f*ATG!vmW>qDb`Y1rVWrdT_j)MH(*rpcP{XW&-qkcO21ek18bzRleBRy z%U%Lw|B^upcVE`Cc0fog!5-_@4yFxWO$i$RC%@lV0R`l^g50E8cSr?U*Avm`i?>NV zQM$JavcfH|a-EI>l(OFAoc7%ClSAjau8AP$^je-wnroD_EPd3xp^ctVKSxvtf^Q-E zV(3_GGwBg(8p_l6MVB5-hT+x0gbNT%FDQ;0buh79(Af!!NWgp1I2y zsm7!ryw+0{tNJAe(rWbMBZcvdjRJWrH>gnRaMjEx+>{N%C>7Cb%#=Hx|eqlEt6?~Oon)#>`WT%gQ7ahp2R*ToGmTP0)UT`{667bbJZ z6yB%FfNBvlkZuF3T z>jik@9CGOdWNn272EjEgS_s*7dj!Er%B9lhEO#L6co%&FcJlNQ*zAWrkoQL}nM1PI z7UZe6fv##+mnpl|`G^x1ue^wq_|OPD_P(lW+%&GR33=Fbm(^LNZ*@+SlVjYCHmZUt znhrDc#4PdHIW#@_CkWLlGf)6j;~B{SOyawtdxosiH^UFha)g$FaLD%4c;R0A6W_9` z4$3b$=DT1uv0^!zKVp{qX`cc!nw4i=Z~5&uF7!8m$9I4$uuUq>L@NKnfpQB31YSX4 zm#LuJn=bqc$6O;D`{m^YiK_(>Cx<^za2a<18{P>Hq==v}cH|o_j=w~@aKQEDZ}*i& zHzc!j>4=PlKKK>4LRGo(^t`#qo#lWVFJ&XvAK|ArtoN^3Qqx7fMfSK2;vnd7CQ*<% z7sxPVmrI#n{2ER_cZMdcQ3(YESwB|*c5-D43rvMka5NAka*GcF``uSaomaS1%y;eK z<@OZOh3y&5NUnK6JjDniRWxuinKdbmEjL8rjVPuex=(`H!PjkkG}=H+URXY)CDr)e z9c?mh+1q)uRlMNMcWqkffJ6}pwq@2WnZY(l#d5R}jJzhz1V*T=i8xrLep7QpoL*`O z);J>gKz(_pHHndYT_XAcLk*pb7pTmd$|Qi>v>m;;c~Wc>0kX=~PfZz=4(!ZpKNk3n zYsyi+l#91}C?dqdO!#d~bT7Ap#n8URm`#XyoFir%r=+d37TZ{*jM-X&e1m-`_h!yyve> zi?%HlXF5Jq%+Md6cnF?@jnn0OkQ`0-EVywx38RaV8HO=b^uGTDX#f$?a%}pS=-lj# zGMO7)bB1ETMybDy8BtOa&UGeA{ON(J?KlQYENX8+Cqne2On#h1t`+Uq%WmVD7QOuk zpv|8C`TcSP81Aziy4rxEFF}w!M;2NG+~Jh_PC0L%MMMP1+ypI;!lF2_sjgB4mPX6` z<_W+b-PzuQiM5^iQ75U!M}tTs3j~qSYE?~6ZCr}X>_nc`n%kp6G)2b(s|Do8NvSJG zk@^#vWn_H^lQ})B%5qEuSj*DpM&_N1uFyzW@bD>Xtg&ug0YHua7nxu>QO>{G2MU{v}=>4I)+F#ZHhhLjBbX+%Q0?M9?6Lf1=X9iv_ zRT%Pyp#kT#)SSPVa79e4nU5w?=n${Yw=MPRJTuEWl|LW`LqTWM;^R3r-5Ui>xK7zv7Bh_9N$iD{1v*VD2sha=J=P8Ym zVjXWwmq&36{SH)dtUS z2F0eO1fp()n2_{WD8I2sVs5%KzrgD-)k~@zLNoWlj%|IvqA0vt7rpHvmC|G)22NF~ zV|(As)mdKxKP_yw1I1>iRfW^mzy7&+w-ji-b;F<^ kqlo+U?=tx3@jZyR`8o;vx`SZ~7@)~?G_I-FtJ+2X4^aUa;s5{u literal 0 HcmV?d00001 diff --git a/common/predictive-text/docs/worker-communication-protocol.md b/common/predictive-text/docs/worker-communication-protocol.md index f5422830f6..63f8864060 100644 --- a/common/predictive-text/docs/worker-communication-protocol.md +++ b/common/predictive-text/docs/worker-communication-protocol.md @@ -414,3 +414,17 @@ keyboard to acknowledge late suggestions, or for the LMLayer to avoid sending late `suggestions` messages. In either case, a `suggestions` message can be identified as appropriate or "late" via its `token` property. + + +## *Informative*: LMLayer worker as a state machine + +The LMLayer worker can be seen in the following states: + + - `unconfigured` + - `model-less` + - `ready` + +The transitions of this diagram correspond to messages as described +above. + +![State machine of the LMLayer](./lmlayer-states.png) From a8d6e50b5661cdb39b25f2f18f625546891c1264 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 7 Mar 2019 08:46:22 +0700 Subject: [PATCH 21/22] Addresses more PR review concerns, renames activateModel -> loadModel. --- .../docs/worker-communication-protocol.md | 5 +++-- common/predictive-text/index.d.ts | 4 ++-- common/predictive-text/index.ts | 8 ++++---- .../unit_tests/headless/top-level-lmlayer.js | 10 +++++----- .../in_browser/cases/worker-dummy-integration.js | 2 +- .../in_browser/cases/worker-wordlist-integration.js | 2 +- web/history.md | 1 + web/source/text/prediction/modelManager.ts | 12 ++++++------ 8 files changed, 23 insertions(+), 21 deletions(-) diff --git a/common/predictive-text/docs/worker-communication-protocol.md b/common/predictive-text/docs/worker-communication-protocol.md index f5422830f6..34a917808b 100644 --- a/common/predictive-text/docs/worker-communication-protocol.md +++ b/common/predictive-text/docs/worker-communication-protocol.md @@ -118,7 +118,7 @@ Currently there are four message types: Message | Direction | Parameters | Expected reply | Uses token --------------|--------------------|---------------------|---------------------|--------------- `config` | LMLayer -> worker | capabilities | No | No -`load` | keyboard → LMLayer | capabilities, model | Yes — `ready` | No +`load` | keyboard → LMLayer | model | Yes — `ready` | No `unload` | keyboard → LMLayer | none | No | No `ready` | LMLayer → keyboard | configuration | No | No `predict` | keyboard → LMLayer | transform, context | Yes — `suggestions` | Yes @@ -178,7 +178,8 @@ to the LMLayer prior to sending `load`. The keyboard **SHOULD NOT** send another message to the keyboard until it receives `ready` message from the LMLayer before sending another message. -The LMLayer needs to know the platform's abilities and restrictions (capabilities), as well as which concrete language model to instantiate. These properties are passed as `capabilities` and `model`, respectively. +The LMLayer needs to know which concrete language model to instantiate. This is provided +by the file at the path specified by the `model` string parameter. ```typescript interface LoadMessage { diff --git a/common/predictive-text/index.d.ts b/common/predictive-text/index.d.ts index 659f2729ca..e638be65d1 100644 --- a/common/predictive-text/index.d.ts +++ b/common/predictive-text/index.d.ts @@ -24,12 +24,12 @@ declare namespace com.keyman.text.prediction { * Initializes the LMLayer worker with the keyboard/platform's capabilities, * as well as a description of the model required. */ - activateModel(model: string): Promise; + loadModel(model: string): Promise; /** * Prepares the LMLayer for reinitialization with a different model/capability set. */ - deactivateModel(); + unloadModel(); predict(transform: Transform, context: Context): Promise; diff --git a/common/predictive-text/index.ts b/common/predictive-text/index.ts index 94115567b6..5e9f9779e6 100644 --- a/common/predictive-text/index.ts +++ b/common/predictive-text/index.ts @@ -34,10 +34,10 @@ * Since the Worker runs in a different thread, the public methods of this class are * asynchronous. Methods of note include: * - * - #activateModel() -- loads a specified model file + * - #loadModel() -- loads a specified model file * - #predict() -- ask the LMLayer to offer suggestions (predictions or corrections) for * the input event - * - #deactivateModel() -- unloads the LMLayer's currently loaded model, preparing it to + * - #unloadModel() -- unloads the LMLayer's currently loaded model, preparing it to * receive (load) a new model * * The top-level LMLayer will automatically starts up its own Web Worker. @@ -88,7 +88,7 @@ namespace com.keyman.text.prediction { /** * Initializes the LMLayer worker with a path to the desired model file. */ - activateModel(modelFilePath: string): Promise { + loadModel(modelFilePath: string): Promise { return new Promise((resolve, _reject) => { this._worker.postMessage({ message: 'load', @@ -105,7 +105,7 @@ namespace com.keyman.text.prediction { * Unloads the previously-active model from memory, resetting the LMLayer to prep * for transition to use of a new model. */ - public deactivateModel() { + public unloadModel() { this._worker.postMessage({ message: 'unload' }); diff --git a/common/predictive-text/unit_tests/headless/top-level-lmlayer.js b/common/predictive-text/unit_tests/headless/top-level-lmlayer.js index bb9d9e0c58..f340044a19 100644 --- a/common/predictive-text/unit_tests/headless/top-level-lmlayer.js +++ b/common/predictive-text/unit_tests/headless/top-level-lmlayer.js @@ -24,12 +24,12 @@ describe('LMLayer', function() { }); }); - describe('#activateModel()', function () { + describe('#loadModel()', function () { it('should accept capabilities and model description', function () { let fakeWorker = createFakeWorker(); let lmLayer = new LMLayer(capabilities(), fakeWorker); - lmLayer.activateModel("./unit_tests/in_browser/resources/models/simple-dummy.js"); + lmLayer.loadModel("./unit_tests/in_browser/resources/models/simple-dummy.js"); assert.isFunction(fakeWorker.onmessage, 'LMLayer failed to set a callback!'); }); @@ -37,7 +37,7 @@ describe('LMLayer', function() { it('should send the `load` message to the LMLayer', async function () { let fakeWorker = createFakeWorker(fakePostMessage); let lmLayer = new LMLayer(capabilities(), fakeWorker); - let configuration = await lmLayer.activateModel("./unit_tests/in_browser/resources/models/simple-dummy.js"); + let configuration = await lmLayer.loadModel("./unit_tests/in_browser/resources/models/simple-dummy.js"); assert.propertyVal(fakeWorker.postMessage, 'callCount', 2); // In the "Worker", assert the message looks right and @@ -76,7 +76,7 @@ describe('LMLayer', function() { }); let lmLayer = new LMLayer(capabilities, fakeWorker); - let actualConfiguration = await lmLayer.activateModel( + let actualConfiguration = await lmLayer.loadModel( { maxLeftContextCodeUnits: 32, }, @@ -86,7 +86,7 @@ describe('LMLayer', function() { } ); - // This SHOULD be called by activateModel(). + // This SHOULD be called by loadModel(). assert.deepEqual(actualConfiguration, expectedConfiguration); }) }); diff --git a/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js b/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js index c042b308a7..f5c6e8ce1c 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js +++ b/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js @@ -17,7 +17,7 @@ describe('LMLayer using dummy model', function () { // We're testing many as asynchronous messages in a row. // this would be cleaner using async/await syntax, but // alas some of our browsers don't support it. - return lmLayer.activateModel( + return lmLayer.loadModel( // We need to provide an absolute path since the worker is based within a blob. document.location.protocol + '//' + document.location.host + "/resources/models/simple-dummy.js" ).then(function (actualConfiguration) { diff --git a/common/predictive-text/unit_tests/in_browser/cases/worker-wordlist-integration.js b/common/predictive-text/unit_tests/in_browser/cases/worker-wordlist-integration.js index ff60c79f2b..690cc3804b 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/worker-wordlist-integration.js +++ b/common/predictive-text/unit_tests/in_browser/cases/worker-wordlist-integration.js @@ -13,7 +13,7 @@ describe('LMLayer using the word list model', function () { // We're testing many as asynchronous messages in a row. // this would be cleaner using async/await syntax, but // alas some of our browsers don't support it. - return lmLayer.activateModel( + return lmLayer.loadModel( // We need to provide an absolute path since the worker is based within a blob. document.location.protocol + '//' + document.location.host + "/resources/models/simple-wordlist.js" ).then(function (_actualConfiguration) { diff --git a/web/history.md b/web/history.md index 3c9d239077..eeb61b9a59 100644 --- a/web/history.md +++ b/web/history.md @@ -7,6 +7,7 @@ * Attachment to all supported element types has now been abstracted, splitting keystroke processing from related DOM management. * Centralizes code paths to improve parity between hardware and OSK-based keystrokes. * In both cases above, significant unit testing was facilitated and has been added as a result, further improving code maintainability into the future. +* Began adding support for our common LMLayer interface for predictive modeling. ## 2019-02-25 11.0.220 stable * 11.0 Stable release diff --git a/web/source/text/prediction/modelManager.ts b/web/source/text/prediction/modelManager.ts index 9739e8efda..c622cb5c74 100644 --- a/web/source/text/prediction/modelManager.ts +++ b/web/source/text/prediction/modelManager.ts @@ -47,18 +47,18 @@ namespace com.keyman.text.prediction { keyman['addEventListener']('keyboardchange', this.onKeyboardChange.bind(this)); } - private deactivateModel() { - this.lmEngine.deactivateModel(); + private unloadModel() { + this.lmEngine.unloadModel(); this.currentModel = null; } - private activateModel(model: ModelSpec) { + private loadModel(model: ModelSpec) { if(!model) { throw new Error("Null reference not allowed."); } let file = model.path; - this.lmEngine.activateModel(file); + this.lmEngine.loadModel(file); this.currentModel = model; } @@ -68,10 +68,10 @@ namespace com.keyman.text.prediction { let model = this.languageModelMap[lgCode]; if(this.currentModel !== model) { - this.deactivateModel(); + this.unloadModel(); if(model) { - this.activateModel(model); + this.loadModel(model); } } } From d573c550804e010600b2a2fd30a3f6f26ec3212a Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Wed, 6 Mar 2019 23:00:26 -0700 Subject: [PATCH 22/22] Check in build.sh if Graphviz is installed. --- common/predictive-text/docs/build.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/common/predictive-text/docs/build.sh b/common/predictive-text/docs/build.sh index ea6464429e..dc2aec4abd 100755 --- a/common/predictive-text/docs/build.sh +++ b/common/predictive-text/docs/build.sh @@ -1,2 +1,18 @@ #!/bin/sh + +# The diagram is built using the Graphviz suite. +# You can get it with most package managers, e.g., +# +# sudo apt installl graphviz # Ubuntu +# +# brew install graphviz # macOS + +# Check if Graphviz/dot is installed +if ! hash dot ; then + echo "Cannot (re)build state diagram" 1>&2 + echo "Missing the Graphviz suite" 1>&2 + echo "Download at $(tput bold)https://www.graphviz.org/$(tput sgr0)" + exit 1 +fi + dot -Tpng lmlayer-states.dot -o lmlayer-states.png