diff --git a/common/core/web/input-processor/package-lock.json b/common/core/web/input-processor/package-lock.json index 96e255d10f..10eeaa2f9a 100644 --- a/common/core/web/input-processor/package-lock.json +++ b/common/core/web/input-processor/package-lock.json @@ -229,6 +229,12 @@ } } }, + "mocha-teamcity-reporter": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mocha-teamcity-reporter/-/mocha-teamcity-reporter-3.0.0.tgz", + "integrity": "sha512-FyGgmtFfW2nDwEZU3mrjQShAAK/zhGivwY4HCsqoDoyeS8vV8HGdq1Dn2P+SFaIoCeXTQ0Z+5xVRyikYaKrW5w==", + "dev": true + }, "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", diff --git a/common/core/web/input-processor/package.json b/common/core/web/input-processor/package.json index f774ca3389..cd0dc7ecb1 100644 --- a/common/core/web/input-processor/package.json +++ b/common/core/web/input-processor/package.json @@ -18,15 +18,20 @@ }, "homepage": "https://github.com/keymanapp/keyman#readme", "devDependencies": { + "@keymanapp/lexical-model-compiler": "^14.0.183", "@keymanapp/resources-gosh": "^14.0.183", + "@types/node": "^11.9.4", "chai": "^4.2.0", "mocha": "^5.2.0", + "mocha-teamcity-reporter": "^3.0.0", + "ts-node": "^8.0.2", "typescript": "^3.8.3" }, "scripts": { "lerna": "cd ../ && npm run lerna --", "tsc": "tsc", - "test": "gosh ./unit_tests/test.sh" + "test": "gosh ./test.sh", + "mocha": "mocha" }, "dependencies": { "@keymanapp/keyboard-processor": "^14.0.183", @@ -34,8 +39,6 @@ "@keymanapp/models-types": "^14.0.183", "@keymanapp/web-environment": "^14.0.183", "@keymanapp/web-utils": "^14.0.183", - "@types/node": "^11.9.4", - "eventemitter3": "^4.0.0", - "ts-node": "^8.0.2" + "eventemitter3": "^4.0.0" } } diff --git a/common/core/web/input-processor/src/text/inputProcessor.ts b/common/core/web/input-processor/src/text/inputProcessor.ts index 20eadd1b24..c4adc7a146 100644 --- a/common/core/web/input-processor/src/text/inputProcessor.ts +++ b/common/core/web/input-processor/src/text/inputProcessor.ts @@ -188,4 +188,15 @@ namespace com.keyman.text { this.languageProcessor.invalidateContext(); } } -} \ No newline at end of file +} + +(function () { + let ns = com.keyman.text; + + // Let the InputProcessor be available both in the browser and in Node. + if (typeof module !== 'undefined' && typeof module.exports !== 'undefined') { + module.exports = ns.InputProcessor; + //@ts-ignore + ns.InputProcessor.com = com; // Export the root namespace so that all InputProcessor classes are accessible by unit tests. + } +}()); \ No newline at end of file diff --git a/common/core/web/input-processor/src/text/prediction/languageProcessor.ts b/common/core/web/input-processor/src/text/prediction/languageProcessor.ts index d61693d275..0645788e22 100644 --- a/common/core/web/input-processor/src/text/prediction/languageProcessor.ts +++ b/common/core/web/input-processor/src/text/prediction/languageProcessor.ts @@ -17,9 +17,15 @@ namespace com.keyman.text.prediction { languages: string[]; /** - * The path/URL to the file that defines the model. + * The path/URL to the file that defines the model. If both `path` and `raw` are specified, + * `path` takes precedence. */ path: string; + + /** + * The raw JS script defining the model. Only used if `path` is not specified. + */ + code: string; } /** @@ -127,11 +133,12 @@ namespace com.keyman.text.prediction { throw new Error("Null reference not allowed."); } - let file = model.path; + let specType: 'file'|'raw' = model.path ? 'file' : 'raw'; + let source = specType == 'file' ? model.path : model.code; let lp = this; // We should wait until the model is successfully loaded before setting our state values. - return this.lmEngine.loadModel(file).then(function(config: Configuration) { + return this.lmEngine.loadModel(source, specType).then(function(config: Configuration) { lp.currentModel = model; lp.configuration = config; @@ -359,7 +366,7 @@ namespace com.keyman.text.prediction { this._mayPredict = false; return false; } - return this.activeModel && this._mayPredict; + return (this.activeModel || false) && this._mayPredict; } public canEnable(): boolean { diff --git a/common/core/web/input-processor/test.sh b/common/core/web/input-processor/test.sh new file mode 100755 index 0000000000..d24fb3df27 --- /dev/null +++ b/common/core/web/input-processor/test.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +# We should work within the script's directory, not the one we were called in. +cd $(dirname "$BASH_SOURCE") + +WORKING_DIRECTORY=`pwd` + +# Include useful testing resource functions +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BASH_SOURCE[0]}")" +. "$(dirname "$THIS_SCRIPT")/../../../../resources/build/build-utils.sh" +. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" +## END STANDARD BUILD SCRIPT INCLUDE + +# A simple utility script to facilitate unit-testing for the LM Layer. +# It's rigged to be callable by NPM to facilitate testing during development when in other folders. + +display_usage ( ) { + echo "test.sh [-skip-package-install|-S] [-CI] [ -? | -h | -help]" + echo " -CI to perform continuous-integration friendly tests and reporting formatted for TeamCity" + echo " -? | -h | -help to display this help information" + echo " -skip-package-install to bypass refreshing dependencies. Useful when called by scripts that pre-fetch" + echo " (or -S)" + echo "" + exit 0 +} + +# Defaults +FLAGS= +CI_REPORTING=0 +FETCH_DEPS=true + +# Parse args +while [[ $# -gt 0 ]] ; do + key="$1" + case $key in + -h|-help|-\?) + display_usage + exit + ;; + -CI) + CI_REPORTING=1 + ;; + -skip-package-install|-S) + FETCH_DEPS=false + ;; + esac + shift # past argument +done + +if [ $FETCH_DEPS = true ]; then + verify_npm_setup +fi + +# Ensures that the lexical model compiler has been built locally. +echo "${TERM_HEADING}Preparing Lexical Model Compiler for test use${NORMAL}" +pushd $WORKING_DIRECTORY/node_modules/@keymanapp/lexical-model-compiler +npm run build +popd + +test-headless ( ) { + if (( CI_REPORTING )); then + FLAGS="$FLAGS --reporter mocha-teamcity-reporter" + fi + + npm run mocha -- --recursive $FLAGS ./tests/cases/ +} + +if [ $FETCH_DEPS = true ]; then + # First, run tests on the keyboard processor. + pushd $WORKING_DIRECTORY/node_modules/@keymanapp/keyboard-processor + ./test.sh -skip-package-install || fail "Tests failed by dependencies; aborting integration tests." + popd +fi + +# Now we run our local tests. +echo "${TERM_HEADING}Running Input Processor test suite${NORMAL}" +test-headless || fail "Input Processor tests failed!" diff --git a/common/core/web/input-processor/tests/cases/inputProcessor.js b/common/core/web/input-processor/tests/cases/inputProcessor.js new file mode 100644 index 0000000000..d054d4a620 --- /dev/null +++ b/common/core/web/input-processor/tests/cases/inputProcessor.js @@ -0,0 +1,45 @@ +var assert = require('chai').assert; +var fs = require("fs"); +var vm = require("vm"); + +let InputProcessor = require('../../dist'); + +// Required initialization setup. +global.com = InputProcessor.com; // exports all keyboard-processor namespacing. +global.keyman = {}; // So that keyboard-based checks against the global `keyman` succeed. + // 10.0+ dependent keyboards, like khmer_angkor, will otherwise fail to load. + +// Initialize supplementary plane string extensions +String.kmwEnableSupplementaryPlane(false); + +// Test the KeyboardProcessor interface. +describe('InputProcessor', function() { + describe('[[constructor]]', function () { + it('should initialize without errors', function () { + let core = new InputProcessor(); + assert.isNotNull(core); + }); + + it('has expected default values after initialization', function () { + let core = new InputProcessor(); + + assert.isOk(core.keyboardProcessor); + assert.isOk(core.languageProcessor); + assert.isOk(core.keyboardInterface); + assert.isUndefined(core.activeKeyboard); // No keyboard should be loaded yet. + assert.isUndefined(core.activeModel); // Same for the model. + + // These checks are lifted from the keyboard-processor init checks found in + // common/core/web/keyboard-processor/tests/cases/basic-init.js. + assert.equal('us', core.keyboardProcessor.baseLayout, 'KeyboardProcessor has unexpected base layout') + assert.isNotNull(global.KeymanWeb, 'KeymanWeb global was not automatically installed'); + assert.equal('default', core.keyboardProcessor.layerId, 'Default layer is not set to "default"'); + assert.isUndefined(core.keyboardProcessor.activeKeyboard, 'Initialized with already-active keyboard'); + + // Lifted from languageProcessor.js - the core should not be changing these with its init. + assert.isUndefined(core.languageProcessor.activeModel); + assert.isFalse(core.languageProcessor.isActive); + assert.isTrue(core.languageProcessor.mayPredict); + }); + }); +}); \ No newline at end of file diff --git a/common/core/web/input-processor/tests/cases/languageProcessor.js b/common/core/web/input-processor/tests/cases/languageProcessor.js new file mode 100644 index 0000000000..0fdfa5aa66 --- /dev/null +++ b/common/core/web/input-processor/tests/cases/languageProcessor.js @@ -0,0 +1,106 @@ +var assert = require('chai').assert; +var fs = require("fs"); +var vm = require("vm"); + +/* + * Unit tests for the Dummy prediction model. + */ + +var LexicalModelCompiler = require('@keymanapp/lexical-model-compiler/dist/lexical-model-compiler/lexical-model-compiler').default; +var path = require('path'); + +let InputProcessor = require('../../dist'); + +// Required initialization setup. +global.com = InputProcessor.com; // exports all keyboard-processor namespacing. +global.keyman = {}; // So that keyboard-based checks against the global `keyman` succeed. + // 10.0+ dependent keyboards, like khmer_angkor, will otherwise fail to load. + +// Initialize supplementary plane string extensions +String.kmwEnableSupplementaryPlane(false); + +let LanguageProcessor = com.keyman.text.prediction.LanguageProcessor; + +// Test the KeyboardProcessor interface. +describe('LanguageProcessor', function() { + describe('[[constructor]]', function () { + it('should initialize without errors', function () { + let lp = new LanguageProcessor(); + assert.isNotNull(lp); + }); + + it('has expected default values after initialization', function () { + let languageProcessor = new LanguageProcessor(); + + // These checks are lifted from the keyboard-processor init checks found in + // common/core/web/keyboard-processor/tests/cases/basic-init.js. + assert.isUndefined(languageProcessor.lmEngine); + assert.isUndefined(languageProcessor.activeModel); + assert.isFalse(languageProcessor.isActive); + assert.isTrue(languageProcessor.mayPredict); + + // Some aspects of initialization must wait until after construction and overall + // load of the core. See /web/source/kmwbase.ts, in the final IIFE. + languageProcessor.init(); + assert.isOk(languageProcessor.lmEngine); + }); + }); + + describe('.predict', function() { + let compiler = new LexicalModelCompiler(); + const MODEL_ID = 'example.qaa.trivial'; + const PATH = path.join(__dirname, '../../node_modules/@keymanapp/lexical-model-compiler/tests/fixtures', MODEL_ID); + + describe('using angle brackets for quotes', function() { + let modelCode = compiler.generateLexicalModelCode(MODEL_ID, { + format: 'trie-1.0', + sources: ['wordlist.tsv'], + punctuation: { + quotesForKeepSuggestion: { open: `«`, close: `»`}, + insertAfterWord: " " , // OGHAM SPACE MARK + } + }, PATH); + + let modelSpec = { + id: MODEL_ID, + languages: ['en'], + code: modelCode + }; + + it("successfully loads the model", function(done) { + let languageProcessor = new LanguageProcessor(); + languageProcessor.init(); + + languageProcessor.loadModel(modelSpec).then(function() { + assert.isOk(languageProcessor.activeModel); // is only set after a successful load. + done(); + }, function(reason) { + assert.fail("Model did not load correctly: " + reason); + }); + }); + + it("generates the expected prediction set", function(done) { + let languageProcessor = new LanguageProcessor(); + languageProcessor.init(); + + let contextSource = new com.keyman.text.Mock("li", 2); + let transcription = contextSource.buildTranscriptionFrom(contextSource, null, null); + + languageProcessor.loadModel(modelSpec).then(function() { + languageProcessor.predict(transcription).then(function(suggestions) { + assert.isOk(suggestions); + assert.equal(suggestions[0].displayAs, '«li»'); + // Is not actually inserting what is expected at the moment. + //assert.equal(suggestions[0].transform.insert, ' '); + assert.equal(suggestions[1].displayAs, 'like'); + assert.equal(suggestions[1].transform.insert, 'like '); + done(); + }).catch(done); + }).catch(function() { + assert.fail("Unexpected model load failure"); + done(); + }); + }); + }); + }); +}); \ No newline at end of file diff --git a/common/core/web/keyboard-processor/src/text/keyboardProcessor.ts b/common/core/web/keyboard-processor/src/text/keyboardProcessor.ts index 8f30d25101..0d414841db 100644 --- a/common/core/web/keyboard-processor/src/text/keyboardProcessor.ts +++ b/common/core/web/keyboard-processor/src/text/keyboardProcessor.ts @@ -674,7 +674,7 @@ namespace com.keyman.text { (function () { let ns = com.keyman.text; - // Let LMLayer be available both in the browser and in Node. + // Let the Keyboard Processor be available both in the browser and in Node. if (typeof module !== 'undefined' && typeof module.exports !== 'undefined') { module.exports = ns.KeyboardProcessor; //@ts-ignore diff --git a/common/core/web/keyboard-processor/test.sh b/common/core/web/keyboard-processor/test.sh index 56ac3e09f2..61ec245675 100755 --- a/common/core/web/keyboard-processor/test.sh +++ b/common/core/web/keyboard-processor/test.sh @@ -64,5 +64,6 @@ pushd "$KEYMAN_ROOT/common/core/web/tools/recorder/src" popd # Run headless (browserless) tests. +echo "${TERM_HEADING}Running Keyboard Processor test suite${NORMAL}" test-headless || fail "Keyboard Processor tests failed!" diff --git a/common/core/web/tests/resources/json/models/angle-punct-dummy.json b/common/core/web/tests/resources/json/models/angle-punct-dummy.json new file mode 100644 index 0000000000..34b3beeb58 --- /dev/null +++ b/common/core/web/tests/resources/json/models/angle-punct-dummy.json @@ -0,0 +1,5 @@ +{ + "id": "angle-punct-dummy", + "languages": ["en"], + "filename":"resources/models/angle-punct-dummy.js" +} \ No newline at end of file diff --git a/common/core/web/tests/resources/json/models/quote-punct-dummy.json b/common/core/web/tests/resources/json/models/quote-punct-dummy.json new file mode 100644 index 0000000000..c6aa45ff09 --- /dev/null +++ b/common/core/web/tests/resources/json/models/quote-punct-dummy.json @@ -0,0 +1,5 @@ +{ + "id": "quote-punct-dummy", + "languages": ["en"], + "filename":"resources/models/quote-punct-dummy.js" +} \ No newline at end of file diff --git a/common/predictive-text/index.ts b/common/predictive-text/index.ts index 2d5ef53f5b..e8d438d50e 100644 --- a/common/predictive-text/index.ts +++ b/common/predictive-text/index.ts @@ -94,15 +94,25 @@ namespace com.keyman.text.prediction { /** * Initializes the LMLayer worker with a path to the desired model file. */ - loadModel(modelFilePath: string): Promise { + loadModel(modelSource: string, loadType: 'file' | 'raw' = 'file'): Promise { return new Promise((resolve, _reject) => { // Sets up so the promise is resolved in the onMessage() callback, when it receives // the 'ready' message. this._declareLMLayerReady = resolve; + let modelSourceSpec: any = { + type: loadType + }; + + if(loadType == 'file') { + modelSourceSpec.file = modelSource; + } else { + modelSourceSpec.code = modelSource; + } + this._worker.postMessage({ message: 'load', - model: modelFilePath + source: modelSourceSpec }); }); } diff --git a/common/predictive-text/unit_tests/headless/top-level-lmlayer.js b/common/predictive-text/unit_tests/headless/top-level-lmlayer.js index f340044a19..a4645e7e27 100644 --- a/common/predictive-text/unit_tests/headless/top-level-lmlayer.js +++ b/common/predictive-text/unit_tests/headless/top-level-lmlayer.js @@ -49,7 +49,11 @@ describe('LMLayer', function() { } assert.propertyVal(data, 'message', 'load'); - assert.isString(data.model); + assert.property(data, 'source'); + assert.propertyVal(data.source, 'type', 'file'); + assert.notProperty(data.source, 'code'); + assert.property(data.source, 'file'); + assert.isString(data.source.file); callAsynchronously(() => fakeWorker.onmessage({ data: { diff --git a/common/predictive-text/unit_tests/headless/worker-dummy-integration.js b/common/predictive-text/unit_tests/headless/worker-dummy-integration.js index 6f6c34d860..19043e86bf 100644 --- a/common/predictive-text/unit_tests/headless/worker-dummy-integration.js +++ b/common/predictive-text/unit_tests/headless/worker-dummy-integration.js @@ -1,4 +1,5 @@ var assert = require('chai').assert; +var fs = require('fs'); let LMLayer = require('../../build/headless'); /* @@ -11,7 +12,7 @@ let LMLayer = require('../../build/headless'); */ describe('LMLayer using dummy model', function () { describe('Prediction', function () { - it('will predict future suggestions', function () { + it('will predict future suggestions (loaded from file)', function () { var lmLayer = new LMLayer(capabilities()); var stripIDs = function(suggestions) { @@ -49,6 +50,47 @@ describe('LMLayer using dummy model', function () { return Promise.resolve(); }); }); + + it('will predict future suggestions (loaded from raw source)', function () { + var lmLayer = new LMLayer(capabilities()); + + var stripIDs = function(suggestions) { + suggestions.forEach(function(suggestion) { + delete suggestion.id; + }); + }; + + // We're running headlessly, so the path can be relative to the npm root directory. + let modelCode = fs.readFileSync("./unit_tests/in_browser/resources/models/simple-dummy.js").toString(); + + // We're testing many as asynchronous messages in a row. + // this would be cleaner using async/await syntax. + // Not done yet, as this test case is a slightly-edited copy of the in-browser version. + return lmLayer.loadModel( + modelCode, 'raw' + ).then(function (actualConfiguration) { + return Promise.resolve(); + }).then(function () { + return lmLayer.predict(zeroTransform(), emptyContext()); + }).then(function (suggestions) { + stripIDs(suggestions); + assert.deepEqual(suggestions, iGotDistractedByHazel()[0]); + return lmLayer.predict(zeroTransform(), emptyContext()); + }).then(function (suggestions) { + stripIDs(suggestions); + assert.deepEqual(suggestions, iGotDistractedByHazel()[1]); + return lmLayer.predict(zeroTransform(), emptyContext()); + }).then(function (suggestions) { + stripIDs(suggestions); + assert.deepEqual(suggestions, iGotDistractedByHazel()[2]); + return lmLayer.predict(zeroTransform(), emptyContext()); + }).then(function (suggestions) { + stripIDs(suggestions); + assert.deepEqual(suggestions, iGotDistractedByHazel()[3]); + lmLayer.shutdown(); + return Promise.resolve(); + }); + }); }); describe('Wordbreaking', function () { diff --git a/common/predictive-text/unit_tests/headless/worker-initialization.js b/common/predictive-text/unit_tests/headless/worker-initialization.js index d5d9b55b1d..bb436ffccc 100644 --- a/common/predictive-text/unit_tests/headless/worker-initialization.js +++ b/common/predictive-text/unit_tests/headless/worker-initialization.js @@ -1,5 +1,6 @@ var assert = require('chai').assert; var sinon = require('sinon'); +var fs = require('fs'); let LMLayerWorker = require('../../build/intermediate'); @@ -25,7 +26,10 @@ describe('LMLayerWorker', function() { // Sending it the `load` message should notify us that it's loaded! worker.onMessage(createMessageEventWithData({ message: 'load', - model: "./unit_tests/in_browser/resources/models/simple-dummy.js" + source: { + type: 'file', + file: "./unit_tests/in_browser/resources/models/simple-dummy.js" + } })); assert(fakePostMessage.calledOnce); }); @@ -69,7 +73,10 @@ describe('LMLayerWorker', function() { // Send a message; we should get something back. worker.onMessage(createMessageEventWithData({ message: 'load', - model: "./unit_tests/in_browser/resources/models/simple-dummy.js" + source: { + type: 'file', + file: "./unit_tests/in_browser/resources/models/simple-dummy.js" + } })); // It called the postMessage() in its global scope. @@ -128,7 +135,7 @@ describe('LMLayerWorker', function() { }, /invalid message/i); }); - it('should send back a "ready" message', function () { + it('should send back a "ready" message when given a file', function () { var fakePostMessage = sinon.fake(); var context = { postMessage: fakePostMessage @@ -140,7 +147,10 @@ describe('LMLayerWorker', function() { worker.onMessage(createMessageEventWithData({ message: 'load', - model: "./unit_tests/in_browser/resources/models/simple-dummy.js" + source: { + type: 'file', + file: "./unit_tests/in_browser/resources/models/simple-dummy.js" + } })); assert(fakePostMessage.calledOnceWith(sinon.match({ @@ -148,6 +158,31 @@ describe('LMLayerWorker', function() { }))); }); + it('should send back a "ready" message when given raw code', function () { + var fakePostMessage = sinon.fake(); + var context = { + postMessage: fakePostMessage + }; + context.importScripts = importScriptsWith(context); + + var worker = LMLayerWorker.install(context); + configWorker(worker); + + let modelCode = fs.readFileSync("./unit_tests/in_browser/resources/models/simple-dummy.js").toString(); + + worker.onMessage(createMessageEventWithData({ + message: 'load', + source: { + type: 'raw', + code: modelCode + } + })); + + assert(fakePostMessage.calledOnceWith(sinon.match({ + message: 'ready' + })), 'mocked callback was not called'); + }); + it('should send back configuration', function () { var fakePostMessage = sinon.fake(); var context = { @@ -162,7 +197,10 @@ describe('LMLayerWorker', function() { var maxCodeUnits = 64; worker.onMessage(createMessageEventWithData({ message: 'load', - model: "./unit_tests/in_browser/resources/models/simple-dummy.js" + source: { + type: 'file', + file: "./unit_tests/in_browser/resources/models/simple-dummy.js" + } })); sinon.assert.calledWithMatch(fakePostMessage, { diff --git a/common/predictive-text/unit_tests/headless/worker-predict.js b/common/predictive-text/unit_tests/headless/worker-predict.js index c9d7eb42d7..4b22f57512 100644 --- a/common/predictive-text/unit_tests/headless/worker-predict.js +++ b/common/predictive-text/unit_tests/headless/worker-predict.js @@ -39,7 +39,10 @@ describe('LMLayerWorker', function () { worker.onMessage(createMessageEventWithData({ message: 'load', - model: "./unit_tests/in_browser/resources/models/simple-dummy.js" + source: { + type: 'file', + file: "./unit_tests/in_browser/resources/models/simple-dummy.js" + } })); sinon.assert.calledWithMatch(fakePostMessage.lastCall, { message: 'ready', diff --git a/common/predictive-text/unit_tests/in_browser/cases/worker.js b/common/predictive-text/unit_tests/in_browser/cases/worker.js index 253155db37..795adc3d06 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/worker.js +++ b/common/predictive-text/unit_tests/in_browser/cases/worker.js @@ -30,7 +30,10 @@ describe('LMLayerWorker', function () { worker.postMessage({ message: 'load', // Since the worker's based in a blob, it's not on the 'same domain'. We need to absolute-path the model file. - model: document.location.protocol + '//' + document.location.host + "/resources/models/simple-dummy.js" + source: { + type: 'file', + file: document.location.protocol + '//' + document.location.host + "/resources/models/simple-dummy.js" + } }); }); }); diff --git a/common/predictive-text/worker/index.ts b/common/predictive-text/worker/index.ts index 158aa488ba..f55e289f06 100644 --- a/common/predictive-text/worker/index.ts +++ b/common/predictive-text/worker/index.ts @@ -80,11 +80,13 @@ class LMLayerWorker { */ private _importScripts: ImportScripts; + private self: any; + private _platformCapabilities: Capabilities; private _hostURL: string; - private _currentModelSource: string; + private _currentModelSource: ModelSourceSpec; constructor(options = { importScripts: null, @@ -130,14 +132,23 @@ class LMLayerWorker { let im = event.data as IncomingMessage; if(im.message == 'load') { let data = im as LoadMessage; - if(data.model == this._currentModelSource) { + let duplicated = false; + if(this._currentModelSource && data.source.type == this._currentModelSource.type) { + if(data.source.type == 'file' && data.source.file == (this._currentModelSource as ModelFile).file) { + duplicated = true; + } else if(data.source.type == 'raw' && data.source.code == (this._currentModelSource as ModelEval).code) { + duplicated = true; + } + } + + if(duplicated) { // Some JS implementations don't allow web workers access to the console. if(typeof console !== 'undefined') { console.warn("Duplicate model load message detected - squashing!"); } return; } else { - this._currentModelSource = data.model; + this._currentModelSource = data.source; } } else if(im.message == 'unload') { this._currentModelSource = null; @@ -256,6 +267,7 @@ class LMLayerWorker { * description and capabilities. */ private transitionToLoadingState() { + let _this = this; this.state = { name: 'modelless', handleMessage: (payload) => { @@ -265,7 +277,20 @@ class LMLayerWorker { } // TODO: validate configuration? - this.loadModelFile(payload.model); + if(payload.source.type == 'file') { + _this.loadModelFile(payload.source.file); + } else { + // Creates a closure capturing all top-level names that the model must be able to reference. + // `eval` runs by scope rules; our virtualized worker needs a special scope for this to work. + // + // Reference: https://stackoverflow.com/a/40108685 + // Note that we don't need `this`, but we do need the namespaces seen below. + let code = payload.source.code; + let evalInContext = function(LMLayerWorker, models, correction, wordBreakers) { + eval(code); + } + evalInContext(_this, models, correction, wordBreakers); + } } }; } @@ -353,6 +378,7 @@ class LMLayerWorker { static install(scope: DedicatedWorkerGlobalScope): LMLayerWorker { let worker = new LMLayerWorker({ postMessage: scope.postMessage, importScripts: scope.importScripts.bind(scope) }); scope.onmessage = worker.onMessage.bind(worker); + worker.self = scope; // Ensures that the worker instance is accessible for loaded model scripts. // Assists unit-testing. diff --git a/common/predictive-text/worker/worker-interfaces.ts b/common/predictive-text/worker/worker-interfaces.ts index ba56438d60..986973faaf 100644 --- a/common/predictive-text/worker/worker-interfaces.ts +++ b/common/predictive-text/worker/worker-interfaces.ts @@ -54,6 +54,27 @@ interface ConfigMessage { capabilities: Capabilities; } +interface ModelFile { + type: 'file'; + + /** + * The model should be loaded from a file via importScripts. + */ + file: string; +} + +interface ModelEval { + type: 'raw'; + + /** + * Rather than loading a file, this specifies the contents that would normally be within + * a .model.ts file. Useful for dynamically-compiled models that occur when testing. + */ + code: string; +} + +type ModelSourceSpec = ModelFile | ModelEval; + /** * The structure of an initialization message. It should include the model (either in * source code or parameter form), as well as the keyboard's capabilities. @@ -64,7 +85,7 @@ interface LoadMessage { /** * The model's compiled JS file. */ - model: string; + source: ModelSourceSpec; } /** diff --git a/web/unit_tests/test.sh b/web/unit_tests/test.sh index 8c9e733e44..c3c72c7dbd 100755 --- a/web/unit_tests/test.sh +++ b/web/unit_tests/test.sh @@ -129,9 +129,8 @@ cd ../tools/recorder # Run our headless tests first. # Since we're using `lerna`, this actually puts us within the projects when run in-repo! -# First: Keyboard Processor tests. -echo "${TERM_HEADING}Running Keyboard Processor test suite${NORMAL}" -pushd $WORKING_DIRECTORY/node_modules/@keymanapp/keyboard-processor +# First: Web-core tests. +pushd $WORKING_DIRECTORY/node_modules/@keymanapp/input-processor ./test.sh $HEADLESS_FLAGS || fail "Tests failed by dependencies; aborting integration tests." # Once done, now we run the integrated (KeymanWeb) tests. popd diff --git a/windows/src/test/unit-tests/Makefile b/windows/src/test/unit-tests/Makefile index 86b9790a59..9fdacbeb68 100644 --- a/windows/src/test/unit-tests/Makefile +++ b/windows/src/test/unit-tests/Makefile @@ -34,6 +34,12 @@ lexical-model-compiler: start /wait ./build.sh -test !endif +# Revert any local changes to the original source; these changes can break +# later sub-builds. + cd $(KEYMAN_ROOT)\developer\js + git restore package.json + git restore package-lock.json + kmcomp-x64-structures: cd $(ROOT)\src\test\unit-tests\group-helper-rsp19902 $(MAKE) $(TARGET)