From 602932b15753ab0a42cff66a7d5aff1ae9c70d31 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Mon, 19 Aug 2019 12:17:08 -0600 Subject: [PATCH 1/7] Move all source files to source/ (matching rest of Keyman conventions). --- developer/js/{ => source}/index.ts | 0 .../lexical-model-compiler/build-trie.ts | 0 .../lexical-model-compiler/lexical-model.ts | 2 +- .../lexical-model-compiler/model-info-file.ts | 0 .../js/{ => source}/lexical-model-compiler/stub.ts | 0 .../{ => source}/package-compiler/kmp-compiler.ts | 0 .../{ => source}/package-compiler/kmp-json-file.ts | 0 .../js/{ => source}/package-compiler/kps-file.ts | 0 developer/js/tsconfig.json | 13 +++---------- 9 files changed, 4 insertions(+), 11 deletions(-) rename developer/js/{ => source}/index.ts (100%) rename developer/js/{ => source}/lexical-model-compiler/build-trie.ts (100%) rename developer/js/{ => source}/lexical-model-compiler/lexical-model.ts (93%) rename developer/js/{ => source}/lexical-model-compiler/model-info-file.ts (100%) rename developer/js/{ => source}/lexical-model-compiler/stub.ts (100%) rename developer/js/{ => source}/package-compiler/kmp-compiler.ts (100%) rename developer/js/{ => source}/package-compiler/kmp-json-file.ts (100%) rename developer/js/{ => source}/package-compiler/kps-file.ts (100%) diff --git a/developer/js/index.ts b/developer/js/source/index.ts similarity index 100% rename from developer/js/index.ts rename to developer/js/source/index.ts diff --git a/developer/js/lexical-model-compiler/build-trie.ts b/developer/js/source/lexical-model-compiler/build-trie.ts similarity index 100% rename from developer/js/lexical-model-compiler/build-trie.ts rename to developer/js/source/lexical-model-compiler/build-trie.ts diff --git a/developer/js/lexical-model-compiler/lexical-model.ts b/developer/js/source/lexical-model-compiler/lexical-model.ts similarity index 93% rename from developer/js/lexical-model-compiler/lexical-model.ts rename to developer/js/source/lexical-model-compiler/lexical-model.ts index 92aa4c7261..d65db4395e 100644 --- a/developer/js/lexical-model-compiler/lexical-model.ts +++ b/developer/js/source/lexical-model-compiler/lexical-model.ts @@ -2,7 +2,7 @@ * Interfaces and constants used by the lexical model compiler. These target * the LMLayer's internal worker code, so we provide those definitions too. */ -/// +/// interface ClassBasedWordBreaker { allowedCharacters?: { initials?: string, medials?: string, finals?: string } | string, diff --git a/developer/js/lexical-model-compiler/model-info-file.ts b/developer/js/source/lexical-model-compiler/model-info-file.ts similarity index 100% rename from developer/js/lexical-model-compiler/model-info-file.ts rename to developer/js/source/lexical-model-compiler/model-info-file.ts diff --git a/developer/js/lexical-model-compiler/stub.ts b/developer/js/source/lexical-model-compiler/stub.ts similarity index 100% rename from developer/js/lexical-model-compiler/stub.ts rename to developer/js/source/lexical-model-compiler/stub.ts diff --git a/developer/js/package-compiler/kmp-compiler.ts b/developer/js/source/package-compiler/kmp-compiler.ts similarity index 100% rename from developer/js/package-compiler/kmp-compiler.ts rename to developer/js/source/package-compiler/kmp-compiler.ts diff --git a/developer/js/package-compiler/kmp-json-file.ts b/developer/js/source/package-compiler/kmp-json-file.ts similarity index 100% rename from developer/js/package-compiler/kmp-json-file.ts rename to developer/js/source/package-compiler/kmp-json-file.ts diff --git a/developer/js/package-compiler/kps-file.ts b/developer/js/source/package-compiler/kps-file.ts similarity index 100% rename from developer/js/package-compiler/kps-file.ts rename to developer/js/source/package-compiler/kps-file.ts diff --git a/developer/js/tsconfig.json b/developer/js/tsconfig.json index 0a05d8ae77..055a956c44 100644 --- a/developer/js/tsconfig.json +++ b/developer/js/tsconfig.json @@ -4,16 +4,9 @@ "module": "commonjs", "outDir": "dist", "sourceMap": true, - "declaration": true, + "declaration": true }, "include": [ - "lexical-model-compiler/**/*", - "package-compiler/**/*", - "index.ts" - ], - "exclude": [ - "node_modules", - "dist", - "tests/**/*.ts" + "source/**/*.ts" ] -} \ No newline at end of file +} From 5e90e955bfa62a642b6ed280acca4e7b87b39fa6 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Mon, 19 Aug 2019 12:25:07 -0600 Subject: [PATCH 2/7] Enable noImplicitReturns. --- developer/js/source/index.ts | 2 ++ developer/js/tsconfig.json | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/developer/js/source/index.ts b/developer/js/source/index.ts index 5ef9ea5c5f..a3d26aadb0 100644 --- a/developer/js/source/index.ts +++ b/developer/js/source/index.ts @@ -182,6 +182,8 @@ export default class LexicalModelCompiler { set_model_metadata('sourcePath', [groupPath, authorPath, bcp47Path].join('/')); fs.writeFileSync(modelInfoFileName, JSON.stringify(model_info, null, 2)); + + return true; }; /** diff --git a/developer/js/tsconfig.json b/developer/js/tsconfig.json index 055a956c44..95daf1fe1c 100644 --- a/developer/js/tsconfig.json +++ b/developer/js/tsconfig.json @@ -1,9 +1,11 @@ { "compilerOptions": { - "target": "es2017", "module": "commonjs", - "outDir": "dist", + "target": "es2017", + "noImplicitReturns": true, + "moduleResolution": "node", "sourceMap": true, + "outDir": "dist", "declaration": true }, "include": [ From 678d853af36fcbfac4876ee25898a7056cf16454 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Mon, 19 Aug 2019 12:29:18 -0600 Subject: [PATCH 3/7] Turn on noUnusedLocals --- developer/js/source/index.ts | 2 -- developer/js/source/lexical-model-compiler/build-trie.ts | 5 ++++- developer/js/tsconfig.json | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/developer/js/source/index.ts b/developer/js/source/index.ts index a3d26aadb0..7b8fab27b4 100644 --- a/developer/js/source/index.ts +++ b/developer/js/source/index.ts @@ -196,8 +196,6 @@ export default class LexicalModelCompiler { * @param sourcePath Where to find auxilary sources files */ generateLexicalModelCode(model_id: string, modelSource: LexicalModelSource, sourcePath: string) { - let oc: LexicalModelCompiled = {id: model_id, format: modelSource.format}; - let sources: string[] = modelSource.sources.map(function(source) { return fs.readFileSync(path.join(sourcePath, source), 'utf8'); }); diff --git a/developer/js/source/lexical-model-compiler/build-trie.ts b/developer/js/source/lexical-model-compiler/build-trie.ts index 14e3a5966c..e73dc70925 100644 --- a/developer/js/source/lexical-model-compiler/build-trie.ts +++ b/developer/js/source/lexical-model-compiler/build-trie.ts @@ -55,7 +55,10 @@ export function parseWordList(contents: string): WordList { if (line.startsWith('#') || line === "") { continue; // skip comments and empty lines } - let [wordform, countText, _comment] = line.split(TAB); + + // The third column is the comment. Always ignored! + let [wordform, countText] = line.split(TAB); + // Clean the word form. // TODO: what happens if we get duplicate forms? wordform = wordform.normalize('NFC').trim(); diff --git a/developer/js/tsconfig.json b/developer/js/tsconfig.json index 95daf1fe1c..4f7428adb6 100644 --- a/developer/js/tsconfig.json +++ b/developer/js/tsconfig.json @@ -3,6 +3,8 @@ "module": "commonjs", "target": "es2017", "noImplicitReturns": true, + "noImplicitThis": true, + "noUnusedLocals": true, "moduleResolution": "node", "sourceMap": true, "outDir": "dist", From ed38a5714f5515fa56af9a244bc9824ebfed9bd6 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Mon, 19 Aug 2019 12:38:19 -0600 Subject: [PATCH 4/7] Add a few more strict checks. --- developer/js/package.json | 2 +- developer/js/tsconfig.json | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/developer/js/package.json b/developer/js/package.json index 21bc00aa8d..07d2dd7599 100644 --- a/developer/js/package.json +++ b/developer/js/package.json @@ -5,7 +5,7 @@ "main": "dist/index.js", "types": "dist/index.d.ts", "scripts": { - "build": "npx tsc -p .", + "build": "tsc", "test": "mocha -r ts-node/register tests/**/test-*.ts" }, "author": "Marc Durdin (SIL)", diff --git a/developer/js/tsconfig.json b/developer/js/tsconfig.json index 4f7428adb6..7d02991cfa 100644 --- a/developer/js/tsconfig.json +++ b/developer/js/tsconfig.json @@ -2,13 +2,14 @@ "compilerOptions": { "module": "commonjs", "target": "es2017", - "noImplicitReturns": true, - "noImplicitThis": true, - "noUnusedLocals": true, "moduleResolution": "node", "sourceMap": true, "outDir": "dist", - "declaration": true + "declaration": true, + "alwaysStrict": true, + "noImplicitReturns": true, + "noImplicitThis": true, + "noUnusedLocals": true }, "include": [ "source/**/*.ts" From b27f9a7a25946c4e4ff0eeabd9aaf00d2b113980 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Mon, 19 Aug 2019 12:48:42 -0600 Subject: [PATCH 5/7] Add minimal README and install deps on build. --- developer/js/README.md | 27 +++++++++++++++++++++++++++ developer/js/build.sh | 6 ++++++ 2 files changed, 33 insertions(+) create mode 100644 developer/js/README.md diff --git a/developer/js/README.md b/developer/js/README.md new file mode 100644 index 0000000000..58737047d3 --- /dev/null +++ b/developer/js/README.md @@ -0,0 +1,27 @@ +Keyman Developer +================ + +This contains the source code of Keyman development tools. + +In order to build [lexical models][], these tools must be built and +compiled. + + +How to build +------------ + + ./build.sh + + +How to run the tests +-------------------- + + ./build.sh -test + + +How to install +-------------- + + npm install + npm run build + npm link diff --git a/developer/js/build.sh b/developer/js/build.sh index 0308dc00e3..e02617ac27 100755 --- a/developer/js/build.sh +++ b/developer/js/build.sh @@ -27,6 +27,7 @@ display_usage ( ) { ################################ Main script ################################ run_tests=0 +install_dependencies=1 # Process command-line arguments while [[ $# -gt 0 ]] ; do @@ -38,6 +39,7 @@ while [[ $# -gt 0 ]] ; do ;; -test) run_tests=1 + install_dependencies=0 ;; *) echo "$0: invalid option: $key" @@ -51,6 +53,10 @@ done type npm >/dev/null ||\ fail "Build environment setup error detected! Please ensure Node.js is installed!" +if (( install_dependencies )) ; then + npm install || fail "Could not download dependencies." +fi + build || fail "Compilation failed." echo "Typescript compilation successful." From ca18b318c948f56e449582b47f8b19908178f604 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Mon, 19 Aug 2019 13:53:49 -0600 Subject: [PATCH 6/7] Export ModelSourceError when... the model source code is invalid! --- developer/js/source/index.ts | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/developer/js/source/index.ts b/developer/js/source/index.ts index 7b8fab27b4..845c36f92b 100644 --- a/developer/js/source/index.ts +++ b/developer/js/source/index.ts @@ -49,7 +49,7 @@ export default class LexicalModelCompiler { * "authorEmail": "nobody@example.com", * "description": "Example wordlist model" * } - * + * * For full documentation, see: * https://help.keyman.com/developer/cloud/model_info/1.0/ */ @@ -90,7 +90,16 @@ export default class LexicalModelCompiler { // Build the compiled lexical model // - let func = this.generateLexicalModelCode(model_id, modelSource, sourcePath); + let func: string; + try { + func = this.generateLexicalModelCode(model_id, modelSource, sourcePath); + } catch (e) { + if (e instanceof ModelSourceError) { + this.logError(e.message); + return false; + } + throw e; + } // // Save full model to build folder as Javascript for use in KeymanWeb @@ -151,10 +160,10 @@ export default class LexicalModelCompiler { // we strip the mailto: from the .kps file for the .model_info set_model_metadata('authorEmail', kmpJsonData.info.author.url.match(/^(mailto\:)?(.+)$/)[2], false); - + // extract the language identifiers from the language metadata // arrays for each of the lexical models in the kmp.json file, - // and merge into a single array of identifiers in the + // and merge into a single array of identifiers in the // .model_info file. model_info.languages = model_info.languages || kmpJsonData.lexicalModels.reduce((a, e) => [].concat(a, e.languages.map((f) => f.id)), []); @@ -163,7 +172,7 @@ export default class LexicalModelCompiler { set_model_metadata('packageFilename', kmpFileName); // Always overwrite with actual file size - model_info.packageFileSize = fs.statSync(model_info.packageFilename).size; + model_info.packageFileSize = fs.statSync(model_info.packageFilename).size; set_model_metadata('jsFilename', modelFileName); @@ -171,12 +180,12 @@ export default class LexicalModelCompiler { model_info.jsFileSize = fs.statSync(model_info.jsFilename).size; // Always overwrite source data - model_info.packageIncludes = kmpJsonData.files.filter((e) => !!e.name.match(/.[ot]tf$/i)).length ? ['fonts'] : []; + model_info.packageIncludes = kmpJsonData.files.filter((e) => !!e.name.match(/.[ot]tf$/i)).length ? ['fonts'] : []; set_model_metadata('version', kmpJsonData.info.version.description); // The minimum Keyman version detected in the package file may be manually set higher by the developer - set_model_metadata('minKeymanVersion', minKeymanVersion, false); + set_model_metadata('minKeymanVersion', minKeymanVersion, false); //TODO: model_info.helpLink = model_info.helpLink || ... if source/help/id.php exists? set_model_metadata('sourcePath', [groupPath, authorPath, bcp47Path].join('/')); @@ -190,7 +199,7 @@ export default class LexicalModelCompiler { * Returns the generated code for the model that will ultimately be loaded by * the LMLayer worker. This code contains all model parameters, and specifies * word breakers and auxilary functions that may be required. - * + * * @param model_id The model ID. TODO: not sure if this is actually required! * @param modelSource A specification of the model to compile * @param sourcePath Where to find auxilary sources files @@ -231,8 +240,7 @@ export default class LexicalModelCompiler { func += `LMLayerWorker.loadModel(new ${modelSource.rootClass}());\n`; break; case "fst-foma-1.0": - this.logError('Unimplemented model format ' + modelSource.format); - return false; + throw new ModelSourceError(`Unimplemented model format: ${modelSource.format}`); case "trie-1.0": func += `LMLayerWorker.loadModel(new models.TrieModel(${ createTrieDataStructure(sources, modelSource.searchTermToKey) @@ -249,8 +257,7 @@ export default class LexicalModelCompiler { func += `}));\n`; break; default: - this.logError('Unknown model format '+modelSource.format); - return false; + throw new ModelSourceError(`Unknown model format: ${modelSource.format}`); } // @@ -269,7 +276,10 @@ export default class LexicalModelCompiler { ); }; - logError(s) { + logError(s: string) { console.error(require('chalk').red(s)); }; }; + +export class ModelSourceError extends Error { +} From 5f696ac6dad0337976f5f1954a4943684f7bafab Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Mon, 19 Aug 2019 13:56:16 -0600 Subject: [PATCH 7/7] Update developer README. --- developer/README.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/developer/README.md b/developer/README.md index 945a576232..dcc4db845f 100644 --- a/developer/README.md +++ b/developer/README.md @@ -10,7 +10,6 @@ platforms. # Lexical Model Compiler - # Package Compiler The package compiler is broadly compatible with the kmcomp .kps @@ -21,7 +20,4 @@ tested or supported. It is likely in the future that the kmcomp # Transition steps -1. Move both of the above compilers into the keyman repo -2. Split .kps and model compilers into separate paths -3. Update lexical-models repo to pull these compiler(s) -4. Refactor compiler code to address https://github.com/keymanapp/lexical-models/issues/28 and https://github.com/keymanapp/lexical-models/issues/29 \ No newline at end of file +1. Split .kps and model compilers into separate paths