diff --git a/developer/README.md b/developer/README.md index 945a576232..dcc4db845f 100644 --- a/developer/README.md +++ b/developer/README.md @@ -10,7 +10,6 @@ platforms. # Lexical Model Compiler - # Package Compiler The package compiler is broadly compatible with the kmcomp .kps @@ -21,7 +20,4 @@ tested or supported. It is likely in the future that the kmcomp # Transition steps -1. Move both of the above compilers into the keyman repo -2. Split .kps and model compilers into separate paths -3. Update lexical-models repo to pull these compiler(s) -4. Refactor compiler code to address https://github.com/keymanapp/lexical-models/issues/28 and https://github.com/keymanapp/lexical-models/issues/29 \ No newline at end of file +1. Split .kps and model compilers into separate paths diff --git a/developer/js/README.md b/developer/js/README.md new file mode 100644 index 0000000000..58737047d3 --- /dev/null +++ b/developer/js/README.md @@ -0,0 +1,27 @@ +Keyman Developer +================ + +This contains the source code of Keyman development tools. + +In order to build [lexical models][], these tools must be built and +compiled. + + +How to build +------------ + + ./build.sh + + +How to run the tests +-------------------- + + ./build.sh -test + + +How to install +-------------- + + npm install + npm run build + npm link diff --git a/developer/js/build.sh b/developer/js/build.sh index 0308dc00e3..e02617ac27 100755 --- a/developer/js/build.sh +++ b/developer/js/build.sh @@ -27,6 +27,7 @@ display_usage ( ) { ################################ Main script ################################ run_tests=0 +install_dependencies=1 # Process command-line arguments while [[ $# -gt 0 ]] ; do @@ -38,6 +39,7 @@ while [[ $# -gt 0 ]] ; do ;; -test) run_tests=1 + install_dependencies=0 ;; *) echo "$0: invalid option: $key" @@ -51,6 +53,10 @@ done type npm >/dev/null ||\ fail "Build environment setup error detected! Please ensure Node.js is installed!" +if (( install_dependencies )) ; then + npm install || fail "Could not download dependencies." +fi + build || fail "Compilation failed." echo "Typescript compilation successful." diff --git a/developer/js/package.json b/developer/js/package.json index 21bc00aa8d..07d2dd7599 100644 --- a/developer/js/package.json +++ b/developer/js/package.json @@ -5,7 +5,7 @@ "main": "dist/index.js", "types": "dist/index.d.ts", "scripts": { - "build": "npx tsc -p .", + "build": "tsc", "test": "mocha -r ts-node/register tests/**/test-*.ts" }, "author": "Marc Durdin (SIL)", diff --git a/developer/js/index.ts b/developer/js/source/index.ts similarity index 93% rename from developer/js/index.ts rename to developer/js/source/index.ts index 5ef9ea5c5f..845c36f92b 100644 --- a/developer/js/index.ts +++ b/developer/js/source/index.ts @@ -49,7 +49,7 @@ export default class LexicalModelCompiler { * "authorEmail": "nobody@example.com", * "description": "Example wordlist model" * } - * + * * For full documentation, see: * https://help.keyman.com/developer/cloud/model_info/1.0/ */ @@ -90,7 +90,16 @@ export default class LexicalModelCompiler { // Build the compiled lexical model // - let func = this.generateLexicalModelCode(model_id, modelSource, sourcePath); + let func: string; + try { + func = this.generateLexicalModelCode(model_id, modelSource, sourcePath); + } catch (e) { + if (e instanceof ModelSourceError) { + this.logError(e.message); + return false; + } + throw e; + } // // Save full model to build folder as Javascript for use in KeymanWeb @@ -151,10 +160,10 @@ export default class LexicalModelCompiler { // we strip the mailto: from the .kps file for the .model_info set_model_metadata('authorEmail', kmpJsonData.info.author.url.match(/^(mailto\:)?(.+)$/)[2], false); - + // extract the language identifiers from the language metadata // arrays for each of the lexical models in the kmp.json file, - // and merge into a single array of identifiers in the + // and merge into a single array of identifiers in the // .model_info file. model_info.languages = model_info.languages || kmpJsonData.lexicalModels.reduce((a, e) => [].concat(a, e.languages.map((f) => f.id)), []); @@ -163,7 +172,7 @@ export default class LexicalModelCompiler { set_model_metadata('packageFilename', kmpFileName); // Always overwrite with actual file size - model_info.packageFileSize = fs.statSync(model_info.packageFilename).size; + model_info.packageFileSize = fs.statSync(model_info.packageFilename).size; set_model_metadata('jsFilename', modelFileName); @@ -171,31 +180,31 @@ export default class LexicalModelCompiler { model_info.jsFileSize = fs.statSync(model_info.jsFilename).size; // Always overwrite source data - model_info.packageIncludes = kmpJsonData.files.filter((e) => !!e.name.match(/.[ot]tf$/i)).length ? ['fonts'] : []; + model_info.packageIncludes = kmpJsonData.files.filter((e) => !!e.name.match(/.[ot]tf$/i)).length ? ['fonts'] : []; set_model_metadata('version', kmpJsonData.info.version.description); // The minimum Keyman version detected in the package file may be manually set higher by the developer - set_model_metadata('minKeymanVersion', minKeymanVersion, false); + set_model_metadata('minKeymanVersion', minKeymanVersion, false); //TODO: model_info.helpLink = model_info.helpLink || ... if source/help/id.php exists? set_model_metadata('sourcePath', [groupPath, authorPath, bcp47Path].join('/')); fs.writeFileSync(modelInfoFileName, JSON.stringify(model_info, null, 2)); + + return true; }; /** * Returns the generated code for the model that will ultimately be loaded by * the LMLayer worker. This code contains all model parameters, and specifies * word breakers and auxilary functions that may be required. - * + * * @param model_id The model ID. TODO: not sure if this is actually required! * @param modelSource A specification of the model to compile * @param sourcePath Where to find auxilary sources files */ generateLexicalModelCode(model_id: string, modelSource: LexicalModelSource, sourcePath: string) { - let oc: LexicalModelCompiled = {id: model_id, format: modelSource.format}; - let sources: string[] = modelSource.sources.map(function(source) { return fs.readFileSync(path.join(sourcePath, source), 'utf8'); }); @@ -231,8 +240,7 @@ export default class LexicalModelCompiler { func += `LMLayerWorker.loadModel(new ${modelSource.rootClass}());\n`; break; case "fst-foma-1.0": - this.logError('Unimplemented model format ' + modelSource.format); - return false; + throw new ModelSourceError(`Unimplemented model format: ${modelSource.format}`); case "trie-1.0": func += `LMLayerWorker.loadModel(new models.TrieModel(${ createTrieDataStructure(sources, modelSource.searchTermToKey) @@ -249,8 +257,7 @@ export default class LexicalModelCompiler { func += `}));\n`; break; default: - this.logError('Unknown model format '+modelSource.format); - return false; + throw new ModelSourceError(`Unknown model format: ${modelSource.format}`); } // @@ -269,7 +276,10 @@ export default class LexicalModelCompiler { ); }; - logError(s) { + logError(s: string) { console.error(require('chalk').red(s)); }; }; + +export class ModelSourceError extends Error { +} diff --git a/developer/js/lexical-model-compiler/build-trie.ts b/developer/js/source/lexical-model-compiler/build-trie.ts similarity index 99% rename from developer/js/lexical-model-compiler/build-trie.ts rename to developer/js/source/lexical-model-compiler/build-trie.ts index 14e3a5966c..e73dc70925 100644 --- a/developer/js/lexical-model-compiler/build-trie.ts +++ b/developer/js/source/lexical-model-compiler/build-trie.ts @@ -55,7 +55,10 @@ export function parseWordList(contents: string): WordList { if (line.startsWith('#') || line === "") { continue; // skip comments and empty lines } - let [wordform, countText, _comment] = line.split(TAB); + + // The third column is the comment. Always ignored! + let [wordform, countText] = line.split(TAB); + // Clean the word form. // TODO: what happens if we get duplicate forms? wordform = wordform.normalize('NFC').trim(); diff --git a/developer/js/lexical-model-compiler/lexical-model.ts b/developer/js/source/lexical-model-compiler/lexical-model.ts similarity index 93% rename from developer/js/lexical-model-compiler/lexical-model.ts rename to developer/js/source/lexical-model-compiler/lexical-model.ts index 92aa4c7261..d65db4395e 100644 --- a/developer/js/lexical-model-compiler/lexical-model.ts +++ b/developer/js/source/lexical-model-compiler/lexical-model.ts @@ -2,7 +2,7 @@ * Interfaces and constants used by the lexical model compiler. These target * the LMLayer's internal worker code, so we provide those definitions too. */ -/// +/// interface ClassBasedWordBreaker { allowedCharacters?: { initials?: string, medials?: string, finals?: string } | string, diff --git a/developer/js/lexical-model-compiler/model-info-file.ts b/developer/js/source/lexical-model-compiler/model-info-file.ts similarity index 100% rename from developer/js/lexical-model-compiler/model-info-file.ts rename to developer/js/source/lexical-model-compiler/model-info-file.ts diff --git a/developer/js/lexical-model-compiler/stub.ts b/developer/js/source/lexical-model-compiler/stub.ts similarity index 100% rename from developer/js/lexical-model-compiler/stub.ts rename to developer/js/source/lexical-model-compiler/stub.ts diff --git a/developer/js/package-compiler/kmp-compiler.ts b/developer/js/source/package-compiler/kmp-compiler.ts similarity index 100% rename from developer/js/package-compiler/kmp-compiler.ts rename to developer/js/source/package-compiler/kmp-compiler.ts diff --git a/developer/js/package-compiler/kmp-json-file.ts b/developer/js/source/package-compiler/kmp-json-file.ts similarity index 100% rename from developer/js/package-compiler/kmp-json-file.ts rename to developer/js/source/package-compiler/kmp-json-file.ts diff --git a/developer/js/package-compiler/kps-file.ts b/developer/js/source/package-compiler/kps-file.ts similarity index 100% rename from developer/js/package-compiler/kps-file.ts rename to developer/js/source/package-compiler/kps-file.ts diff --git a/developer/js/tsconfig.json b/developer/js/tsconfig.json index 0a05d8ae77..7d02991cfa 100644 --- a/developer/js/tsconfig.json +++ b/developer/js/tsconfig.json @@ -1,19 +1,17 @@ { "compilerOptions": { - "target": "es2017", "module": "commonjs", - "outDir": "dist", + "target": "es2017", + "moduleResolution": "node", "sourceMap": true, + "outDir": "dist", "declaration": true, + "alwaysStrict": true, + "noImplicitReturns": true, + "noImplicitThis": true, + "noUnusedLocals": true }, "include": [ - "lexical-model-compiler/**/*", - "package-compiler/**/*", - "index.ts" - ], - "exclude": [ - "node_modules", - "dist", - "tests/**/*.ts" + "source/**/*.ts" ] -} \ No newline at end of file +}