Merge pull request #1981 from keymanapp/developer-cleanup

[Developer] Reorganize developer/js with a more conventional file structure

developer/README.md

@@ -10,7 +10,6 @@ platforms.
 # Lexical Model Compiler
 
 
-
 # Package Compiler
 
 The package compiler is broadly compatible with the kmcomp .kps
@@ -21,7 +20,4 @@ tested or supported. It is likely in the future that the kmcomp
 
 # Transition steps
 
-1. Move both of the above compilers into the keyman repo
-2. Split .kps and model compilers into separate paths
-3. Update lexical-models repo to pull these compiler(s)
-4. Refactor compiler code to address https://github.com/keymanapp/lexical-models/issues/28 and https://github.com/keymanapp/lexical-models/issues/29
+1. Split .kps and model compilers into separate paths

developer/js/README.md

@@ -0,0 +1,27 @@
+Keyman Developer
+================
+
+This contains the source code of Keyman development tools.
+
+In order to build [lexical models][], these tools must be built and
+compiled.
+
+
+How to build
+------------
+
+    ./build.sh
+
+
+How to run the tests
+--------------------
+
+    ./build.sh -test
+
+
+How to install
+--------------
+
+    npm install
+    npm run build
+    npm link

developer/js/build.sh

@@ -27,6 +27,7 @@ display_usage ( ) {
 ################################ Main script ################################
 
 run_tests=0
+install_dependencies=1
 
 # Process command-line arguments
 while [[ $# -gt 0 ]] ; do
@@ -38,6 +39,7 @@ while [[ $# -gt 0 ]] ; do
       ;;
     -test)
       run_tests=1
+      install_dependencies=0
       ;;
     *)
       echo "$0: invalid option: $key"
@@ -51,6 +53,10 @@ done
 type npm >/dev/null ||\
     fail "Build environment setup error detected!  Please ensure Node.js is installed!"
 
+if (( install_dependencies )) ; then
+  npm install || fail "Could not download dependencies."
+fi
+
 build || fail "Compilation failed."
 echo "Typescript compilation successful."
 

developer/js/package.json

@@ -5,7 +5,7 @@
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {
-    "build": "npx tsc -p .",
+    "build": "tsc",
     "test": "mocha -r ts-node/register tests/**/test-*.ts"
   },
   "author": "Marc Durdin (SIL)",

developer/js/source/index.ts

@@ -49,7 +49,7 @@ export default class LexicalModelCompiler {
      *    "authorEmail": "nobody@example.com",
      *    "description": "Example wordlist model"
      *  }
-     * 
+     *
      * For full documentation, see:
      * https://help.keyman.com/developer/cloud/model_info/1.0/
      */
@@ -90,7 +90,16 @@ export default class LexicalModelCompiler {
     // Build the compiled lexical model
     //
 
-    let func = this.generateLexicalModelCode(model_id, modelSource, sourcePath);
+    let func: string;
+    try {
+      func = this.generateLexicalModelCode(model_id, modelSource, sourcePath);
+    } catch (e) {
+      if (e instanceof ModelSourceError) {
+        this.logError(e.message);
+        return false;
+      }
+      throw e;
+    }
 
     //
     // Save full model to build folder as Javascript for use in KeymanWeb
@@ -151,10 +160,10 @@ export default class LexicalModelCompiler {
 
     // we strip the mailto: from the .kps file for the .model_info
     set_model_metadata('authorEmail', kmpJsonData.info.author.url.match(/^(mailto\:)?(.+)$/)[2], false);
-    
+
     // extract the language identifiers from the language metadata
     // arrays for each of the lexical models in the kmp.json file,
-    // and merge into a single array of identifiers in the 
+    // and merge into a single array of identifiers in the
     // .model_info file.
 
     model_info.languages = model_info.languages || kmpJsonData.lexicalModels.reduce((a, e) => [].concat(a, e.languages.map((f) => f.id)), []);
@@ -163,7 +172,7 @@ export default class LexicalModelCompiler {
     set_model_metadata('packageFilename', kmpFileName);
 
     // Always overwrite with actual file size
-    model_info.packageFileSize = fs.statSync(model_info.packageFilename).size; 
+    model_info.packageFileSize = fs.statSync(model_info.packageFilename).size;
 
     set_model_metadata('jsFilename', modelFileName);
 
@@ -171,31 +180,31 @@ export default class LexicalModelCompiler {
     model_info.jsFileSize = fs.statSync(model_info.jsFilename).size;
 
     // Always overwrite source data
-    model_info.packageIncludes = kmpJsonData.files.filter((e) => !!e.name.match(/.[ot]tf$/i)).length ? ['fonts'] : []; 
+    model_info.packageIncludes = kmpJsonData.files.filter((e) => !!e.name.match(/.[ot]tf$/i)).length ? ['fonts'] : [];
 
     set_model_metadata('version', kmpJsonData.info.version.description);
 
     // The minimum Keyman version detected in the package file may be manually set higher by the developer
-    set_model_metadata('minKeymanVersion', minKeymanVersion, false); 
+    set_model_metadata('minKeymanVersion', minKeymanVersion, false);
 
     //TODO: model_info.helpLink = model_info.helpLink || ... if source/help/id.php exists?
     set_model_metadata('sourcePath', [groupPath, authorPath, bcp47Path].join('/'));
 
     fs.writeFileSync(modelInfoFileName, JSON.stringify(model_info, null, 2));
+
+    return true;
   };
 
   /**
    * Returns the generated code for the model that will ultimately be loaded by
    * the LMLayer worker. This code contains all model parameters, and specifies
    * word breakers and auxilary functions that may be required.
-   * 
+   *
    * @param model_id      The model ID. TODO: not sure if this is actually required!
    * @param modelSource   A specification of the model to compile
    * @param sourcePath    Where to find auxilary sources files
    */
   generateLexicalModelCode(model_id: string, modelSource: LexicalModelSource, sourcePath: string) {
-    let oc: LexicalModelCompiled = {id: model_id, format: modelSource.format};
-
     let sources: string[] = modelSource.sources.map(function(source) {
       return fs.readFileSync(path.join(sourcePath, source), 'utf8');
     });
@@ -231,8 +240,7 @@ export default class LexicalModelCompiler {
         func += `LMLayerWorker.loadModel(new ${modelSource.rootClass}());\n`;
         break;
       case "fst-foma-1.0":
-        this.logError('Unimplemented model format ' + modelSource.format);
-        return false;
+        throw new ModelSourceError(`Unimplemented model format: ${modelSource.format}`);
       case "trie-1.0":
         func += `LMLayerWorker.loadModel(new models.TrieModel(${
           createTrieDataStructure(sources, modelSource.searchTermToKey)
@@ -249,8 +257,7 @@ export default class LexicalModelCompiler {
         func += `}));\n`;
         break;
       default:
-        this.logError('Unknown model format '+modelSource.format);
-        return false;
+        throw new ModelSourceError(`Unknown model format: ${modelSource.format}`);
     }
 
     //
@@ -269,7 +276,10 @@ export default class LexicalModelCompiler {
     );
   };
 
-  logError(s) {
+  logError(s: string) {
     console.error(require('chalk').red(s));
   };
 };
+
+export class ModelSourceError extends Error {
+}

developer/js/source/lexical-model-compiler/build-trie.ts

@@ -55,7 +55,10 @@ export function parseWordList(contents: string): WordList {
     if (line.startsWith('#') || line === "") {
       continue; // skip comments and empty lines
     }
-    let [wordform, countText, _comment] = line.split(TAB);
+
+    // The third column is the comment. Always ignored!
+    let [wordform, countText] = line.split(TAB);
+
     // Clean the word form.
     // TODO: what happens if we get duplicate forms?
     wordform = wordform.normalize('NFC').trim();

developer/js/source/lexical-model-compiler/lexical-model.ts

@@ -2,7 +2,7 @@
  * Interfaces and constants used by the lexical model compiler. These target
  * the LMLayer's internal worker code, so we provide those definitions too.
  */
-/// <reference path="../../../common/predictive-text/worker/worker-compiler-interfaces.d.ts" />
+/// <reference path="../../../../common/predictive-text/worker/worker-compiler-interfaces.d.ts" />
 
 interface ClassBasedWordBreaker {
   allowedCharacters?: { initials?: string, medials?: string, finals?: string } | string,

developer/js/tsconfig.json

@@ -1,19 +1,17 @@
 {
   "compilerOptions": {
-    "target": "es2017",
     "module": "commonjs",
-    "outDir": "dist",
+    "target": "es2017",
+    "moduleResolution": "node",
     "sourceMap": true,
+    "outDir": "dist",
     "declaration": true,
+    "alwaysStrict": true,
+    "noImplicitReturns": true,
+    "noImplicitThis": true,
+    "noUnusedLocals": true
   },
   "include": [
-    "lexical-model-compiler/**/*",
-    "package-compiler/**/*",
-    "index.ts"
-  ],
-  "exclude": [
-    "node_modules",
-    "dist",
-    "tests/**/*.ts"
+    "source/**/*.ts"
   ]
-}
+}