Merge pull request #1702 from keymanapp/lmlayer-cli

[LMLayer] Command line interface for quick testing of models

common/predictive-text/index.ts

@@ -90,14 +90,14 @@ namespace com.keyman.text.prediction {
      */
     loadModel(modelFilePath: string): Promise<Configuration> {
       return new Promise((resolve, _reject) => {
+        // Sets up so the promise is resolved in the onMessage() callback, when it receives
+        // the 'ready' message.
+        this._declareLMLayerReady = resolve;
+
         this._worker.postMessage({
           message: 'load',
           model: modelFilePath
         });
-
-        // Sets up so the promise is resolved in the onMessage() callback, when it receives
-        // the 'ready' message.
-        this._declareLMLayerReady = resolve;
       });
     }
 

common/predictive-text/worker/models/wordlist-model.ts

@@ -45,7 +45,18 @@
     private _wordlist: string[];
 
     constructor(wordlist: string[]) {
-      this._wordlist = wordlist;
+      // XXX: This may be given an Array of pairs, where the first element is
+      //      the word and the second is the count. e.g.,
+      //        [["elephant", 2], ["zebra", 3], ["octopuses", 13]]
+      //
+      //      Ignore the count when constructing our wordlist. e.g.,
+      //        ["elephant", "zebra", "octopuses"]
+      if (wordlist.length && Array.isArray(wordlist[0])) {
+        // @ts-ignore
+        this._wordlist = wordlist.map(([word, _count]) => word);
+      } else {
+        this._wordlist = wordlist;
+      }
     }
 
     configure(capabilities: Capabilities): Configuration {

common/test/predictive-text/.gitignore

@@ -0,0 +1 @@
+node_modules/

common/test/predictive-text/README.md

@@ -0,0 +1,66 @@
+Predictive text testing CLI
+===========================
+
+Provides a command line interface to try out predictive text models.
+
+Install
+-------
+
+**NOTE**: Requires Node >= 10.0
+
+Install locally with `npm`:
+
+    npm install
+
+You can install globally like so:
+
+    npm install -g .
+
+Usage
+-----
+
+Start it like so:
+
+    $ ./index.js -f path/to/model.js  # local
+    $ lmlayer-cli . -f path/to/model.js  # global
+
+You will be presented with a prompt. Type any phrase in the model's
+language.
+
+ * Press <kbd>Tab</kbd> to start selecting a suggestion.
+ * Press <kbd>Tab</kbd> again to select the next suggestion.
+ * Press <kbd>Shift</kbd>+<kbd>Tab</kbd> to select the previous suggestion.
+ * Press <kbd>Enter</kbd> to accept the selected suggestion.
+ * Press <kbd>Ctrl</kbd>+C or <kbd>Ctrl</kbd>+D to quit.
+
+```
+> He
+[Hello] [Hey] [He he]
+```
+
+Defining the `LMPATH` environment variable
+------------------------------------------
+
+You can also load models by their model ID. For example,
+
+    $ ./index.js example.en.model  # local
+    $ lmlayer-cli example.en.model  # global
+
+For this to work, you need to define the `LMPATH` environment variable.
+The CLI will search the directory specified in `LMPATH` to find the
+particular model.
+
+For example, say you have
+[lexical-models](https://github.com/keymanapp/lexical-models) cloned in
+your home directory. To be able to access any of the `release` build
+lexical models, export the `LMPATH` variable as follows:
+
+```sh
+export LMPATH="${HOME}/lexical-models/release"
+```
+
+This line usually goes in your shell startup file, e.g., `~/.bashrc`,
+`~/.bash_profile`, or `~/.zshrc`, etc.
+
+Then, any lexical model built in `~/lexical-models/release` can be used
+in the command line interface.

common/test/predictive-text/index.js

@@ -0,0 +1,460 @@
+#!/usr/bin/env node
+
+/**
+ * Implements a simple CLI for interactively testing the given predictive text
+ * model.
+ */
+
+const path = require('path');
+const readline = require('readline');
+
+const {EventIterator} = require('event-iterator');
+const program =  require('commander');
+
+// Load the most recent LMLayer code locally.
+const LMLayer = require('../../predictive-text');
+
+
+///////////////////////////////// Constants /////////////////////////////////
+
+const WORKER_DEBUG = false;
+const EXIT_USAGE = 1;
+
+/** "Control sequence introducer" for ANSI escape codes: */
+const CSI = '\033[';
+/**
+ * ANSI escape codes to deal with the screen and the cursor.
+ * See https://en.wikipedia.org/wiki/ANSI_escape_code#CSI_sequences
+ */
+const ANSI = {
+  CURSOR_NEXT_LINE: CSI + 'E', // Bring cursor to the beginning of the next line.
+  ERASE_IN_LINE(n=2) { return CSI + n + 'K'; }, // Erase the entire line.
+  SAVE_CURSOR_POSITION: CSI + 's', // Remembers the current cursor position.
+  RESTORE_CURSOR_POSITION: CSI + 'u', // Moves the cursor to the previously stored position.
+  BOLD: CSI + '1m', // Set bold text.
+  FAINT: CSI + '2m', // Set "faint" text.
+  REVERSE_VIDEO: CSI + '7m', // Invert the text colour (swap background and foreground colours).
+  NORMAL_VIDEO: CSI + '27m', // Uninvert the text colour (swap background and foreground colours).
+  NORMAL: CSI + 'm', // Set all graphics renditions attributes off.
+};
+
+
+//////////////////////////////////// main ////////////////////////////////////
+
+main();
+
+function main() {
+  // Command line options:
+  program
+    .version(require('./package.json').version)
+    .usage('(-f <model-file> | <model-id>)')
+    .description('CLI for trying lexical models.')
+    .arguments('[model-id]')
+    .option('-f, --model-file <file>', 'path to model file')
+    .parse(process.argv);
+
+  let modelFile = determineModelFile(program);
+
+  // Ensure we're running in the terminal
+  if (!process.stdin.isTTY) {
+    throw new Error('must be run from interactive terminal');
+  }
+
+  // Show a quick "how-to" message.
+  console.log(unindent(`
+    # Testing ${modelFile}
+    #
+    # Type text in the model's language. Suggestions will appear automatically.
+    #
+    #  * Press ${ANSI.BOLD}<Tab>${ANSI.NORMAL} to select a suggestion.
+    #  * Press ${ANSI.BOLD}<Tab>${ANSI.NORMAL} again to select the next suggestion.
+    #  * Press ${ANSI.BOLD}<Shift>${ANSI.NORMAL}+${ANSI.BOLD}<Tab>${ANSI.NORMAL} to select the previous suggestion.
+    #  * Press ${ANSI.BOLD}<Enter>${ANSI.NORMAL} to accept the suggestion.
+    #  * Press ${ANSI.BOLD}<Ctrl>${ANSI.NORMAL}+${ANSI.BOLD}<C>${ANSI.NORMAL} to quit.
+  `))
+
+  // The command line proper handles asynchronous keypresses, hence the rest
+  // must also be asynchronous.
+  asyncRepl(modelFile)
+    .then(_ => process.exit(0))
+    .catch(err => {
+      console.error(err);
+      process.exit(127);
+    });
+}
+
+
+async function asyncRepl(modelFile) {
+  // Load the LMLayer and the desired model.
+  let lm = new LMLayer({}, createAsyncWorker());
+  let config = await lm.loadModel(modelFile);
+
+  // Setup the REPL
+  // > type some text
+  let buffer = '';  // the text to predict on
+  // To support the prediction API, the previous buffer must be kept
+  // to apply suggested transforms correctly:
+  // See: https://github.com/keymanapp/keyman/blob/99caf52b38fddedfb0969b20c0e6b27be243c348/common/predictive-text/docs/worker-communication-protocol.md#message-predict
+  let previousBuffer = null;
+  let suggestions = [];
+  let selectedSuggestionIndex = null;  // which suggestion is currently suggested
+
+  // Draw the prompt for the first time.
+  redrawPrompt();
+
+  for await (let [char, keypress] of keypressesFromStdin()) {
+    if (wantsQuit(keypress)) {
+      process.stdout.write('\n');
+      process.exit(0);
+    }
+
+    if (keypress.name === 'tab') {
+      let n = suggestions.length;
+      if (n === 0) {
+        // nothing to select.
+        selectedSuggestionIndex = null;
+        continue;
+      }
+
+      // Nothing selected, but there are suggestions -- then select the first
+      // one!
+      if (selectedSuggestionIndex === null) {
+        selectedSuggestionIndex = 0;
+        renderSuggestions();
+        continue;
+      }
+
+      if (keypress.shift === false) {
+        // Select the next suggestion.
+        selectedSuggestionIndex = (selectedSuggestionIndex + 1) % n;
+      } else {
+        // Select the previous suggestion.
+
+        // Note that JavaScript's modulous operator is annoying with negative
+        // numbers, so use an if statement to handle wrapping.
+        selectedSuggestionIndex = selectedSuggestionIndex - 1;
+        // Wrap around to the end.
+        if (selectedSuggestionIndex < 0) {
+          selectedSuggestionIndex = n - 1;
+        }
+      }
+
+      renderSuggestions(suggestions, selectedSuggestionIndex);
+
+    } else if (keypress.name === 'return') {
+      // Accept the currently selected
+      let acceptedSuggestion = suggestions[selectedSuggestionIndex];
+
+      if (!acceptedSuggestion) {
+        // There's no suggestion to insert!
+        continue;
+      }
+
+      applyTransformToActiveBuffer(acceptedSuggestion.transform);
+      redrawPrompt();
+
+      // Ask for suggestions.
+      suggestions = await askForPredictions();
+      renderSuggestions();
+
+    } else if (keypress.name === 'backspace') {
+      if (buffer.length === 0) {
+        // nothing to do if the buffer is empty
+        continue;
+      }
+
+      // Remove the last character.
+      deleteLastCodepoint();
+      redrawPrompt();
+
+      // Ask for suggestions again
+      suggestions = await askForPredictions();
+      renderSuggestions();
+
+    } else {
+      // Handle a keypress of a letter or symbol.
+      let {transform, context} = insertCharacter(char);
+      redrawPrompt();
+      suggestions = await askForPredictions(transform, context);
+      renderSuggestions();
+    }
+  }
+
+  // Helpers
+
+  /**
+   * Asks the LMLayer for predictions with the given transform and context.
+   * Asynchronously returns the new suggestions.
+   */
+  async function askForPredictions(transform = nullTransform(), context = currentContext()) {
+    return Array.from(await lm.predict(transform, context));
+  }
+
+  /**
+   * Returns the null transform.
+   */
+  function nullTransform() {
+    return { insert: '', deleteLeft: 0 };
+  }
+
+  /**
+   * Returns the current state of the buffer.
+   */
+  function currentContext() {
+    return {
+      left: buffer,
+      startOfBuffer: buffer.length === 0,
+      endOfBuffer: true
+    };
+  }
+
+  /**
+   * Draws the prompt, clearing what was on the screen.
+   */
+  function redrawPrompt() {
+    // Erases the ENTIRE line, returning to the beginning of the line.
+    process.stdout.write(ANSI.ERASE_IN_LINE() + '\r');
+    // Write the buffer!
+    process.stdout.write(`> ${buffer}`);
+  }
+
+  /**
+   * Inserts a character at the end of the buffer.  Returns the transform and
+   * context of the character insert.
+   *
+   * Note: this mutates the buffer!
+   */
+  function insertCharacter(char) {
+    previousBuffer = buffer;
+    buffer += char;
+
+    let transform = {
+      insert: char,
+      deleteLeft: 0
+    };
+
+    let context = {
+      left: previousBuffer,
+      startOfBuffer: previousBuffer.length === 0 ? true : false,
+      endOfBuffer: true,
+    };
+
+    return { transform, context };
+  }
+
+  /**
+   * Mutates the buffer by removing the last code point. This handles
+   * surrogate pairs.
+   */
+  function deleteLastCodepoint() {
+    let lastIndex = buffer.length - 1;
+    // When the last character is a low surrogate (0xDC00-0xDFFF), this means
+    // we have a surrogate pair! We must delete two 16-bit code units.
+    if ('\uDC00' <= buffer[lastIndex] && buffer[lastIndex] <= '\uDFFF') {
+      console.assert('\uD800'  <= buffer[lastIndex - 1] && buffer[lastIndex - 1] <= '\uDBFF');
+      lastIndex = lastIndex - 1;
+    }
+    buffer = buffer.substr(0, lastIndex);
+  }
+
+  /**
+   * Mutates buffer by applying the given transform.
+   */
+  function applyTransformToActiveBuffer(transform) {
+    let tmpBuffer = buffer;
+    buffer = previousBuffer
+      .substr(0, previousBuffer.length - transform.deleteLeft) + transform.insert;
+    previousBuffer = tmpBuffer;
+  }
+
+  /**
+   * Render the given suggestions on the next line after the cursor.
+   * Looks like this:
+   *
+   *    [suggestions] [appear] [here]
+   */
+  function renderSuggestions() {
+    // Wherever we are, save the current cursor position.
+    process.stdout.write(ANSI.SAVE_CURSOR_POSITION);
+
+    // Jump to next line and erase it to write suggestions.
+    process.stdout.write(ANSI.CURSOR_NEXT_LINE + ANSI.ERASE_IN_LINE());
+
+    if (!suggestions || suggestions.length === 0) {
+      process.stdout.write(`${ANSI.FAINT}no suggestions${ANSI.NORMAL}`);
+    } else {
+      // Format the displayed suggestions.
+      let line = suggestions
+        .map(({displayAs}, index) => {
+          if (index === selectedSuggestionIndex) {
+            return `${ANSI.REVERSE_VIDEO}[${displayAs}]${ANSI.NORMAL_VIDEO}`;
+          }
+          return `[${displayAs}]`
+        }).join(' ');
+      process.stdout.write(line);
+    }
+
+    process.stdout.write(ANSI.RESTORE_CURSOR_POSITION);
+  }
+}
+
+
+function createAsyncWorker() {
+  // XXX: import the LMLayerWorker directly -- I know where it is built.
+  const LMLayerWorker = require('../../predictive-text/build/intermediate');
+  const vm = require('vm');
+  const fs = require('fs');
+
+  let worker = {
+    postMessage(message) {
+      logInternalWorkerMessage('top-level', message);
+      workerScope.onmessage({ data: message });
+    },
+    onmessage(message) {
+      throw new Error('this should be reassigned!');
+    }
+  };
+
+  let workerScope = {
+    LMLayerWorker,
+    postMessage(message) {
+      logInternalWorkerMessage('worker', message);
+      worker.onmessage({ data: message });
+    },
+    importScripts(uri) {
+      let sourceCode = fs.readFileSync(uri, 'UTF-8');
+      logInternalWorkerMessage('worker', 'execing file:', uri);
+      vm.runInContext(sourceCode, workerScope, {
+        filename: uri,
+        displayErrors: true,
+        breakOnSigint: true,
+      });
+    }
+  };
+  vm.createContext(workerScope);
+
+  let internal = LMLayerWorker.install(workerScope);
+
+  // XXX: The current generated file expects these functions, but they do not,
+  //      and will not exist! Stub them out:
+  LMLayerWorker['loadWordBreaker'] = function () {
+    console.warn('ignoring word breaker');
+  };
+  LMLayerWorker['loadModel'] = function (uri) {
+    // delegate to the internal LMLayerWorker instance:
+    internal.loadModel(uri);
+  }
+
+  return worker;
+}
+
+
+///////////////////////////////// Utilities /////////////////////////////////
+
+/**
+ * Figure out the path to the model file from the command line.
+ */
+function determineModelFile(program) {
+  // invoked as: predictive-text -f path/to/model.js
+  if (program.modelFile) {
+    return program.modelFile;
+  }
+
+  // invoked as: predictive-text author.bcp47.uniq
+  // will lookup the model in using LMPATH.
+  if (program.args[0]) {
+    // Find the model file under the LMPath.
+    let modelID = program.args[0];
+    let lmPath = process.env['LMPATH'];
+    let [author, bcp47, uniq] = modelID.split('.');
+
+    let LMPATH = process.env['LMPATH'];
+    if (!LMPATH) {
+      usageError('Environment variable LMPATH undefined!');
+    }
+
+    return path.join(LMPATH, author, `${bcp47}.${uniq}`, 'build', `${modelID}.model.js`);
+  }
+
+  // A model is not specified in any way on the command line. Error out!
+  usageError('You did not specify a model!');
+}
+
+/**
+ * Allows you to iterate over each keypress event from stdin.
+ *
+ * Each iteration will yield a two-valued array of [char, keypress].
+ * `char` is the string emitted by the keypress. This MAY be undefined!
+ * `keypress` is an object with the following properties:
+ *
+ *    sequence: string; // the raw sequence recieved by the terminal driver.
+ *    name: string;     // name of the primary key pressed
+ *    ctrl: bool;       // was ctrl pressed?
+ *    meta: bool;       // was meta (Command/Windows key) pressed?
+ *    shift: bool;      // was shift pressed?
+ */
+function keypressesFromStdin() {
+  let stream = process.stdin;
+
+  readline.emitKeypressEvents(stream);
+  if (stream.isTTY) {
+    stream.setRawMode(true);
+  }
+  stream.resume();
+
+  return new EventIterator(
+    (push, stop, fail) => {
+      stream.on('keypress', (char, keypress) => push([char, keypress]));
+      stream.on('close', stop);
+      stream.on('error', fail);
+    }
+  );
+}
+
+/**
+ * Loges messages that pass between the top-level and the worker, only when
+ * WORKER_DEBUG is enabled.
+ */
+function logInternalWorkerMessage(role, ...args) {
+  if (WORKER_DEBUG) {
+    console.log(`[${role}]`, ...args);
+  }
+}
+
+/**
+ * Unindents a template literal.
+ */
+function unindent(string) {
+  let lines = string.split('\n');
+  // Remove empty lines from the top and bottom.
+  if (lines[0] === '') {
+    lines.shift();
+  }
+  if (lines[lines.length -1] === '') {
+    lines.pop();
+  }
+
+  let numLeadingSpaces = lines[0].match(/^ */)[0].length;
+
+  return lines.map(line => line.substr(numLeadingSpaces)).join('\n');
+}
+
+/**
+ * Call to quit the program with an error due to incorrect command line usage.
+ */
+function usageError(message) {
+  console.error(`${program.name()}: ${message}\n`);
+  program.outputHelp();
+  process.exit(EXIT_USAGE);
+}
+
+/**
+ * True if the keypress looks like something that typically quits the program.
+ */
+function wantsQuit(keypress) {
+  const CTRL_C = '\u0003';
+  const CTRL_D = '\u0004';
+  const CTRL_BACKSLASH = '\u001C';
+  let code = keypress.sequence;
+  return code === CTRL_C || code === CTRL_D || code === CTRL_BACKSLASH;
+}

common/test/predictive-text/package-lock.json

@@ -0,0 +1,18 @@
+{
+  "name": "lmlayer-cli",
+  "version": "0.1.0",
+  "lockfileVersion": 1,
+  "requires": true,
+  "dependencies": {
+    "commander": {
+      "version": "2.20.0",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.0.tgz",
+      "integrity": "sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ=="
+    },
+    "event-iterator": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/event-iterator/-/event-iterator-1.2.0.tgz",
+      "integrity": "sha512-Daq7YUl0Mv1i4QEgzGQlz0jrx7hUFNyLGbiF+Ap7NCMCjDLCCnolyj6s0TAc6HmrBziO5rNVHsPwGMp7KdRPvw=="
+    }
+  }
+}

common/test/predictive-text/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "lmlayer-cli",
+  "version": "0.1.0",
+  "description": "Command line interface to the predictive text engine.",
+  "main": "index.js",
+  "bin": "./index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "Eddie Antonio Santos <Eddie.Santos@nrc-cnrc.gc.ca>",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/keymanapp/keyman.git"
+  },
+  "dependencies": {
+    "commander": "^2.20.0",
+    "event-iterator": "^1.2.0"
+  }
+}