diff --git a/common/predictive-text/index.ts b/common/predictive-text/index.ts index 5e9f9779e6..7bafc4ca0e 100644 --- a/common/predictive-text/index.ts +++ b/common/predictive-text/index.ts @@ -90,14 +90,14 @@ namespace com.keyman.text.prediction { */ loadModel(modelFilePath: string): Promise { return new Promise((resolve, _reject) => { + // Sets up so the promise is resolved in the onMessage() callback, when it receives + // the 'ready' message. + this._declareLMLayerReady = resolve; + this._worker.postMessage({ message: 'load', model: modelFilePath }); - - // Sets up so the promise is resolved in the onMessage() callback, when it receives - // the 'ready' message. - this._declareLMLayerReady = resolve; }); } diff --git a/common/predictive-text/worker/models/wordlist-model.ts b/common/predictive-text/worker/models/wordlist-model.ts index 2f3d564eb3..b2c167b42b 100644 --- a/common/predictive-text/worker/models/wordlist-model.ts +++ b/common/predictive-text/worker/models/wordlist-model.ts @@ -45,7 +45,18 @@ private _wordlist: string[]; constructor(wordlist: string[]) { - this._wordlist = wordlist; + // XXX: This may be given an Array of pairs, where the first element is + // the word and the second is the count. e.g., + // [["elephant", 2], ["zebra", 3], ["octopuses", 13]] + // + // Ignore the count when constructing our wordlist. e.g., + // ["elephant", "zebra", "octopuses"] + if (wordlist.length && Array.isArray(wordlist[0])) { + // @ts-ignore + this._wordlist = wordlist.map(([word, _count]) => word); + } else { + this._wordlist = wordlist; + } } configure(capabilities: Capabilities): Configuration { diff --git a/common/test/predictive-text/.gitignore b/common/test/predictive-text/.gitignore new file mode 100644 index 0000000000..c2658d7d1b --- /dev/null +++ b/common/test/predictive-text/.gitignore @@ -0,0 +1 @@ +node_modules/ diff --git a/common/test/predictive-text/README.md b/common/test/predictive-text/README.md new file mode 100644 index 0000000000..a6c3af5d08 --- /dev/null +++ b/common/test/predictive-text/README.md @@ -0,0 +1,66 @@ +Predictive text testing CLI +=========================== + +Provides a command line interface to try out predictive text models. + +Install +------- + +**NOTE**: Requires Node >= 10.0 + +Install locally with `npm`: + + npm install + +You can install globally like so: + + npm install -g . + +Usage +----- + +Start it like so: + + $ ./index.js -f path/to/model.js # local + $ lmlayer-cli . -f path/to/model.js # global + +You will be presented with a prompt. Type any phrase in the model's +language. + + * Press Tab to start selecting a suggestion. + * Press Tab again to select the next suggestion. + * Press Shift+Tab to select the previous suggestion. + * Press Enter to accept the selected suggestion. + * Press Ctrl+C or Ctrl+D to quit. + +``` +> He +[Hello] [Hey] [He he] +``` + +Defining the `LMPATH` environment variable +------------------------------------------ + +You can also load models by their model ID. For example, + + $ ./index.js example.en.model # local + $ lmlayer-cli example.en.model # global + +For this to work, you need to define the `LMPATH` environment variable. +The CLI will search the directory specified in `LMPATH` to find the +particular model. + +For example, say you have +[lexical-models](https://github.com/keymanapp/lexical-models) cloned in +your home directory. To be able to access any of the `release` build +lexical models, export the `LMPATH` variable as follows: + +```sh +export LMPATH="${HOME}/lexical-models/release" +``` + +This line usually goes in your shell startup file, e.g., `~/.bashrc`, +`~/.bash_profile`, or `~/.zshrc`, etc. + +Then, any lexical model built in `~/lexical-models/release` can be used +in the command line interface. diff --git a/common/test/predictive-text/index.js b/common/test/predictive-text/index.js new file mode 100755 index 0000000000..bd617d2219 --- /dev/null +++ b/common/test/predictive-text/index.js @@ -0,0 +1,460 @@ +#!/usr/bin/env node + +/** + * Implements a simple CLI for interactively testing the given predictive text + * model. + */ + +const path = require('path'); +const readline = require('readline'); + +const {EventIterator} = require('event-iterator'); +const program = require('commander'); + +// Load the most recent LMLayer code locally. +const LMLayer = require('../../predictive-text'); + + +///////////////////////////////// Constants ///////////////////////////////// + +const WORKER_DEBUG = false; +const EXIT_USAGE = 1; + +/** "Control sequence introducer" for ANSI escape codes: */ +const CSI = '\033['; +/** + * ANSI escape codes to deal with the screen and the cursor. + * See https://en.wikipedia.org/wiki/ANSI_escape_code#CSI_sequences + */ +const ANSI = { + CURSOR_NEXT_LINE: CSI + 'E', // Bring cursor to the beginning of the next line. + ERASE_IN_LINE(n=2) { return CSI + n + 'K'; }, // Erase the entire line. + SAVE_CURSOR_POSITION: CSI + 's', // Remembers the current cursor position. + RESTORE_CURSOR_POSITION: CSI + 'u', // Moves the cursor to the previously stored position. + BOLD: CSI + '1m', // Set bold text. + FAINT: CSI + '2m', // Set "faint" text. + REVERSE_VIDEO: CSI + '7m', // Invert the text colour (swap background and foreground colours). + NORMAL_VIDEO: CSI + '27m', // Uninvert the text colour (swap background and foreground colours). + NORMAL: CSI + 'm', // Set all graphics renditions attributes off. +}; + + +//////////////////////////////////// main //////////////////////////////////// + +main(); + +function main() { + // Command line options: + program + .version(require('./package.json').version) + .usage('(-f | )') + .description('CLI for trying lexical models.') + .arguments('[model-id]') + .option('-f, --model-file ', 'path to model file') + .parse(process.argv); + + let modelFile = determineModelFile(program); + + // Ensure we're running in the terminal + if (!process.stdin.isTTY) { + throw new Error('must be run from interactive terminal'); + } + + // Show a quick "how-to" message. + console.log(unindent(` + # Testing ${modelFile} + # + # Type text in the model's language. Suggestions will appear automatically. + # + # * Press ${ANSI.BOLD}${ANSI.NORMAL} to select a suggestion. + # * Press ${ANSI.BOLD}${ANSI.NORMAL} again to select the next suggestion. + # * Press ${ANSI.BOLD}${ANSI.NORMAL}+${ANSI.BOLD}${ANSI.NORMAL} to select the previous suggestion. + # * Press ${ANSI.BOLD}${ANSI.NORMAL} to accept the suggestion. + # * Press ${ANSI.BOLD}${ANSI.NORMAL}+${ANSI.BOLD}${ANSI.NORMAL} to quit. + `)) + + // The command line proper handles asynchronous keypresses, hence the rest + // must also be asynchronous. + asyncRepl(modelFile) + .then(_ => process.exit(0)) + .catch(err => { + console.error(err); + process.exit(127); + }); +} + + +async function asyncRepl(modelFile) { + // Load the LMLayer and the desired model. + let lm = new LMLayer({}, createAsyncWorker()); + let config = await lm.loadModel(modelFile); + + // Setup the REPL + // > type some text + let buffer = ''; // the text to predict on + // To support the prediction API, the previous buffer must be kept + // to apply suggested transforms correctly: + // See: https://github.com/keymanapp/keyman/blob/99caf52b38fddedfb0969b20c0e6b27be243c348/common/predictive-text/docs/worker-communication-protocol.md#message-predict + let previousBuffer = null; + let suggestions = []; + let selectedSuggestionIndex = null; // which suggestion is currently suggested + + // Draw the prompt for the first time. + redrawPrompt(); + + for await (let [char, keypress] of keypressesFromStdin()) { + if (wantsQuit(keypress)) { + process.stdout.write('\n'); + process.exit(0); + } + + if (keypress.name === 'tab') { + let n = suggestions.length; + if (n === 0) { + // nothing to select. + selectedSuggestionIndex = null; + continue; + } + + // Nothing selected, but there are suggestions -- then select the first + // one! + if (selectedSuggestionIndex === null) { + selectedSuggestionIndex = 0; + renderSuggestions(); + continue; + } + + if (keypress.shift === false) { + // Select the next suggestion. + selectedSuggestionIndex = (selectedSuggestionIndex + 1) % n; + } else { + // Select the previous suggestion. + + // Note that JavaScript's modulous operator is annoying with negative + // numbers, so use an if statement to handle wrapping. + selectedSuggestionIndex = selectedSuggestionIndex - 1; + // Wrap around to the end. + if (selectedSuggestionIndex < 0) { + selectedSuggestionIndex = n - 1; + } + } + + renderSuggestions(suggestions, selectedSuggestionIndex); + + } else if (keypress.name === 'return') { + // Accept the currently selected + let acceptedSuggestion = suggestions[selectedSuggestionIndex]; + + if (!acceptedSuggestion) { + // There's no suggestion to insert! + continue; + } + + applyTransformToActiveBuffer(acceptedSuggestion.transform); + redrawPrompt(); + + // Ask for suggestions. + suggestions = await askForPredictions(); + renderSuggestions(); + + } else if (keypress.name === 'backspace') { + if (buffer.length === 0) { + // nothing to do if the buffer is empty + continue; + } + + // Remove the last character. + deleteLastCodepoint(); + redrawPrompt(); + + // Ask for suggestions again + suggestions = await askForPredictions(); + renderSuggestions(); + + } else { + // Handle a keypress of a letter or symbol. + let {transform, context} = insertCharacter(char); + redrawPrompt(); + suggestions = await askForPredictions(transform, context); + renderSuggestions(); + } + } + + // Helpers + + /** + * Asks the LMLayer for predictions with the given transform and context. + * Asynchronously returns the new suggestions. + */ + async function askForPredictions(transform = nullTransform(), context = currentContext()) { + return Array.from(await lm.predict(transform, context)); + } + + /** + * Returns the null transform. + */ + function nullTransform() { + return { insert: '', deleteLeft: 0 }; + } + + /** + * Returns the current state of the buffer. + */ + function currentContext() { + return { + left: buffer, + startOfBuffer: buffer.length === 0, + endOfBuffer: true + }; + } + + /** + * Draws the prompt, clearing what was on the screen. + */ + function redrawPrompt() { + // Erases the ENTIRE line, returning to the beginning of the line. + process.stdout.write(ANSI.ERASE_IN_LINE() + '\r'); + // Write the buffer! + process.stdout.write(`> ${buffer}`); + } + + /** + * Inserts a character at the end of the buffer. Returns the transform and + * context of the character insert. + * + * Note: this mutates the buffer! + */ + function insertCharacter(char) { + previousBuffer = buffer; + buffer += char; + + let transform = { + insert: char, + deleteLeft: 0 + }; + + let context = { + left: previousBuffer, + startOfBuffer: previousBuffer.length === 0 ? true : false, + endOfBuffer: true, + }; + + return { transform, context }; + } + + /** + * Mutates the buffer by removing the last code point. This handles + * surrogate pairs. + */ + function deleteLastCodepoint() { + let lastIndex = buffer.length - 1; + // When the last character is a low surrogate (0xDC00-0xDFFF), this means + // we have a surrogate pair! We must delete two 16-bit code units. + if ('\uDC00' <= buffer[lastIndex] && buffer[lastIndex] <= '\uDFFF') { + console.assert('\uD800' <= buffer[lastIndex - 1] && buffer[lastIndex - 1] <= '\uDBFF'); + lastIndex = lastIndex - 1; + } + buffer = buffer.substr(0, lastIndex); + } + + /** + * Mutates buffer by applying the given transform. + */ + function applyTransformToActiveBuffer(transform) { + let tmpBuffer = buffer; + buffer = previousBuffer + .substr(0, previousBuffer.length - transform.deleteLeft) + transform.insert; + previousBuffer = tmpBuffer; + } + + /** + * Render the given suggestions on the next line after the cursor. + * Looks like this: + * + * [suggestions] [appear] [here] + */ + function renderSuggestions() { + // Wherever we are, save the current cursor position. + process.stdout.write(ANSI.SAVE_CURSOR_POSITION); + + // Jump to next line and erase it to write suggestions. + process.stdout.write(ANSI.CURSOR_NEXT_LINE + ANSI.ERASE_IN_LINE()); + + if (!suggestions || suggestions.length === 0) { + process.stdout.write(`${ANSI.FAINT}no suggestions${ANSI.NORMAL}`); + } else { + // Format the displayed suggestions. + let line = suggestions + .map(({displayAs}, index) => { + if (index === selectedSuggestionIndex) { + return `${ANSI.REVERSE_VIDEO}[${displayAs}]${ANSI.NORMAL_VIDEO}`; + } + return `[${displayAs}]` + }).join(' '); + process.stdout.write(line); + } + + process.stdout.write(ANSI.RESTORE_CURSOR_POSITION); + } +} + + +function createAsyncWorker() { + // XXX: import the LMLayerWorker directly -- I know where it is built. + const LMLayerWorker = require('../../predictive-text/build/intermediate'); + const vm = require('vm'); + const fs = require('fs'); + + let worker = { + postMessage(message) { + logInternalWorkerMessage('top-level', message); + workerScope.onmessage({ data: message }); + }, + onmessage(message) { + throw new Error('this should be reassigned!'); + } + }; + + let workerScope = { + LMLayerWorker, + postMessage(message) { + logInternalWorkerMessage('worker', message); + worker.onmessage({ data: message }); + }, + importScripts(uri) { + let sourceCode = fs.readFileSync(uri, 'UTF-8'); + logInternalWorkerMessage('worker', 'execing file:', uri); + vm.runInContext(sourceCode, workerScope, { + filename: uri, + displayErrors: true, + breakOnSigint: true, + }); + } + }; + vm.createContext(workerScope); + + let internal = LMLayerWorker.install(workerScope); + + // XXX: The current generated file expects these functions, but they do not, + // and will not exist! Stub them out: + LMLayerWorker['loadWordBreaker'] = function () { + console.warn('ignoring word breaker'); + }; + LMLayerWorker['loadModel'] = function (uri) { + // delegate to the internal LMLayerWorker instance: + internal.loadModel(uri); + } + + return worker; +} + + +///////////////////////////////// Utilities ///////////////////////////////// + +/** + * Figure out the path to the model file from the command line. + */ +function determineModelFile(program) { + // invoked as: predictive-text -f path/to/model.js + if (program.modelFile) { + return program.modelFile; + } + + // invoked as: predictive-text author.bcp47.uniq + // will lookup the model in using LMPATH. + if (program.args[0]) { + // Find the model file under the LMPath. + let modelID = program.args[0]; + let lmPath = process.env['LMPATH']; + let [author, bcp47, uniq] = modelID.split('.'); + + let LMPATH = process.env['LMPATH']; + if (!LMPATH) { + usageError('Environment variable LMPATH undefined!'); + } + + return path.join(LMPATH, author, `${bcp47}.${uniq}`, 'build', `${modelID}.model.js`); + } + + // A model is not specified in any way on the command line. Error out! + usageError('You did not specify a model!'); +} + +/** + * Allows you to iterate over each keypress event from stdin. + * + * Each iteration will yield a two-valued array of [char, keypress]. + * `char` is the string emitted by the keypress. This MAY be undefined! + * `keypress` is an object with the following properties: + * + * sequence: string; // the raw sequence recieved by the terminal driver. + * name: string; // name of the primary key pressed + * ctrl: bool; // was ctrl pressed? + * meta: bool; // was meta (Command/Windows key) pressed? + * shift: bool; // was shift pressed? + */ +function keypressesFromStdin() { + let stream = process.stdin; + + readline.emitKeypressEvents(stream); + if (stream.isTTY) { + stream.setRawMode(true); + } + stream.resume(); + + return new EventIterator( + (push, stop, fail) => { + stream.on('keypress', (char, keypress) => push([char, keypress])); + stream.on('close', stop); + stream.on('error', fail); + } + ); +} + +/** + * Loges messages that pass between the top-level and the worker, only when + * WORKER_DEBUG is enabled. + */ +function logInternalWorkerMessage(role, ...args) { + if (WORKER_DEBUG) { + console.log(`[${role}]`, ...args); + } +} + +/** + * Unindents a template literal. + */ +function unindent(string) { + let lines = string.split('\n'); + // Remove empty lines from the top and bottom. + if (lines[0] === '') { + lines.shift(); + } + if (lines[lines.length -1] === '') { + lines.pop(); + } + + let numLeadingSpaces = lines[0].match(/^ */)[0].length; + + return lines.map(line => line.substr(numLeadingSpaces)).join('\n'); +} + +/** + * Call to quit the program with an error due to incorrect command line usage. + */ +function usageError(message) { + console.error(`${program.name()}: ${message}\n`); + program.outputHelp(); + process.exit(EXIT_USAGE); +} + +/** + * True if the keypress looks like something that typically quits the program. + */ +function wantsQuit(keypress) { + const CTRL_C = '\u0003'; + const CTRL_D = '\u0004'; + const CTRL_BACKSLASH = '\u001C'; + let code = keypress.sequence; + return code === CTRL_C || code === CTRL_D || code === CTRL_BACKSLASH; +} diff --git a/common/test/predictive-text/package-lock.json b/common/test/predictive-text/package-lock.json new file mode 100644 index 0000000000..b6e04296a2 --- /dev/null +++ b/common/test/predictive-text/package-lock.json @@ -0,0 +1,18 @@ +{ + "name": "lmlayer-cli", + "version": "0.1.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "commander": { + "version": "2.20.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.0.tgz", + "integrity": "sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==" + }, + "event-iterator": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/event-iterator/-/event-iterator-1.2.0.tgz", + "integrity": "sha512-Daq7YUl0Mv1i4QEgzGQlz0jrx7hUFNyLGbiF+Ap7NCMCjDLCCnolyj6s0TAc6HmrBziO5rNVHsPwGMp7KdRPvw==" + } + } +} diff --git a/common/test/predictive-text/package.json b/common/test/predictive-text/package.json new file mode 100644 index 0000000000..f4a5ed2b22 --- /dev/null +++ b/common/test/predictive-text/package.json @@ -0,0 +1,20 @@ +{ + "name": "lmlayer-cli", + "version": "0.1.0", + "description": "Command line interface to the predictive text engine.", + "main": "index.js", + "bin": "./index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "Eddie Antonio Santos ", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/keymanapp/keyman.git" + }, + "dependencies": { + "commander": "^2.20.0", + "event-iterator": "^1.2.0" + } +}