erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-10-17 19:37:39 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

chore(web): restores feature-esmodule-web-engine to prior state

Browse source 
Revert "Merge pull request #8553 from keymanapp/revert-8549-chore/master-to-esmodule-merge-a17s9"

This reverts commit 518750f94e, reversing
changes made to 6df63eddc5.
This commit is contained in:
Joshua A. Horton 2023-03-31 12:53:00 +07:00
parent 518750f94e
commit 7f4debdfbf
529 changed files with 50154 additions and 27847 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
android/KMEA/app/src/main/assets/android-host.js
View file

@ -28,9 +28,9 @@ function init() {
keyman.init({'app':device,'fonts':'packages/',root:'./'});
keyman.util.setOption('attachType','manual');
keyman.oninserttext = insertText;
keyman.showKeyboardList = showMenu;
keyman.menuKeyUp = menuKeyUp;
keyman.hideKeyboard = hideKeyboard;
window.showKeyboardList = showMenu;
window.hideKeyboard = hideKeyboard;
window.menuKeyUp = menuKeyUp;
keyman.getOskHeight = getOskHeight;
keyman.getOskWidth = getOskWidth;
keyman.beepKeyboard = beepKeyboard;
@ -170,10 +170,8 @@ function deregisterModel(modelID) {
function enableSuggestions(model, mayPredict, mayCorrect) {
// Set the options first so that KMW's ModelManager can properly handle model enablement states
// the moment we actually register the new model.
keyman.osk.banner.setOptions({
'mayPredict': mayPredict,
'mayCorrect': mayCorrect
});
keyman.core.languageProcessor.mayPredict = mayPredict;
keyman.core.languageProcessor.mayCorrect = mayCorrect;
registerModel(model);
}
@ -186,7 +184,7 @@ function setBannerOptions(mayPredict) {
function registerModel(model) {
//window.console.log('registerModel: ' + model);
keyman.registerModel(model);
keyman.addModel(model);
}
function resetContext() {

33
common/models/templates/build-bundler.js Normal file
View file

@ -0,0 +1,33 @@
/*
* Note: while this file is not meant to exist long-term, it provides a nice
* low-level proof-of-concept for esbuild bundling of the various Web submodules.
*
* Add some extra code at the end of src/index.ts and run it to verify successful bundling!
*/
import esbuild from 'esbuild';
import { spawn } from 'child_process';
await esbuild.build({
bundle: true,
sourcemap: true,
format: "esm",
nodePaths: ['..'],
entryPoints: ['build/obj/index.js'],
outfile: "build/lib/index.mjs",
tsconfig: 'tsconfig.json',
target: "es5"
});
// Bundled CommonJS (classic Node) module version
esbuild.buildSync({
bundle: true,
sourcemap: true,
format: "cjs",
nodePaths: ['..'],
entryPoints: ['build/obj/index.js'],
outfile: "build/lib/index.cjs",
tsconfig: 'tsconfig.json',
target: "es5"
});

64
common/models/templates/build.sh Executable file
View file

@ -0,0 +1,64 @@
#!/usr/bin/env bash
#
# Compile our sourcemap-path remapping module for use by Web builds, releases, etc.
#
set -eu
## START STANDARD BUILD SCRIPT INCLUDE
# adjust relative paths as necessary
THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BASH_SOURCE[0]}")"
. "$(dirname "$THIS_SCRIPT")/../../../resources/build/build-utils.sh"
## END STANDARD BUILD SCRIPT INCLUDE
. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh"
# This script runs from its own folder
cd "$(dirname "$THIS_SCRIPT")"
################################ Main script ################################
builder_describe "Builds the predictive-text model template implementation module" \
"@../../web/keyman-version" \
"@../wordbreakers" \
"clean" \
"configure" \
"build" \
"test"
builder_describe_outputs \
configure /node_modules \
build build/lib/index.mjs
builder_parse "$@"
### CONFIGURE ACTIONS
if builder_start_action configure; then
verify_npm_setup
builder_finish_action success configure
fi
### CLEAN ACTIONS
if builder_start_action clean; then
rm -rf build/
builder_finish_action success clean
fi
### BUILD ACTIONS
if builder_start_action build; then
npm run tsc -- -b
node build-bundler.js
# Declaration bundling.
npm run tsc -- --emitDeclarationOnly --outFile ./build/lib/index.d.ts
builder_finish_action success build
fi
if builder_start_action test; then
npm run mocha -- --require test/helpers.js --recursive test
builder_finish_action success test
fi

25
common/models/templates/package.json
View file

@ -16,8 +16,19 @@
],
"homepage": "https://github.com/keymanapp/keyman#readme",
"license": "MIT",
"main": "index.js",
"types": "index.d.ts",
"main": "./build/obj/index.js",
"types": "./build/obj/index.d.ts",
"exports": {
".": "./build/obj/index.js",
"./lib": {
"types": "./build/lib/index.d.ts",
"import": "./build/lib/index.mjs",
"require": "./build/lib/index.cjs"
},
"./obj/*.js": "./build/obj/*.js"
},
"type": "module",
"directories": {
"test": "test"
},
@ -32,8 +43,9 @@
"url": "git+https://github.com/keymanapp/keyman.git"
},
"scripts": {
"build": "tsc",
"pretest": "tsc -b tsconfig.bundled.json",
"build": "gosh ./build.sh",
"tsc": "tsc",
"mocha": "mocha",
"test": "mocha -r test/helpers.js"
},
"bugs": {
@ -50,6 +62,9 @@
"typescript": "^4.9.5"
},
"dependencies": {
"@keymanapp/models-wordbreakers": "*"
"@keymanapp/keyman-version": "*",
"@keymanapp/models-types": "*",
"@keymanapp/models-wordbreakers": "*",
"@keymanapp/resources-gosh": "*"
}
}

273
common/models/templates/src/common.ts
View file

@ -1,148 +1,151 @@
namespace models {
export const SENTINEL_CODE_UNIT = '\uFDD0';
// Allows the kmwstring bindings to resolve.
import { extendString } from "@keymanapp/web-utils";
export function applyTransform(transform: Transform, context: Context): Context {
// First, get the current context
let fullLeftContext = context.left || '';
let lLen = fullLeftContext.kmwLength();
let lDel = lLen < transform.deleteLeft ? lLen : transform.deleteLeft;
extendString();
let leftContext = fullLeftContext.kmwSubstr(0, lLen - lDel) + (transform.insert || '');
export const SENTINEL_CODE_UNIT = '\uFDD0';
let fullRightContext = context.right || '';
let rLen = fullRightContext.kmwLength();
let rDel = rLen < transform.deleteRight ? rLen : transform.deleteRight;
export function applyTransform(transform: Transform, context: Context): Context {
// First, get the current context
let fullLeftContext = context.left || '';
let lLen = fullLeftContext.kmwLength();
let lDel = lLen < transform.deleteLeft ? lLen : transform.deleteLeft;
let rightContext = fullRightContext.kmwSubstr(rDel);
let leftContext = fullLeftContext.kmwSubstr(0, lLen - lDel) + (transform.insert || '');
return {
left: leftContext,
right: rightContext,
startOfBuffer: context.startOfBuffer,
endOfBuffer: context.endOfBuffer,
casingForm: context.casingForm
};
}
let fullRightContext = context.right || '';
let rLen = fullRightContext.kmwLength();
let rDel = (rLen < (transform.deleteRight ?? 0)) ? rLen : (transform.deleteRight ?? 0);
/**
* Merges two Transforms as if they were applied to a `Context` successively.
* @param first
* @param second
*/
export function buildMergedTransform(first: Transform, second: Transform): Transform {
// These exist to avoid parameter mutation.
let mergedFirstInsert: string = first.insert;
let mergedSecondDelete: number = second.deleteLeft;
let rightContext = fullRightContext.kmwSubstr(rDel);
// The 'fun' case: the second Transform wants to delete something from the first.
if(second.deleteLeft) {
let firstLength = first.insert.kmwLength();
if(firstLength <= second.deleteLeft) {
mergedFirstInsert = '';
mergedSecondDelete = second.deleteLeft - firstLength;
} else {
mergedFirstInsert = first.insert.kmwSubstr(0, firstLength - second.deleteLeft);
mergedSecondDelete = 0;
}
}
return {
left: leftContext,
right: rightContext,
startOfBuffer: context.startOfBuffer,
endOfBuffer: context.endOfBuffer,
casingForm: context.casingForm
};
}
return {
insert: mergedFirstInsert + second.insert,
deleteLeft: first.deleteLeft + mergedSecondDelete,
// As `first` would affect the context before `second` could take effect,
// this is the correct way to merge `deleteRight`.
deleteRight: (first.deleteRight || 0) + (second.deleteRight || 0)
/**
* Merges two Transforms as if they were applied to a `Context` successively.
* @param first
* @param second
*/
export function buildMergedTransform(first: Transform, second: Transform): Transform {
// These exist to avoid parameter mutation.
let mergedFirstInsert: string = first.insert;
let mergedSecondDelete: number = second.deleteLeft;
// The 'fun' case: the second Transform wants to delete something from the first.
if(second.deleteLeft) {
let firstLength = first.insert.kmwLength();
if(firstLength <= second.deleteLeft) {
mergedFirstInsert = '';
mergedSecondDelete = second.deleteLeft - firstLength;
} else {
mergedFirstInsert = first.insert.kmwSubstr(0, firstLength - second.deleteLeft);
mergedSecondDelete = 0;
}
}
/**
* Checks whether or not the specified UCS-2 character corresponds to a UTF-16 high surrogate.
*
* @param char A single JavaScript (UCS-2) char corresponding to a single code unit.
*/
export function isHighSurrogate(char: string): boolean;
/**
* Checks whether or not the specified UCS-2 character corresponds to a UTF-16 high surrogate.
*
* @param codeUnit A code unit corresponding to a single UCS-2 char.
*/
export function isHighSurrogate(codeUnit: number): boolean;
export function isHighSurrogate(codeUnit: string|number): boolean {
if(typeof codeUnit == 'string') {
codeUnit = codeUnit.charCodeAt(0);
}
return codeUnit >= 0xD800 && codeUnit <= 0xDBFF;
}
/**
* Checks whether or not the specified UCS-2 character corresponds to a UTF-16 low surrogate.
*
* @param char A single JavaScript (UCS-2) char corresponding to a single code unit.
*/
export function isLowSurrogate(char: string): boolean;
/**
* Checks whether or not the specified UCS-2 character corresponds to a UTF-16 low surrogate.
*
* @param codeUnit A code unit corresponding to a single UCS-2 char.
*/
export function isLowSurrogate(codeUnit: number): boolean;
export function isLowSurrogate(codeUnit: string|number): boolean {
if(typeof codeUnit == 'string') {
codeUnit = codeUnit.charCodeAt(0);
}
return codeUnit >= 0xDC00 && codeUnit <= 0xDFFF;
}
export function isSentinel(char: string): boolean {
return char == models.SENTINEL_CODE_UNIT;
}
/**
* Builds a Suggestion based on a Transform corresponding to a predictive-text op.
*
* Assumes that the Transform's `insert` property represents a completed word,
* as models generally delete the whole prefix, replacing it with the full lexical entry.
* @param transform
*/
export function transformToSuggestion(transform: Transform): Suggestion;
export function transformToSuggestion(transform: Transform, p: number): WithOutcome<Suggestion>;
export function transformToSuggestion(transform: Transform, p?: number): Outcome<Suggestion> {
let suggestion: Outcome<Suggestion> = {
transform: transform,
transformId: transform.id,
displayAs: transform.insert
};
if(p === 0 || p) {
suggestion.p = p;
}
return suggestion;
}
export function defaultApplyCasing(casing: CasingForm, text: string): string {
switch(casing) {
case 'lower':
return text.toLowerCase();
case 'upper':
return text.toUpperCase();
case 'initial':
// The length of the first code unit, as measured in code points.
let headUnitLength = 1;
// Is the first character a high surrogate, indicating possible use of UTF-16
// surrogate pairs? Also, is the string long enough for there to BE a pair?
if(text.length > 1 && isHighSurrogate(text.charAt(0))) {
// It's possible, so now we check for low surrogates.
if(isLowSurrogate(text.charCodeAt(1))) {
// We have a surrogate pair; this pair is the 'first' character.
headUnitLength = 2;
}
}
// Capitalizes the first code unit of the string, leaving the rest intact.
return text.substring(0, headUnitLength).toUpperCase() .concat(text.substring(headUnitLength));
}
return {
insert: mergedFirstInsert + second.insert,
deleteLeft: first.deleteLeft + mergedSecondDelete,
// As `first` would affect the context before `second` could take effect,
// this is the correct way to merge `deleteRight`.
deleteRight: (first.deleteRight || 0) + (second.deleteRight || 0)
}
}
/**
* Checks whether or not the specified UCS-2 character corresponds to a UTF-16 high surrogate.
*
* @param char A single JavaScript (UCS-2) char corresponding to a single code unit.
*/
export function isHighSurrogate(char: string): boolean;
/**
* Checks whether or not the specified UCS-2 character corresponds to a UTF-16 high surrogate.
*
* @param codeUnit A code unit corresponding to a single UCS-2 char.
*/
export function isHighSurrogate(codeUnit: number): boolean;
export function isHighSurrogate(codeUnit: string|number): boolean {
if(typeof codeUnit == 'string') {
codeUnit = codeUnit.charCodeAt(0);
}
return codeUnit >= 0xD800 && codeUnit <= 0xDBFF;
}
/**
* Checks whether or not the specified UCS-2 character corresponds to a UTF-16 low surrogate.
*
* @param char A single JavaScript (UCS-2) char corresponding to a single code unit.
*/
export function isLowSurrogate(char: string): boolean;
/**
* Checks whether or not the specified UCS-2 character corresponds to a UTF-16 low surrogate.
*
* @param codeUnit A code unit corresponding to a single UCS-2 char.
*/
export function isLowSurrogate(codeUnit: number): boolean;
export function isLowSurrogate(codeUnit: string|number): boolean {
if(typeof codeUnit == 'string') {
codeUnit = codeUnit.charCodeAt(0);
}
return codeUnit >= 0xDC00 && codeUnit <= 0xDFFF;
}
export function isSentinel(char: string): boolean {
return char == SENTINEL_CODE_UNIT;
}
/**
* Builds a Suggestion based on a Transform corresponding to a predictive-text op.
*
* Assumes that the Transform's `insert` property represents a completed word,
* as models generally delete the whole prefix, replacing it with the full lexical entry.
* @param transform
*/
export function transformToSuggestion(transform: Transform): Suggestion;
export function transformToSuggestion(transform: Transform, p: number): WithOutcome<Suggestion>;
export function transformToSuggestion(transform: Transform, p?: number): Outcome<Suggestion> {
let suggestion: Outcome<Suggestion> = {
transform: transform,
transformId: transform.id,
displayAs: transform.insert
};
if(p === 0 || p) {
suggestion.p = p;
}
return suggestion;
}
export function defaultApplyCasing(casing: CasingForm, text: string): string {
switch(casing) {
case 'lower':
return text.toLowerCase();
case 'upper':
return text.toUpperCase();
case 'initial':
// The length of the first code unit, as measured in code points.
let headUnitLength = 1;
// Is the first character a high surrogate, indicating possible use of UTF-16
// surrogate pairs? Also, is the string long enough for there to BE a pair?
if(text.length > 1 && isHighSurrogate(text.charAt(0))) {
// It's possible, so now we check for low surrogates.
if(isLowSurrogate(text.charCodeAt(1))) {
// We have a surrogate pair; this pair is the 'first' character.
headUnitLength = 2;
}
}
// Capitalizes the first code unit of the string, leaving the rest intact.
return text.substring(0, headUnitLength).toUpperCase() .concat(text.substring(headUnitLength));
}
}

27
common/models/templates/src/index.ts
View file

@ -1,7 +1,22 @@
/// <reference path="./trie-model.ts" />
/// <reference path="./quote-behavior.ts" />
import {
SENTINEL_CODE_UNIT, applyTransform, buildMergedTransform, isHighSurrogate, isLowSurrogate, isSentinel,
transformToSuggestion, defaultApplyCasing
} from "./common.js";
import PriorityQueue, { Comparator } from "./priority-queue.js";
import QuoteBehavior from "./quote-behavior.js";
import { Tokenization, tokenize, getLastPreCaretToken, wordbreak } from "./tokenization.js";
import TrieModel, { TrieModelOptions } from "./trie-model.js";
// Add all namespaces defined here to the global scope:
if (typeof module != 'undefined' && typeof module.exports != 'undefined') {
module.exports['models'] = models;
}
import { extendString } from "@keymanapp/web-utils";
// This package requires our string-extension functions.
extendString();
export {
SENTINEL_CODE_UNIT, applyTransform, buildMergedTransform, isHighSurrogate, isLowSurrogate, isSentinel,
transformToSuggestion, defaultApplyCasing, // "common.ts"
PriorityQueue, Comparator, // "priority-queue.ts"
QuoteBehavior, // "quote-behavior.ts",
Tokenization, tokenize, getLastPreCaretToken, wordbreak, // "tokenization.ts"
TrieModel, TrieModelOptions // "trie-model.ts"
};

395
common/models/templates/src/priority-queue.ts
View file

@ -4,216 +4,215 @@
* Defines a mildly abstracted priority queue implementation.
*/
namespace models {
/**
* Used to compare two instances of a type.
* @returns
* - value < 0 if `a` should come before `b`
* - value > 0 if `b` should come before `a`
* - 0 if they should be treated equally.
*/
export type Comparator<Type> = (a: Type, b: Type) => number;
export default class PriorityQueue<Type> {
private comparator: Comparator<Type>;
private heap: Type[];
/**
* Used to compare two instances of a type.
* @returns
* - value < 0 if `a` should come before `b`
* - value > 0 if `b` should come before `a`
* - 0 if they should be treated equally.
* Constructs an empty priority queue.
* @param comparator A `Comparator` returning negative values when and only when
* the first parameter should precede the second parameter.
*/
export type Comparator<Type> = (a: Type, b: Type) => number;
constructor(comparator: Comparator<Type>, initialEntries: Type[] = []) {
// TODO: We may wish to allow options specifying a limit or threshold for adding
// items to the priority queue. Possibly both.
//
// When that time comes, consider a min-max heap.
// https://en.wikipedia.org/wiki/Min-max_heap
this.comparator = comparator;
export class PriorityQueue<Type> {
this.heap = Array.from(initialEntries);
this.heapify();
}
private comparator: Comparator<Type>;
private heap: Type[];
private static leftChildIndex(index: number): number {
return index * 2 + 1;
}
/**
* Constructs an empty priority queue.
* @param comparator A `Comparator` returning negative values when and only when
* the first parameter should precede the second parameter.
*/
constructor(comparator: Comparator<Type>, initialEntries: Type[] = []) {
// TODO: We may wish to allow options specifying a limit or threshold for adding
// items to the priority queue. Possibly both.
//
// When that time comes, consider a min-max heap.
// https://en.wikipedia.org/wiki/Min-max_heap
this.comparator = comparator;
private static rightChildIndex(index: number): number {
return index * 2 + 2;
}
this.heap = Array.from(initialEntries);
this.heapify();
private static parentIndex(index: number): number {
return Math.floor((index-1)/2);
}
/**
* Maintains internal state, rearranging the internal state until all heap constraints
* are properly satisfied.
* - O(N) when 'heapifying' the whole heap
* - O(N) worst-case for partial heap operations (as part of an enqueueAll)
* <p>
*/
private heapify(): void;
private heapify(start: number, end: number): void;
private heapify(start?: number, end?: number): void {
if(start == undefined || end == undefined) {
this.heapify(0, this.count - 1);
return;
}
private static leftChildIndex(index: number): number {
return index * 2 + 1;
}
// Use of 'indices' here is a bit of a customization.
// At the cost of (temporary) extra storage space, we can more efficiently enqueue
// multiple elements simultaneously.
let queuedIndices: number[] = [];
let lastParent = -1;
private static rightChildIndex(index: number): number {
return index * 2 + 2;
}
private static parentIndex(index: number): number {
return Math.floor((index-1)/2);
}
/**
* Maintains internal state, rearranging the internal state until all heap constraints
* are properly satisfied.
* - O(N) when 'heapifying' the whole heap
* - O(N) worst-case for partial heap operations (as part of an enqueueAll)
* <p>
*/
private heapify(): void;
private heapify(start: number, end: number): void;
private heapify(start?: number, end?: number): void {
if(start == undefined || end == undefined) {
this.heapify(0, this.count - 1);
}
// Use of 'indices' here is a bit of a customization.
// At the cost of (temporary) extra storage space, we can more efficiently enqueue
// multiple elements simultaneously.
let queuedIndices: number[] = [];
let lastParent = -1;
for(let i = end; i >= start; i--) {
let parent = PriorityQueue.parentIndex(i);
if(this.siftDown(i) && parent < start && lastParent != parent) {
// We only need to queue examination for a heap node if its children have changed
// and it isn't already being examined.
queuedIndices.push(parent);
lastParent = parent;
}
}
lastParent = -1;
while(queuedIndices.length > 0) {
let index = queuedIndices.shift();
let parent = PriorityQueue.parentIndex(index);
if(this.siftDown(index) && parent >= 0 && lastParent != parent) {
// We only need to queue examination for a heap node if its children have changed.
queuedIndices.push(parent);
lastParent = parent;
}
for(let i = end; i >= start; i--) {
let parent = PriorityQueue.parentIndex(i);
if(this.siftDown(i) && parent < start && lastParent != parent) {
// We only need to queue examination for a heap node if its children have changed
// and it isn't already being examined.
queuedIndices.push(parent);
lastParent = parent;
}
}
/**
* Returns the number of elements currently held by the priority queue.
*/
get count(): number {
return this.heap.length;
}
/**
* Returns the highest-priority item within the priority queue.
* <p>
* Is O(1).
*/
peek() {
return this.heap[0]; // undefined if it doesn't exist... which is completely correct.
}
/**
* Inserts a new element into the priority queue, placing it in order.
* <p>
* Is O(log N), where N = # of items in the priority queue.
* @param element
*/
enqueue(element: Type) {
let index = this.heap.length;
this.heap.push(element);
let parent = PriorityQueue.parentIndex;
let parentIndex = parent(index);
while(index !== 0 && this.comparator(this.heap[index], this.heap[parentIndex]) < 0) {
let a = this.heap[index];
this.heap[index] = this.heap[parentIndex];
this.heap[parentIndex] = a;
index = parentIndex;
parentIndex = parent(index);
lastParent = -1;
while(queuedIndices.length > 0) {
let index = queuedIndices.shift() as number;
let parent = PriorityQueue.parentIndex(index);
if(this.siftDown(index) && parent >= 0 && lastParent != parent) {
// We only need to queue examination for a heap node if its children have changed.
queuedIndices.push(parent);
lastParent = parent;
}
}
/**
* Efficiently batch-enqueues multiple elements.
* Worst-case is the _better_ of the following:
* - O(`elements.count` + `heap.count`) - large element counts will trigger in-place
* heap reconstruction.
* - O(`elements.count` * log(`heap.count`)) - logarithmic when elements.count << heap.count
* @param elements A group of elements to enqueue simultaneously.
*/
enqueueAll(elements: Type[]) {
if(elements.length == 0) {
return;
}
let firstIndex = this.count
this.heap = this.heap.concat(elements);
let firstParent = PriorityQueue.parentIndex(firstIndex);
// The 'parent' of index 0 will return -1, which is illegal.
this.heapify(firstParent >= 0 ? firstParent : 0, PriorityQueue.parentIndex(this.count-1));
}
/**
* Removes the highest-priority element from the queue, returning it.
* <p>
* Is O(log N), where N = number of items in the priority queue.
*/
dequeue(): Type {
if(this.count == 0) {
return undefined;
}
const root = this.heap[0];
let tail = this.heap.pop();
if(this.heap.length > 0) {
this.heap[0] = tail;
this.siftDown(0);
}
return root;
}
/**
* Compares the entry at the specified index against its children,
* propagating it downward within the heap until heap requirements are specified.
* <p>
* Is O(log N), where N = number of items in the priority queue.
*
* @param index The index of the top-most node that must be examined
* for repositioning.
* @returns `true` if a swap occurred, `false` otherwise.
*/
private siftDown(index: number): boolean {
let leftIndex = PriorityQueue.leftChildIndex(index);
let rightIndex = PriorityQueue.rightChildIndex(index);
let topMostIndex = index;
if(leftIndex < this.heap.length && this.comparator(this.heap[leftIndex], this.heap[topMostIndex]) < 0) {
topMostIndex = leftIndex;
}
if(rightIndex < this.heap.length && this.comparator(this.heap[rightIndex], this.heap[topMostIndex]) < 0) {
topMostIndex = rightIndex;
}
if(topMostIndex != index) {
let a = this.heap[index];
this.heap[index] = this.heap[topMostIndex];
this.heap[topMostIndex] = a;
this.siftDown(topMostIndex);
return true;
} else {
return false;
}
}
/**
* Returns an array containing all entries of the priority queue.
* Altering the returned array will not affect the queue, but mutating
* the array's elements can cause unintended side effects.
*
* This function makes no guarantees on the ordering of the returned elements;
* they will almost certainly be unsorted.
*/
toArray(): Type[] {
return Array.from(this.heap);
}
}
}
/**
* Returns the number of elements currently held by the priority queue.
*/
get count(): number {
return this.heap.length;
}
/**
* Returns the highest-priority item within the priority queue.
* <p>
* Is O(1).
*/
peek() {
return this.heap[0]; // undefined if it doesn't exist... which is completely correct.
}
/**
* Inserts a new element into the priority queue, placing it in order.
* <p>
* Is O(log N), where N = # of items in the priority queue.
* @param element
*/
enqueue(element: Type) {
let index = this.heap.length;
this.heap.push(element);
let parent = PriorityQueue.parentIndex;
let parentIndex = parent(index);
while(index !== 0 && this.comparator(this.heap[index], this.heap[parentIndex]) < 0) {
let a = this.heap[index];
this.heap[index] = this.heap[parentIndex];
this.heap[parentIndex] = a;
index = parentIndex;
parentIndex = parent(index);
}
}
/**
* Efficiently batch-enqueues multiple elements.
* Worst-case is the _better_ of the following:
* - O(`elements.count` + `heap.count`) - large element counts will trigger in-place
* heap reconstruction.
* - O(`elements.count` * log(`heap.count`)) - logarithmic when elements.count << heap.count
* @param elements A group of elements to enqueue simultaneously.
*/
enqueueAll(elements: Type[]) {
if(elements.length == 0) {
return;
}
let firstIndex = this.count
this.heap = this.heap.concat(elements);
let firstParent = PriorityQueue.parentIndex(firstIndex);
// The 'parent' of index 0 will return -1, which is illegal.
this.heapify(firstParent >= 0 ? firstParent : 0, PriorityQueue.parentIndex(this.count-1));
}
/**
* Removes the highest-priority element from the queue, returning it.
* <p>
* Is O(log N), where N = number of items in the priority queue.
*/
dequeue(): Type | undefined {
if(this.count == 0) {
return undefined;
}
const root = this.heap[0];
let tail = this.heap.pop() as Type;
if(this.heap.length > 0) {
this.heap[0] = tail;
this.siftDown(0);
}
return root;
}
/**
* Compares the entry at the specified index against its children,
* propagating it downward within the heap until heap requirements are specified.
* <p>
* Is O(log N), where N = number of items in the priority queue.
*
* @param index The index of the top-most node that must be examined
* for repositioning.
* @returns `true` if a swap occurred, `false` otherwise.
*/
private siftDown(index: number): boolean {
let leftIndex = PriorityQueue.leftChildIndex(index);
let rightIndex = PriorityQueue.rightChildIndex(index);
let topMostIndex = index;
if(leftIndex < this.heap.length && this.comparator(this.heap[leftIndex], this.heap[topMostIndex]) < 0) {
topMostIndex = leftIndex;
}
if(rightIndex < this.heap.length && this.comparator(this.heap[rightIndex], this.heap[topMostIndex]) < 0) {
topMostIndex = rightIndex;
}
if(topMostIndex != index) {
let a = this.heap[index];
this.heap[index] = this.heap[topMostIndex];
this.heap[topMostIndex] = a;
this.siftDown(topMostIndex);
return true;
} else {
return false;
}
}
/**
* Returns an array containing all entries of the priority queue.
* Altering the returned array will not affect the queue, but mutating
* the array's elements can cause unintended side effects.
*
* This function makes no guarantees on the ordering of the returned elements;
* they will almost certainly be unsorted.
*/
toArray(): Type[] {
return Array.from(this.heap);
}
}

72
common/models/templates/src/quote-behavior.ts
View file

@ -1,38 +1,38 @@
namespace models {
export enum QuoteBehavior {
noQuotes = "no-quotes",
useQuotes = "use-quotes",
default = "default-quotes"
}
// For an "enum/namespace merge". See https://www.typescriptlang.org/docs/handbook/declaration-merging.html#merging-namespaces-with-classes
export namespace QuoteBehavior {
/**
* Applies the specified quote behavior to the provided `Transform`, mutating it as appropriate.
* @param transform The Transform to be mutated
* @param punctuation The active `LexicalModelPunctuation` settings
* @param defaultTo The default quote behavior to use (in case the current value is `.default`)
*/
export function apply(behavior: QuoteBehavior, text: string, punctuation: LexicalModelPunctuation, defaultTo: QuoteBehavior): string {
if(defaultTo == QuoteBehavior.default || !defaultTo) {
throw "Specified quote behavior may be ambiguous - default behavior not specified (may not be .default)";
}
if(behavior == QuoteBehavior.default) {
behavior = defaultTo;
}
switch(behavior) {
case QuoteBehavior.noQuotes:
return text;
case QuoteBehavior.useQuotes:
let {open, close} = punctuation.quotesForKeepSuggestion;
// This part's simple enough, at least.
return open + text + close;
default:
throw "Unsupported quote behavior state detected; implementation missing!";
}
export enum QuoteBehavior {
noQuotes = "no-quotes",
useQuotes = "use-quotes",
default = "default-quotes"
}
// For an "enum/namespace merge". See https://www.typescriptlang.org/docs/handbook/declaration-merging.html#merging-namespaces-with-classes
export namespace QuoteBehavior {
/**
* Applies the specified quote behavior to the provided `Transform`, mutating it as appropriate.
* @param transform The Transform to be mutated
* @param punctuation The active `LexicalModelPunctuation` settings
* @param defaultTo The default quote behavior to use (in case the current value is `.default`)
*/
export function apply(behavior: QuoteBehavior, text: string, punctuation: LexicalModelPunctuation, defaultTo: QuoteBehavior): string {
if(defaultTo == QuoteBehavior.default || !defaultTo) {
throw "Specified quote behavior may be ambiguous - default behavior not specified (may not be .default)";
}
if(behavior == QuoteBehavior.default) {
behavior = defaultTo;
}
switch(behavior) {
case QuoteBehavior.noQuotes:
return text;
case QuoteBehavior.useQuotes:
let {open, close} = punctuation.quotesForKeepSuggestion;
// This part's simple enough, at least.
return open + text + close;
default:
throw "Unsupported quote behavior state detected; implementation missing!";
}
}
}
}
export default QuoteBehavior;

234
common/models/templates/src/tokenization.ts
View file

@ -1,129 +1,127 @@
// While we _could_ define this within @keymanapp/models-wordbreakers instead, it's probably
// better to leave that package as _just_ the wordbreakers.
// better to leave that package as _just_ the wordbreakers.
namespace models {
export interface Tokenization {
/**
* An array of tokens to the left of the caret. If the caret is in the middle of a token,
* only the part to the left of the caret is included.
*/
left: USVString[],
/**
* An array of tokens to the right of the caret. If the caret is in the middle of a token,
* only the part to the right of the caret is included.
*/
right: USVString[],
/**
* A flag indicating whether or not the caret's position in the context caused a token
* to be split. If `true`, the last entry of `left` is from the same token as the first
* entry of `right`.
*/
caretSplitsToken: boolean
}
export function tokenize(wordBreaker: WordBreakingFunction, context?: Partial<Context>): Tokenization {
context = context || {
left: undefined,
startOfBuffer: undefined,
endOfBuffer: undefined
};
let leftSpans = wordBreaker(context.left || '') || [];
let rightSpans = wordBreaker(context.right || '') || [];
let leftTail: Span;
if(leftSpans.length > 0) {
leftTail = leftSpans[leftSpans.length - 1];
}
// Handle any desired special handling for directly-pre-caret scenarios - where for this
// _specific_ context, we should not make a token division where one normally would exist otherwise.
//
// One notable example: word-final apostrophe is tokenized separate from preceding text, but
// word-internal apostrophe is treated as part of the same word (i.e, English contractions).
// But, if the user is editing text and the caret is directly after a caret, there's a notable
// chance they may in the middle of typing a contraction. Refer to
// https://github.com/keymanapp/keyman/issues/6572.
if(leftSpans.length > 1) {
const leftTailBase = leftSpans[leftSpans.length - 2];
// If the final two pre-caret spans are adjacent - without intervening whitespace...
if(leftTailBase.end == leftTail.start) {
// Ideal: if(leftTailBase is standard-char-class && leftTail is single-quote-class)
// But we don't have character class access here; it's all wordbreaker-function internal.
// Upon inspection of the wordbreaker data definitions... the single-quote-class is ONLY "'".
// So... we'll just be lazy for now and append the `'`.
if(leftTail.text == "'") {
let mergedSpan: Span = {
text: leftTailBase.text + leftTail.text,
start: leftTailBase.start,
end: leftTail.end,
length: leftTailBase.length + leftTail.length
};
leftSpans.pop(); // leftTail
leftSpans.pop(); // leftTailBase
leftSpans.push(mergedSpan);
leftTail = mergedSpan; // don't forget to update the `leftTail` Span!
}
}
}
// With any 'detokenization' cases already handled, we may now begin to build the return object.
let tokenization: Tokenization = {
left: leftSpans.map(span => span.text),
right: rightSpans.map(span => span.text),
// A default initialization of the value.
caretSplitsToken: false
};
// Now the hard part - determining whether or not the caret caused a token split.
if(leftSpans.length > 0 && rightSpans.length > 0) {
let rightHead = rightSpans[0];
// If tokenization includes all characters on each side of the caret,
// we have a good candidate for a caret-splitting scenario.
let leftSuffixWordbreak = leftTail.end != context.left.length;
let rightPrefixWordbreak = rightHead.start != 0;
if(leftSuffixWordbreak || rightPrefixWordbreak) {
// Bypass the final test, as we already know the caret didn't split a token.
// (The tokenization process already removed characters between the two.)
return tokenization;
}
// Worth note - some languages don't use wordbreaking characters. So, a final check:
//
// Does the wordbreaker split a merge of the 'two center' tokens?
// If not, then the caret is responsible for the split.
if(wordBreaker(leftTail.text + rightHead.text).length == 1) {
tokenization.caretSplitsToken = true;
}
}
return tokenization;
}
export interface Tokenization {
/**
* An array of tokens to the left of the caret. If the caret is in the middle of a token,
* only the part to the left of the caret is included.
*/
left: USVString[],
/**
* Get the last word of the phrase before the caret or nothing.
* @param fullLeftContext the entire left context of the string.
* An array of tokens to the right of the caret. If the caret is in the middle of a token,
* only the part to the right of the caret is included.
*/
export function getLastPreCaretToken(wordBreaker: WordBreakingFunction, context: Context): string {
let tokenization = tokenize(wordBreaker, context);
if (tokenization.left.length > 0) {
return tokenization.left.pop();
right: USVString[],
/**
* A flag indicating whether or not the caret's position in the context caused a token
* to be split. If `true`, the last entry of `left` is from the same token as the first
* entry of `right`.
*/
caretSplitsToken: boolean
}
export function tokenize(wordBreaker: WordBreakingFunction, context?: Partial<Context>): Tokenization {
context = context || {
left: undefined,
startOfBuffer: undefined,
endOfBuffer: undefined
};
let leftSpans = wordBreaker(context.left || '') || [];
let rightSpans = wordBreaker(context.right || '') || [];
let leftTail: Span;
if(leftSpans.length > 0) {
leftTail = leftSpans[leftSpans.length - 1];
}
// Handle any desired special handling for directly-pre-caret scenarios - where for this
// _specific_ context, we should not make a token division where one normally would exist otherwise.
//
// One notable example: word-final apostrophe is tokenized separate from preceding text, but
// word-internal apostrophe is treated as part of the same word (i.e, English contractions).
// But, if the user is editing text and the caret is directly after a caret, there's a notable
// chance they may in the middle of typing a contraction. Refer to
// https://github.com/keymanapp/keyman/issues/6572.
if(leftSpans.length > 1) {
const leftTailBase = leftSpans[leftSpans.length - 2];
// If the final two pre-caret spans are adjacent - without intervening whitespace...
if(leftTailBase.end == leftTail!.start) {
// Ideal: if(leftTailBase is standard-char-class && leftTail is single-quote-class)
// But we don't have character class access here; it's all wordbreaker-function internal.
// Upon inspection of the wordbreaker data definitions... the single-quote-class is ONLY "'".
// So... we'll just be lazy for now and append the `'`.
if(leftTail!.text == "'") {
let mergedSpan: Span = {
text: leftTailBase.text + leftTail!.text,
start: leftTailBase.start,
end: leftTail!.end,
length: leftTailBase.length + leftTail!.length
};
leftSpans.pop(); // leftTail
leftSpans.pop(); // leftTailBase
leftSpans.push(mergedSpan);
leftTail = mergedSpan; // don't forget to update the `leftTail` Span!
}
}
}
// With any 'detokenization' cases already handled, we may now begin to build the return object.
let tokenization: Tokenization = {
left: leftSpans.map(span => span.text),
right: rightSpans.map(span => span.text),
// A default initialization of the value.
caretSplitsToken: false
};
// Now the hard part - determining whether or not the caret caused a token split.
if(leftSpans.length > 0 && rightSpans.length > 0) {
let rightHead = rightSpans[0];
// If tokenization includes all characters on each side of the caret,
// we have a good candidate for a caret-splitting scenario.
let leftSuffixWordbreak = leftTail!.end != context.left!.length;
let rightPrefixWordbreak = rightHead.start != 0;
if(leftSuffixWordbreak || rightPrefixWordbreak) {
// Bypass the final test, as we already know the caret didn't split a token.
// (The tokenization process already removed characters between the two.)
return tokenization;
}
return '';
// Worth note - some languages don't use wordbreaking characters. So, a final check:
//
// Does the wordbreaker split a merge of the 'two center' tokens?
// If not, then the caret is responsible for the split.
if(wordBreaker(leftTail!.text + rightHead.text).length == 1) {
tokenization.caretSplitsToken = true;
}
}
// While it is currently identical to getLastWord, this may change in the future.
// It's best not to write ourselves into a corner on this one, as disambiguating later
// would likely be pretty painful.
export function wordbreak(wordBreaker: WordBreakingFunction, context: Context): USVString {
return getLastPreCaretToken(wordBreaker, context);
return tokenization;
}
/**
* Get the last word of the phrase before the caret or nothing.
* @param fullLeftContext the entire left context of the string.
*/
export function getLastPreCaretToken(wordBreaker: WordBreakingFunction, context: Context): string {
let tokenization = tokenize(wordBreaker, context);
if (tokenization.left.length > 0) {
return tokenization.left.pop() as string;
}
return '';
}
// While it is currently identical to getLastWord, this may change in the future.
// It's best not to write ourselves into a corner on this one, as disambiguating later
// would likely be pretty painful.
export function wordbreak(wordBreaker: WordBreakingFunction, context: Context): USVString {
return getLastPreCaretToken(wordBreaker, context);
}

928
common/models/templates/src/trie-model.ts
View file

@ -24,10 +24,14 @@
// Worth noting: we're starting to get quite a 'library' of common model/LMLayer functionality.
// Should probably make a 'lm-utils' submodule.
/// <reference path="common.ts" />
/// <reference path="priority-queue.ts" />
/// <reference path="tokenization.ts" />
/// <reference types="@keymanapp/models-types" />
// Allows the kmwstring bindings to resolve.
import { extendString } from "@keymanapp/web-utils";
import { default as defaultWordBreaker } from "@keymanapp/models-wordbreakers";
import { applyTransform, isHighSurrogate, isSentinel, SENTINEL_CODE_UNIT, transformToSuggestion } from "./common.js";
import { getLastPreCaretToken } from "./tokenization.js";
import PriorityQueue from "./priority-queue.js";
/**
* @file trie-model.ts
@ -35,524 +39,510 @@
* Defines a simple word list (unigram) model.
*/
namespace models {
/** Upper bound on the amount of suggestions to generate. */
const MAX_SUGGESTIONS = 12;
/** Upper bound on the amount of suggestions to generate. */
const MAX_SUGGESTIONS = 12;
/**
* Additional arguments to pass into the model, in addition to the model
* parameters themselves.
*/
export interface TrieModelOptions {
/**
* How to break words in a phrase.
*/
wordBreaker?: WordBreakingFunction;
/**
* This should simplify a search term into a key.
*/
searchTermToKey?: (searchTerm: string) => string;
/**
* Additional arguments to pass into the model, in addition to the model
* parameters themselves.
* Indicates that the model's written form has 'casing' behavior.
*/
interface TrieModelOptions {
/**
* How to break words in a phrase.
*/
wordBreaker?: WordBreakingFunction;
/**
* This should simplify a search term into a key.
*/
searchTermToKey?: (searchTerm: string) => string;
languageUsesCasing?: boolean;
/**
* Indicates that the model's written form has 'casing' behavior.
*/
languageUsesCasing?: boolean;
/**
* Specifies a function used to apply the language's casing rules to a word.
*/
applyCasing?: CasingFunction;
/**
* Specifies a function used to apply the language's casing rules to a word.
*/
applyCasing?: CasingFunction;
/**
* Any punctuation to expose to the user.
*/
punctuation?: LexicalModelPunctuation;
}
/**
* Any punctuation to expose to the user.
*/
punctuation?: LexicalModelPunctuation;
/**
* Used to determine the probability of an entry from the trie.
*/
type TextWithProbability = {
text: string;
// TODO: use negative-log scaling instead?
p: number; // real-number weight, from 0 to 1
}
/**
* @class TrieModel
*
* Defines a trie-backed word list model, or the unigram model.
* Unigram models throw away all preceding words, and search
* for the next word exclusively. As such, they can perform simple
* prefix searches within words, however they are not very good
* at predicting the next word.
*/
export default class TrieModel implements LexicalModel {
configuration?: Configuration;
private _trie: Trie;
readonly breakWords: WordBreakingFunction;
readonly punctuation?: LexicalModelPunctuation;
readonly languageUsesCasing?: boolean;
readonly applyCasing?: CasingFunction;
constructor(trieData: {root: Node, totalWeight: number}, options: TrieModelOptions = {}) {
this.languageUsesCasing = options.languageUsesCasing;
this.applyCasing = options.applyCasing;
this._trie = new Trie(
trieData['root'],
trieData['totalWeight'],
options.searchTermToKey as Wordform2Key || defaultSearchTermToKey
);
this.breakWords = options.wordBreaker || defaultWordBreaker;
this.punctuation = options.punctuation;
}
/**
* Used to determine the probability of an entry from the trie.
*/
type TextWithProbability = {
text: string;
// TODO: use negative-log scaling instead?
p: number; // real-number weight, from 0 to 1
configure(capabilities: Capabilities): Configuration {
return this.configuration = {
leftContextCodePoints: capabilities.maxLeftContextCodePoints,
rightContextCodePoints: capabilities.maxRightContextCodePoints ?? 0
};
}
/**
* @class TrieModel
*
* Defines a trie-backed word list model, or the unigram model.
* Unigram models throw away all preceding words, and search
* for the next word exclusively. As such, they can perform simple
* prefix searches within words, however they are not very good
* at predicting the next word.
*/
export class TrieModel implements LexicalModel {
configuration: Configuration;
private _trie: Trie;
readonly breakWords: WordBreakingFunction;
readonly punctuation?: LexicalModelPunctuation;
readonly languageUsesCasing?: boolean;
toKey(text: USVString): USVString {
return this._trie.toKey(text);
}
readonly applyCasing?: CasingFunction;
constructor(trieData: object, options: TrieModelOptions = {}) {
this.languageUsesCasing = options.languageUsesCasing;
this.applyCasing = options.applyCasing;
this._trie = new Trie(
trieData['root'],
trieData['totalWeight'],
options.searchTermToKey as Wordform2Key || defaultSearchTermToKey
);
this.breakWords = options.wordBreaker || getDefaultWordBreaker();
this.punctuation = options.punctuation;
}
configure(capabilities: Capabilities): Configuration {
return this.configuration = {
leftContextCodePoints: capabilities.maxLeftContextCodePoints,
rightContextCodePoints: capabilities.maxRightContextCodePoints
};
}
toKey(text: USVString): USVString {
return this._trie.toKey(text);
}
predict(transform: Transform, context: Context): Distribution<Suggestion> {
// Special-case the empty buffer/transform: return the top suggestions.
if (!transform.insert && !context.left && !context.right && context.startOfBuffer && context.endOfBuffer) {
return makeDistribution(this._trie.firstN(MAX_SUGGESTIONS).map(({text, p}) => ({
transform: {
insert: text,
deleteLeft: 0
},
displayAs: text,
p: p
})));
}
// Compute the results of the keystroke:
let newContext = models.applyTransform(transform, context);
// Computes the different in word length after applying the transform above.
let leftDelOffset = transform.deleteLeft - transform.insert.kmwLength();
// All text to the left of the cursor INCLUDING anything that has
// just been typed.
let prefix = models.getLastPreCaretToken(this.breakWords, newContext);
// Return suggestions from the trie.
return makeDistribution(this._trie.lookup(prefix).map(({text, p}) =>
models.transformToSuggestion({
predict(transform: Transform, context: Context): Distribution<Suggestion> {
// Special-case the empty buffer/transform: return the top suggestions.
if (!transform.insert && !context.left && !context.right && context.startOfBuffer && context.endOfBuffer) {
return makeDistribution(this._trie.firstN(MAX_SUGGESTIONS).map(({text, p}) => ({
transform: {
insert: text,
// Delete whatever the prefix that the user wrote.
deleteLeft: leftDelOffset + prefix.kmwLength()
// Note: a separate capitalization/orthography engine can take this
// result and transform it as needed.
deleteLeft: 0
},
p
)));
/* Helper */
function makeDistribution(suggestions: WithOutcome<Suggestion>[]): Distribution<Suggestion> {
let distribution: Distribution<Suggestion> = [];
for(let s of suggestions) {
distribution.push({sample: s, p: s.p});
}
return distribution;
}
displayAs: text,
p: p
})));
}
get wordbreaker(): WordBreakingFunction {
return this.breakWords;
}
// Compute the results of the keystroke:
let newContext = applyTransform(transform, context);
public traverseFromRoot(): LexiconTraversal {
return new TrieModel.Traversal(this._trie['root'], '');
}
// Computes the different in word length after applying the transform above.
let leftDelOffset = transform.deleteLeft - transform.insert.kmwLength();
private static Traversal = class implements LexiconTraversal {
/**
* The lexical prefix corresponding to the current traversal state.
*/
prefix: String;
// All text to the left of the cursor INCLUDING anything that has
// just been typed.
let prefix = getLastPreCaretToken(this.breakWords, newContext);
/**
* The current traversal node. Serves as the 'root' of its own sub-Trie,
* and we cannot navigate back to its parent.
*/
root: Node;
// Return suggestions from the trie.
return makeDistribution(this._trie.lookup(prefix).map(({text, p}) =>
transformToSuggestion({
insert: text,
// Delete whatever the prefix that the user wrote.
deleteLeft: leftDelOffset + prefix.kmwLength()
// Note: a separate capitalization/orthography engine can take this
// result and transform it as needed.
},
p
)));
constructor(root: Node, prefix: string) {
this.root = root;
this.prefix = prefix;
/* Helper */
function makeDistribution(suggestions: WithOutcome<Suggestion>[]): Distribution<Suggestion> {
let distribution: Distribution<Suggestion> = [];
for(let s of suggestions) {
distribution.push({sample: s, p: s.p});
}
*children(): Generator<{char: string, traversal: () => LexiconTraversal}> {
let root = this.root;
return distribution;
}
}
if(root.type == 'internal') {
for(let entry of root.values) {
let entryNode = root.children[entry];
get wordbreaker(): WordBreakingFunction {
return this.breakWords;
}
// UTF-16 astral plane check.
if(models.isHighSurrogate(entry)) {
// First code unit of a UTF-16 code point.
// For now, we'll just assume the second always completes such a char.
//
// Note: Things get nasty here if this is only sometimes true; in the future,
// we should compile-time enforce that this assumption is always true if possible.
if(entryNode.type == 'internal') {
let internalNode = entryNode;
for(let lowSurrogate of internalNode.values) {
let prefix = this.prefix + entry + lowSurrogate;
yield {
char: entry + lowSurrogate,
traversal: function() { return new TrieModel.Traversal(internalNode.children[lowSurrogate], prefix) }
}
}
} else {
// Determine how much of the 'leaf' entry has no Trie nodes, emulate them.
let fullText = entryNode.entries[0].key;
entry = entry + fullText[this.prefix.length + 1]; // The other half of the non-BMP char.
let prefix = this.prefix + entry;
public traverseFromRoot(): LexiconTraversal {
return new TrieModel.Traversal(this._trie['root'], '');
}
private static Traversal = class implements LexiconTraversal {
/**
* The lexical prefix corresponding to the current traversal state.
*/
prefix: String;
/**
* The current traversal node. Serves as the 'root' of its own sub-Trie,
* and we cannot navigate back to its parent.
*/
root: Node;
constructor(root: Node, prefix: string) {
this.root = root;
this.prefix = prefix;
}
*children(): Generator<{char: string, traversal: () => LexiconTraversal}> {
let root = this.root;
if(root.type == 'internal') {
for(let entry of root.values) {
let entryNode = root.children[entry];
// UTF-16 astral plane check.
if(isHighSurrogate(entry)) {
// First code unit of a UTF-16 code point.
// For now, we'll just assume the second always completes such a char.
//
// Note: Things get nasty here if this is only sometimes true; in the future,
// we should compile-time enforce that this assumption is always true if possible.
if(entryNode.type == 'internal') {
let internalNode = entryNode;
for(let lowSurrogate of internalNode.values) {
let prefix = this.prefix + entry + lowSurrogate;
yield {
char: entry,
traversal: function () {return new TrieModel.Traversal(entryNode, prefix)}
char: entry + lowSurrogate,
traversal: function() { return new TrieModel.Traversal(internalNode.children[lowSurrogate], prefix) }
}
}
} else if(models.isSentinel(entry)) {
continue;
} else if(!entry) {
// Prevent any accidental 'null' or 'undefined' entries from having an effect.
continue;
} else {
// Determine how much of the 'leaf' entry has no Trie nodes, emulate them.
let fullText = entryNode.entries[0].key;
entry = entry + fullText[this.prefix.length + 1]; // The other half of the non-BMP char.
let prefix = this.prefix + entry;
yield {
char: entry,
traversal: function() { return new TrieModel.Traversal(entryNode, prefix)}
traversal: function () {return new TrieModel.Traversal(entryNode, prefix)}
}
}
}
return;
} else { // type == 'leaf'
let prefix = this.prefix;
let children = root.entries.filter(function(entry) {
return entry.key != prefix && prefix.length < entry.key.length;
})
for(let {key} of children) {
let nodeKey = key[prefix.length];
if(models.isHighSurrogate(nodeKey)) {
// Merge the other half of an SMP char in!
nodeKey = nodeKey + key[prefix.length+1];
}
} else if(isSentinel(entry)) {
continue;
} else if(!entry) {
// Prevent any accidental 'null' or 'undefined' entries from having an effect.
continue;
} else {
let prefix = this.prefix + entry;
yield {
char: nodeKey,
traversal: function() { return new TrieModel.Traversal(root, prefix + nodeKey)}
char: entry,
traversal: function() { return new TrieModel.Traversal(entryNode, prefix)}
}
};
return;
}
}
return;
} else { // type == 'leaf'
let prefix = this.prefix;
let children = root.entries.filter(function(entry) {
return entry.key != prefix && prefix.length < entry.key.length;
})
for(let {key} of children) {
let nodeKey = key[prefix.length];
if(isHighSurrogate(nodeKey)) {
// Merge the other half of an SMP char in!
nodeKey = nodeKey + key[prefix.length+1];
}
yield {
char: nodeKey,
traversal: function() { return new TrieModel.Traversal(root, prefix + nodeKey)}
}
};
return;
}
}
get entries(): string[] {
if(this.root.type == 'leaf') {
let prefix = this.prefix;
let matches = this.root.entries.filter(function(entry) {
return entry.key == prefix;
});
get entries(): string[] {
if(this.root.type == 'leaf') {
let prefix = this.prefix;
let matches = this.root.entries.filter(function(entry) {
return entry.key == prefix;
});
return matches.map(function(value) { return value.content });
return matches.map(function(value) { return value.content });
} else {
let matchingLeaf = this.root.children[SENTINEL_CODE_UNIT];
if(matchingLeaf && matchingLeaf.type == 'leaf') {
return matchingLeaf.entries.map(function(value) { return value.content });
} else {
let matchingLeaf = this.root.children[models.SENTINEL_CODE_UNIT];
if(matchingLeaf && matchingLeaf.type == 'leaf') {
return matchingLeaf.entries.map(function(value) { return value.content });
} else {
return [];
}
return [];
}
}
}
};
}
};
/////////////////////////////////////////////////////////////////////////////////
// What remains in this file is the trie implementation proper. Note: to //
// reduce bundle size, any functions/methods related to creating the trie have //
// been removed. //
/////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////////
// What remains in this file is the trie implementation proper. Note: to //
// reduce bundle size, any functions/methods related to creating the trie have //
// been removed. //
/////////////////////////////////////////////////////////////////////////////////
/**
* An **opaque** type for a string that is exclusively used as a search key in
* the trie. There should be a function that converts arbitrary strings
* (queries) and converts them into a standard search key for a given language
* model.
*
* Fun fact: This opaque type has ALREADY saved my bacon and found a bug!
*/
type SearchKey = string & { _: 'SearchKey'};
/**
* The priority queue will always pop the most weighted item. There can only
* be two kinds of items right now: nodes, and entries; both having a weight
* attribute.
*/
type Weighted = Node | Entry;
/**
* A function that converts a string (word form or query) into a search key
* (secretly, this is also a string).
*/
interface Wordform2Key {
(wordform: string): SearchKey;
}
// The following trie implementation has been (heavily) derived from trie-ing
// by Conrad Irwin.
// trie-ing is copyright (C) 2015–2017 Conrad Irwin.
// Distributed under the terms of the MIT license:
// https://github.com/ConradIrwin/trie-ing/blob/df55d7af7068d357829db9e0a7faa8a38add1d1d/LICENSE
type Node = InternalNode | Leaf;
/**
* An internal node in the trie. Internal nodes NEVER contain entries; if an
* internal node should contain an entry, then it has a dummy leaf node (see
* below), that can be accessed by node.children["\uFDD0"].
*/
interface InternalNode {
type: 'internal';
weight: number;
/** Maintains the keys of children in descending order of weight. */
values: string[]; // TODO: As an optimization, "values" can be a single string!
/**
* Maps a single UTF-16 code unit to a child node in the trie. This child
* node may be a leaf or an internal node. The keys of this object are kept
* in sorted order in the .values array.
*/
children: { [codeunit: string]: Node };
}
/** Only leaf nodes actually contain entries (i.e., the words proper). */
interface Leaf {
type: 'leaf';
weight: number;
entries: Entry[];
}
/**
* An entry in the prefix trie (stored in leaf nodes exclusively!)
*/
interface Entry {
/** The actual word form, stored in the trie. */
content: string;
/** A search key that usually simplifies the word form, for ease of search. */
key: SearchKey;
weight: number;
}
/**
* Wrapper class for the trie and its nodes.
*/
class Trie {
private root: Node;
/** The total weight of the entire trie. */
private totalWeight: number;
/**
* Converts arbitrary strings to a search key. The trie is built up of
* search keys; not each entry's word form!
*/
toKey: Wordform2Key;
constructor(root: Node, totalWeight: number, wordform2key: Wordform2Key) {
this.root = root;
this.toKey = wordform2key;
this.totalWeight = totalWeight;
}
/**
* An **opaque** type for a string that is exclusively used as a search key in
* the trie. There should be a function that converts arbitrary strings
* (queries) and converts them into a standard search key for a given language
* model.
* Lookups an arbitrary prefix (a query) in the trie. Returns the top 3
* results in sorted order.
*
* Fun fact: This opaque type has ALREADY saved my bacon and found a bug!
* @param prefix
*/
type SearchKey = string & { _: 'SearchKey'};
/**
* The priority queue will always pop the most weighted item. There can only
* be two kinds of items right now: nodes, and entries; both having a weight
* attribute.
*/
type Weighted = Node | Entry;
/**
* A function that converts a string (word form or query) into a search key
* (secretly, this is also a string).
*/
interface Wordform2Key {
(wordform: string): SearchKey;
}
// The following trie implementation has been (heavily) derived from trie-ing
// by Conrad Irwin.
// trie-ing is copyright (C) 2015–2017 Conrad Irwin.
// Distributed under the terms of the MIT license:
// https://github.com/ConradIrwin/trie-ing/blob/df55d7af7068d357829db9e0a7faa8a38add1d1d/LICENSE
type Node = InternalNode | Leaf;
/**
* An internal node in the trie. Internal nodes NEVER contain entries; if an
* internal node should contain an entry, then it has a dummy leaf node (see
* below), that can be accessed by node.children["\uFDD0"].
*/
interface InternalNode {
type: 'internal';
weight: number;
/** Maintains the keys of children in descending order of weight. */
values: string[]; // TODO: As an optimization, "values" can be a single string!
/**
* Maps a single UTF-16 code unit to a child node in the trie. This child
* node may be a leaf or an internal node. The keys of this object are kept
* in sorted order in the .values array.
*/
children: { [codeunit: string]: Node };
}
/** Only leaf nodes actually contain entries (i.e., the words proper). */
interface Leaf {
type: 'leaf';
weight: number;
entries: Entry[];
}
/**
* An entry in the prefix trie (stored in leaf nodes exclusively!)
*/
interface Entry {
/** The actual word form, stored in the trie. */
content: string;
/** A search key that usually simplifies the word form, for ease of search. */
key: SearchKey;
weight: number;
}
/**
* Wrapper class for the trie and its nodes.
*/
class Trie {
private root: Node;
/** The total weight of the entire trie. */
private totalWeight: number;
/**
* Converts arbitrary strings to a search key. The trie is built up of
* search keys; not each entry's word form!
*/
toKey: Wordform2Key;
constructor(root: Node, totalWeight: number, wordform2key: Wordform2Key) {
this.root = root;
this.toKey = wordform2key;
this.totalWeight = totalWeight;
lookup(prefix: string): TextWithProbability[] {
let searchKey = this.toKey(prefix);
let lowestCommonNode = findPrefix(this.root, searchKey);
if (lowestCommonNode === null) {
return [];
}
/**
* Lookups an arbitrary prefix (a query) in the trie. Returns the top 3
* results in sorted order.
*
* @param prefix
*/
lookup(prefix: string): TextWithProbability[] {
let searchKey = this.toKey(prefix);
let lowestCommonNode = findPrefix(this.root, searchKey);
if (lowestCommonNode === null) {
return [];
}
return getSortedResults(lowestCommonNode, searchKey, this.totalWeight);
}
/**
* Returns the top N suggestions from the trie.
* @param n How many suggestions, maximum, to return.
*/
firstN(n: number): TextWithProbability[] {
return getSortedResults(this.root, '' as SearchKey, this.totalWeight, n);
}
return getSortedResults(lowestCommonNode, searchKey, this.totalWeight);
}
/**
* Finds the deepest descendent in the trie with the given prefix key.
*
* This means that a search in the trie for a given prefix has a best-case
* complexity of O(m) where m is the length of the prefix.
*
* @param key The prefix to search for.
* @param index The index in the prefix. Initially 0.
* Returns the top N suggestions from the trie.
* @param n How many suggestions, maximum, to return.
*/
function findPrefix(node: Node, key: SearchKey, index: number = 0): Node | null {
// An important note - the Trie itself is built on a per-JS-character basis,
// not on a UTF-8 character-code basis.
if (node.type === 'leaf' || index === key.length) {
return node;
}
// So, for SMP models, we need to match each char of the supplementary pair
// in sequence. Each has its own node in the Trie.
let char = key[index];
if (node.children[char]) {
return findPrefix(node.children[char], key, index + 1);
}
return null;
}
/**
* Returns all entries matching the given prefix, in descending order of
* weight.
*
* @param prefix the prefix to match.
* @param results the current results
* @param queue
*/
function getSortedResults(node: Node, prefix: SearchKey, N: number, limit = MAX_SUGGESTIONS): TextWithProbability[] {
let queue = new PriorityQueue(function(a: Weighted, b: Weighted) {
// In case of Trie compilation issues that emit `null` or `undefined`
return (b ? b.weight : 0) - (a ? a.weight : 0);
});
let results: TextWithProbability[] = [];
if (node.type === 'leaf') {
// Assuming the values are sorted, we can just add all of the values in the
// leaf, until we reach the limit.
for (let item of node.entries) {
// String.startsWith is not supported on certain Android (5.0) devices we wish to support.
// Requires a minimum of Chrome 36, as opposed to 5.0's default of 35.
if (item.key.indexOf(prefix) == 0) {
let { content, weight } = item;
results.push({
text: content,
p: weight / N
});
if (results.length >= limit) {
return results;
}
}
}
} else {
queue.enqueue(node);
let next: Weighted;
while (next = queue.dequeue()) {
if (isNode(next)) {
// When a node is next up in the queue, that means that next least
// likely suggestion is among its decsendants.
// So we search all of its descendants!
if (next.type === 'leaf') {
queue.enqueueAll(next.entries);
} else {
// XXX: alias `next` so that TypeScript can be SURE that internal is
// in fact an internal node. Because of the callback binding to the
// original definition of node (i.e., a Node | Entry), this will not
// type-check otherwise.
let internal = next;
queue.enqueueAll(next.values.map(char => {
return internal.children[char];
}));
}
} else {
// When an entry is up next in the queue, we just add its contents to
// the results!
results.push({
text: next.content,
p: next.weight / N
});
if (results.length >= limit) {
return results;
}
}
}
}
return results;
}
/** TypeScript type guard that returns whether the thing is a Node. */
function isNode(x: Entry | Node): x is Node {
return 'type' in x;
}
/**
* Converts wordforms into an indexable form. It does this by
* normalizing into NFD, removing diacritics, and then converting
* the result to lowercase.
*
* This is a very naïve implementation, that I only think will work on
* some languages that use the Latin script. As of 2020-04-08, only
* 4 out of 11 (36%) of published language models use the Latin script,
* so this might not actually be a great default.
*
* This uses String.prototype.normalize() to convert normalize into NFD.
* NFD is an easy way to separate a Latin character from its diacritics;
* Even then, some Latin-based orthographies use code points that,
* under NFD normalization, do NOT decompose into an ASCII letter and a
* combining diacritical mark (e.g., SENĆOŦEN).
*
* Use this only in early iterations of the model. For a production lexical
* model, you SHOULD write/generate your own key function, tailored to your
* language.
*/
function defaultSearchTermToKey(wordform: string): SearchKey {
/**
* N.B.: this is (slightly) DIFFERENT than the version in
* keymanapp/lexical-model-compiler/build-trie
* as this is for compatibility for models built
* BEFORE the searchTermToKey function was bundled with
* all models.
*
* This compatibility version lowercases AFTER removing diacritics;
* the new version (bundled in future models) lowercases,
* NFD normalizes, THEN removes diacritics.
*/
return wordform
.normalize('NFD')
// Remove all combining diacritics (if input is in NFD)
// common to Latin-orthographies.
.replace(/[\u0300-\u036f]/g, '')
.toLowerCase() as SearchKey;
}
function getDefaultWordBreaker() {
let namespace: {};
// @ts-ignore
if (typeof wordBreakers !== 'undefined') {
// @ts-ignore
namespace = wordBreakers;
} else {
namespace = require('@keymanapp/models-wordBreakers').wordBreakers;
}
return namespace['default'];
firstN(n: number): TextWithProbability[] {
return getSortedResults(this.root, '' as SearchKey, this.totalWeight, n);
}
}
/**
* Finds the deepest descendent in the trie with the given prefix key.
*
* This means that a search in the trie for a given prefix has a best-case
* complexity of O(m) where m is the length of the prefix.
*
* @param key The prefix to search for.
* @param index The index in the prefix. Initially 0.
*/
function findPrefix(node: Node, key: SearchKey, index: number = 0): Node | null {
// An important note - the Trie itself is built on a per-JS-character basis,
// not on a UTF-8 character-code basis.
if (node.type === 'leaf' || index === key.length) {
return node;
}
// So, for SMP models, we need to match each char of the supplementary pair
// in sequence. Each has its own node in the Trie.
let char = key[index];
if (node.children[char]) {
return findPrefix(node.children[char], key, index + 1);
}
return null;
}
/**
* Returns all entries matching the given prefix, in descending order of
* weight.
*
* @param prefix the prefix to match.
* @param results the current results
* @param queue
*/
function getSortedResults(node: Node, prefix: SearchKey, N: number, limit = MAX_SUGGESTIONS): TextWithProbability[] {
let queue = new PriorityQueue(function(a: Weighted, b: Weighted) {
// In case of Trie compilation issues that emit `null` or `undefined`
return (b ? b.weight : 0) - (a ? a.weight : 0);
});
let results: TextWithProbability[] = [];
if (node.type === 'leaf') {
// Assuming the values are sorted, we can just add all of the values in the
// leaf, until we reach the limit.
for (let item of node.entries) {
// String.startsWith is not supported on certain Android (5.0) devices we wish to support.
// Requires a minimum of Chrome 36, as opposed to 5.0's default of 35.
if (item.key.indexOf(prefix) == 0) {
let { content, weight } = item;
results.push({
text: content,
p: weight / N
});
if (results.length >= limit) {
return results;
}
}
}
} else {
queue.enqueue(node);
let next: Weighted | undefined;
while (next = queue.dequeue()) {
if (isNode(next)) {
// When a node is next up in the queue, that means that next least
// likely suggestion is among its decsendants.
// So we search all of its descendants!
if (next.type === 'leaf') {
queue.enqueueAll(next.entries);
} else {
// XXX: alias `next` so that TypeScript can be SURE that internal is
// in fact an internal node. Because of the callback binding to the
// original definition of node (i.e., a Node | Entry), this will not
// type-check otherwise.
let internal = next;
queue.enqueueAll(next.values.map(char => {
return internal.children[char];
}));
}
} else {
// When an entry is up next in the queue, we just add its contents to
// the results!
results.push({
text: next.content,
p: next.weight / N
});
if (results.length >= limit) {
return results;
}
}
}
}
return results;
}
/** TypeScript type guard that returns whether the thing is a Node. */
function isNode(x: Entry | Node): x is Node {
return 'type' in x;
}
/**
* Converts wordforms into an indexable form. It does this by
* normalizing into NFD, removing diacritics, and then converting
* the result to lowercase.
*
* This is a very naïve implementation, that I only think will work on
* some languages that use the Latin script. As of 2020-04-08, only
* 4 out of 11 (36%) of published language models use the Latin script,
* so this might not actually be a great default.
*
* This uses String.prototype.normalize() to convert normalize into NFD.
* NFD is an easy way to separate a Latin character from its diacritics;
* Even then, some Latin-based orthographies use code points that,
* under NFD normalization, do NOT decompose into an ASCII letter and a
* combining diacritical mark (e.g., SENĆOŦEN).
*
* Use this only in early iterations of the model. For a production lexical
* model, you SHOULD write/generate your own key function, tailored to your
* language.
*/
function defaultSearchTermToKey(wordform: string): SearchKey {
/**
* N.B.: this is (slightly) DIFFERENT than the version in
* keymanapp/lexical-model-compiler/build-trie
* as this is for compatibility for models built
* BEFORE the searchTermToKey function was bundled with
* all models.
*
* This compatibility version lowercases AFTER removing diacritics;
* the new version (bundled in future models) lowercases,
* NFD normalizes, THEN removes diacritics.
*/
return wordform
.normalize('NFD')
// Remove all combining diacritics (if input is in NFD)
// common to Latin-orthographies.
.replace(/[\u0300-\u036f]/g, '')
.toLowerCase() as SearchKey;
}

18
common/models/templates/test/helpers.js
View file

@ -1,5 +1,5 @@
const path = require('path');
const assert = require('assert');
import path from 'path';
import { assert } from 'chai';
var _ = global;
@ -9,7 +9,12 @@ var _ = global;
// Ensure that we can successfully load the module & apply kmwLength, as it's
// needed for some of the unit tests.
require('../build/index.bundled.js');
// Verify that the KMW string extensions are loaded via side-effect.
import * as models from '@keymanapp/models-templates/lib';
import { createRequire } from "module";
import { fileURLToPath } from 'url';
assert.ok('💩'.kmwLength);
@ -17,6 +22,13 @@ assert.ok('💩'.kmwLength);
* Load JSON fixtures from a well-known place.
*/
_.jsonFixture = function (name) {
// The most straight-forward way... is to use CommonJS-style require to load JSON.
// Fortunately, Node provides the tools needed to recreate it.
const require = createRequire(import.meta.url);
// ES-module mode also leaves out `__dirname`, so we rebuild that too.
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
return require(path.join(__dirname, 'fixtures', `${name}.json`));
}

4
common/models/templates/test/test-common.js
View file

@ -2,8 +2,8 @@
* Unit tests for common utility functions/methods.
*/
var assert = require('chai').assert;
var models = require('../build/index.bundled.js').models;
import { assert } from 'chai';
import * as models from '@keymanapp/models-templates';
describe('Common utility functions', function() {
// TODO: unit tests for other common utility functions

4
common/models/templates/test/test-priority-queue.js
View file

@ -2,8 +2,8 @@
* Unit tests for the priority queue.
*/
var assert = require('chai').assert;
var PriorityQueue = require('../build/index.bundled.js').models.PriorityQueue;
import { assert } from 'chai';
import { PriorityQueue } from '@keymanapp/models-templates';
describe('Priority queue', function() {
it('can act as a min-heap', function () {

4
common/models/templates/test/test-quote-behavior.js
View file

@ -2,8 +2,8 @@
* Unit tests for quote behaviors.
*/
var assert = require('chai').assert;
var QuoteBehavior = require('../build/index.bundled.js').models.QuoteBehavior;
import { assert } from 'chai';
import { QuoteBehavior } from '@keymanapp/models-templates';
describe('Quote behaviors', function() {
describe('Script directionality', function() {

8
common/models/templates/test/test-tokenization.js
View file

@ -2,9 +2,9 @@
* Unit tests for common utility functions/methods.
*/
var assert = require('chai').assert;
var models = require('../build/index.bundled.js').models;
var wordBreakers = require('@keymanapp/models-wordbreakers').wordBreakers;
import { assert } from 'chai';
import * as models from "@keymanapp/models-templates";
import * as wordBreakers from "@keymanapp/models-wordbreakers";
describe('Tokenization functions', function() {
describe('tokenize', function() {
@ -210,7 +210,7 @@ describe('Tokenization functions', function() {
// Step 2: Allow shifting a defined 'constant' span without mutating the definition.
let shiftSpan = function(span, delta) {
// Avoid mutating the parameter!
shiftedSpan = {
let shiftedSpan = {
text: span.text,
start: span.start + delta,
end: span.end + delta,

4
common/models/templates/test/test-trie-model.js
View file

@ -2,8 +2,8 @@
* Unit tests for the Trie prediction model.
*/
var assert = require('chai').assert;
var TrieModel = require('../build/index.bundled.js').models.TrieModel;
import { assert } from 'chai';
import { TrieModel } from '@keymanapp/models-templates';
describe('LMLayerWorker trie model for word lists', function() {
describe('instantiation', function () {

26
common/models/templates/test/test-trie-traversal.js
View file

@ -2,8 +2,8 @@
* Unit tests for the Trie prediction model.
*/
var assert = require('chai').assert;
var TrieModel = require('../build/index.bundled.js').models.TrieModel;
import { assert } from 'chai';
import { TrieModel } from '@keymanapp/models-templates';
// Useful for tests related to strings with supplementary pairs.
var smpForUnicode = function(code){
@ -23,7 +23,7 @@ describe('Trie traversal abstractions', function() {
let rootKeys = ['t', 'o', 'a', 'i', 'w', 'h', 'f', 'b', 'n', 'y', 's', 'm',
'u', 'c', 'd', 'l', 'e', 'j', 'p', 'g', 'v', 'k', 'r', 'q']
for(child of rootTraversal.children()) {
for(let child of rootTraversal.children()) {
let keyIndex = rootKeys.indexOf(child.char);
assert.notEqual(keyIndex, -1);
rootKeys.splice(keyIndex, 1);
@ -43,7 +43,7 @@ describe('Trie traversal abstractions', function() {
let tSuccess = false;
let hSuccess = false;
let eSuccess = false;
for(child of rootTraversal.children()) {
for(let child of rootTraversal.children()) {
if(child.char == 't') {
tSuccess = true;
let traversalInner1 = child.traversal();
@ -51,7 +51,7 @@ describe('Trie traversal abstractions', function() {
assert.isArray(child.traversal().entries);
assert.isEmpty(child.traversal().entries);
for(tChild of traversalInner1.children()) {
for(let tChild of traversalInner1.children()) {
if(tChild.char == 'h') {
hSuccess = true;
let traversalInner2 = tChild.traversal();
@ -59,7 +59,7 @@ describe('Trie traversal abstractions', function() {
assert.isEmpty(tChild.traversal().entries);
assert.isArray(tChild.traversal().entries);
for(hChild of traversalInner2.children()) {
for(let hChild of traversalInner2.children()) {
if(hChild.char == 'e') {
eSuccess = true;
let traversalInner3 = hChild.traversal();
@ -68,7 +68,7 @@ describe('Trie traversal abstractions', function() {
assert.isDefined(traversalInner3.entries);
assert.equal(traversalInner3.entries[0], "the");
for(eChild of traversalInner3.children()) {
for(let eChild of traversalInner3.children()) {
let keyIndex = eKeys.indexOf(eChild.char);
assert.notEqual(keyIndex, -1, "Did not find char '" + eChild.char + "' in array!");
eKeys.splice(keyIndex, 1);
@ -96,21 +96,21 @@ describe('Trie traversal abstractions', function() {
// 't', 'r', 'o' have siblings, but these don't.
let leafChildSequence = ['u', 'b', 'l', 'e'];
let eSuccess = false;
for(child of rootTraversal.children()) {
for(let child of rootTraversal.children()) {
if(child.char == 't') {
let traversalInner1 = child.traversal();
assert.isDefined(traversalInner1);
assert.isArray(child.traversal().entries);
assert.isEmpty(child.traversal().entries);
for(tChild of traversalInner1.children()) {
for(let tChild of traversalInner1.children()) {
if(tChild.char == 'r') {
let traversalInner2 = tChild.traversal();
assert.isDefined(traversalInner2);
assert.isArray(tChild.traversal().entries);
assert.isEmpty(tChild.traversal().entries);
for(rChild of traversalInner2.children()) {
for(let rChild of traversalInner2.children()) {
if(rChild.char == 'o') {
let curChild = rChild;
@ -176,7 +176,7 @@ describe('Trie traversal abstractions', function() {
let aSuccess = false;
let pSuccess = false;
let eSuccess = false;
for(child of rootTraversal.children()) {
for(let child of rootTraversal.children()) {
if(child.char == smpA) {
aSuccess = true;
let traversalInner1 = child.traversal();
@ -184,7 +184,7 @@ describe('Trie traversal abstractions', function() {
assert.isArray(child.traversal().entries);
assert.isEmpty(child.traversal().entries);
for(aChild of traversalInner1.children()) {
for(let aChild of traversalInner1.children()) {
if(aChild.char == smpP) {
pSuccess = true;
let traversalInner2 = aChild.traversal();
@ -192,7 +192,7 @@ describe('Trie traversal abstractions', function() {
assert.isArray(aChild.traversal().entries);
assert.isEmpty(aChild.traversal().entries);
for(pChild of traversalInner2.children()) {
for(let pChild of traversalInner2.children()) {
let keyIndex = pKeys.indexOf(pChild.char);
assert.notEqual(keyIndex, -1, "Did not find char '" + pChild.char + "' in array!");
pKeys.splice(keyIndex, 1);

21
common/models/templates/tsconfig.bundled.json
View file

@ -1,21 +0,0 @@
{
// This variant of the tsconfig.json exists to create a 'leaf', 'bundled'
// version of the models/templates build product. The same reference
// cannot be prepended twice in a composite tsc build, posing problems
// for certain down-line builds if the two tsconfigs are not differentiated.
"extends": "./tsconfig.json",
"compilerOptions": {
"outFile": "build/index.bundled.js",
},
"references": [
{ "path": "../../web/keyman-version", "prepend": true},
{ "path": "../../web/utils", "prepend": true },
{ "path": "../types" }
],
"include": [
"src/**/*.ts"
],
"exclude": [
"test"
]
}

16
common/models/templates/tsconfig.json
View file

@ -1,16 +1,26 @@
{
"extends": "../../../tsconfig-base.json",
"compilerOptions": {
"allowJs": false,
"declaration": true,
"module": "none",
"outFile": "./build/index.js",
"module": "es6",
"moduleResolution": "node16",
"sourceMap": true,
"sourceRoot": "/common/models/templates/src",
"inlineSources": true,
"strict": true,
"lib": ["es6"],
"types": ["node"],
"target": "es5",
"baseUrl": "./",
"outDir": "build/obj",
"tsBuildInfoFile": "build/obj/tsconfig.tsbuildinfo",
"rootDir": "./src"
},
"references": [
{ "path": "../types" },
{ "path": "../../web/utils" },
{ "path": "../types" }
{ "path": "../../models/wordbreakers" }
],
"include": [
"src/**/*.ts"

33
common/models/wordbreakers/build-bundler.js Normal file
View file

@ -0,0 +1,33 @@
/*
* Note: while this file is not meant to exist long-term, it provides a nice
* low-level proof-of-concept for esbuild bundling of the various Web submodules.
*
* Add some extra code at the end of src/index.ts and run it to verify successful bundling!
*/
import esbuild from 'esbuild';
import { spawn } from 'child_process';
await esbuild.build({
bundle: true,
sourcemap: true,
format: "esm",
nodePaths: ['..'],
entryPoints: ['build/obj/index.js'],
outfile: "build/lib/index.mjs",
tsconfig: 'tsconfig.json',
target: "es5"
});
// Bundled CommonJS (classic Node) module version
esbuild.buildSync({
bundle: true,
sourcemap: true,
format: "cjs",
nodePaths: ['..'],
entryPoints: ['build/obj/index.js'],
outfile: "build/lib/index.cjs",
tsconfig: 'tsconfig.json',
target: "es5"
});

62
common/models/wordbreakers/build.sh Executable file
View file

@ -0,0 +1,62 @@
#!/usr/bin/env bash
#
# Compile our sourcemap-path remapping module for use by Web builds, releases, etc.
#
set -eu
## START STANDARD BUILD SCRIPT INCLUDE
# adjust relative paths as necessary
THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BASH_SOURCE[0]}")"
. "$(dirname "$THIS_SCRIPT")/../../../resources/build/build-utils.sh"
## END STANDARD BUILD SCRIPT INCLUDE
. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh"
# This script runs from its own folder
cd "$(dirname "$THIS_SCRIPT")"
################################ Main script ################################
builder_describe "Builds the predictive-text wordbreaker implementation module" \
"clean" \
"configure" \
"build" \
"test"
builder_describe_outputs \
configure /node_modules \
build build/lib/index.mjs
builder_parse "$@"
### CONFIGURE ACTIONS
if builder_start_action configure; then
verify_npm_setup
builder_finish_action success configure
fi
### CLEAN ACTIONS
if builder_start_action clean; then
rm -rf build/
builder_finish_action success clean
fi
### BUILD ACTIONS
if builder_start_action build; then
npm run tsc -- -b
node build-bundler.js
# Declaration bundling.
npm run tsc -- --emitDeclarationOnly --outFile ./build/lib/index.d.ts
builder_finish_action success build
fi
if builder_start_action test; then
npm run mocha
builder_finish_action success test
fi

19
common/models/wordbreakers/package.json
View file

@ -14,7 +14,16 @@
],
"homepage": "https://github.com/keymanapp/keyman",
"license": "MIT",
"main": "build/index.js",
"main": "build/obj/index.js",
"types": "build/obj/index.d.ts",
"exports": {
".": "./build/obj/index.js",
"./lib": {
"import": "./build/lib/index.mjs",
"require": "./build/lib/index.cjs"
},
"./obj/*.js": "./build/obj/*.js"
},
"directories": {
"lib": "lib",
"test": "test"
@ -30,16 +39,19 @@
"url": "git+https://github.com/keymanapp/keyman.git"
},
"scripts": {
"build": "tsc",
"build": "gosh ./build.sh",
"tsc": "tsc",
"prepublish": "npm run build",
"pretest": "npm run build",
"test": "mocha"
"test": "mocha",
"mocha": "mocha"
},
"bugs": {
"url": "https://github.com/keymanapp/keyman/issues"
},
"devDependencies": {
"@keymanapp/models-types": "*",
"@keymanapp/resources-gosh": "*",
"@types/chai": "^4.2.11",
"@types/mocha": "^7.0.2",
"chai": "^4.3.4",
@ -47,6 +59,7 @@
"ts-node": "^10.9.1",
"typescript": "^4.9.5"
},
"type": "module",
"dependencies": {
"@types/node": "^14.0.3"
}

61
common/models/wordbreakers/src/ascii.ts
View file

@ -1,40 +1,39 @@
///<reference types="@keymanapp/models-types" />
namespace wordBreakers {
/**
* Splits ASCII words.
*
* @param phrase
*/
export function ascii(phrase: string): Span[] {
let matchWord = /[A-Za-z0-9']+/g;
let words: Span[] = [];
let match: RegExpExecArray | null;
while ((match = matchWord.exec(phrase)) !== null) {
words.push(new RegExpDerivedSpan(match[0], match.index));
}
return words;
/**
* Splits ASCII words.
*
* @param phrase
*/
export default function ascii(phrase: string): Span[] {
let matchWord = /[A-Za-z0-9']+/g;
let words: Span[] = [];
let match: RegExpExecArray | null;
while ((match = matchWord.exec(phrase)) !== null) {
words.push(new RegExpDerivedSpan(match[0], match.index));
}
/**
* A concrete span class that derives its properties from the result of
* RegExp.exec() array.
*/
class RegExpDerivedSpan implements Span {
readonly text: string;
readonly start: number;
return words;
}
constructor(text: string, start: number) {
this.text = text;
this.start = start;
}
/**
* A concrete span class that derives its properties from the result of
* RegExp.exec() array.
*/
class RegExpDerivedSpan implements Span {
readonly text: string;
readonly start: number;
get length(): number {
return this.text.length;
}
constructor(text: string, start: number) {
this.text = text;
this.start = start;
}
get end(): number {
return this.start + this.text.length;
}
get length(): number {
return this.text.length;
}
get end(): number {
return this.start + this.text.length;
}
}

5
common/models/wordbreakers/src/default/data.ts
View file

@ -1,6 +1,5 @@
namespace wordBreakers {
export namespace data {
// Automatically generated file. DO NOT MODIFY.
/**
* Valid values for a word break property.
*/
@ -1801,5 +1800,3 @@ export const WORD_BREAK_PROPERTY: [number, WordBreakProperty][] = [
[/*start*/ 0xE0100, WordBreakProperty.Extend],
[/*start*/ 0xE01F0, WordBreakProperty.Other],
];
}
}

1133
common/models/wordbreakers/src/default/index.ts
View file

File diff suppressed because it is too large Load diff

11
common/models/wordbreakers/src/index.ts
View file

@ -1,8 +1,5 @@
/// <reference path="./ascii.ts" />
/// <reference path="./default/index.ts" />
/// <reference path="./placeholder.ts" />
import placeholder from "./placeholder.js";
import ascii from "./ascii.js";
import default_ from "./default/index.js";
// Let the word breakers be available both in the browser and in Node.
if (typeof module !== "undefined" && (module as any).exports) {
(module as any).exports['wordBreakers'] = wordBreakers;
}
export { placeholder, ascii, default_ as default, default_ as defaultWordbreaker };

43
common/models/wordbreakers/src/placeholder.ts
View file

@ -1,24 +1,21 @@
namespace wordBreakers {
/**
* A **VERY** dumb word breaker that simply splits at words. Do not use this
* word breaker!
*
* @param phrase The phrase in which to break words.
* @deprecated Use a word breaker tailored to your language instead!
*/
export function placeholder(phrase: string): Span[] {
let nextStart = 0;
return phrase.split(/\s+/).map(utterance => {
// XXX: The indices are NOT accurate to the original phrase!
let span = {
start: nextStart,
end: nextStart + utterance.length,
text: utterance,
length: utterance.length
};
nextStart = span.end;
return span;
});
}
/**
* A **VERY** dumb word breaker that simply splits at words. Do not use this
* word breaker!
*
* @param phrase The phrase in which to break words.
* @deprecated Use a word breaker tailored to your language instead!
*/
export default function placeholder(phrase: string): Span[] {
let nextStart = 0;
return phrase.split(/\s+/).map(utterance => {
// XXX: The indices are NOT accurate to the original phrase!
let span = {
start: nextStart,
end: nextStart + utterance.length,
text: utterance,
length: utterance.length
};
nextStart = span.end;
return span;
});
}

4
common/models/wordbreakers/test/test-ascii-word-breaker.js
View file

@ -1,5 +1,5 @@
var assert = require('chai').assert;
var breakASCIIWords = require('../build').wordBreakers['ascii'];
import { assert } from 'chai';
import { ascii as breakASCIIWords } from '@keymanapp/models-wordbreakers';
describe('The ASCII word breaker', function () {
it('should break simple English sentences', function () {

4
common/models/wordbreakers/test/test-default-word-breaker.js
View file

@ -2,8 +2,8 @@
* Smoke-test the default
*/
const assert = require('chai').assert;
const breakWords = require('../build').wordBreakers['default'];
import { assert } from 'chai';
import { default as breakWords } from '@keymanapp/models-wordbreakers';
const SHY = '\u00AD'; // Other, Format. The "Soft HYphen" - usually invisible unless needed for word-wrapping.

4
common/models/wordbreakers/test/test-placeholder-word-breaker.js
View file

@ -1,6 +1,6 @@
const assert = require('chai').assert;
const breakWords = require('../build').wordBreakers['placeholder'];
import { assert } from 'chai';
import { placeholder as breakWords } from '@keymanapp/models-wordbreakers';
describe('The placeholder word breaker', function () {
it('should break simple English sentences', function () {

10
common/models/wordbreakers/tsconfig.json
View file

@ -2,14 +2,20 @@
"extends": "../../../tsconfig-base.json",
"compilerOptions": {
"module": "none",
"module": "es6",
"lib": ["es6"],
"target": "es5",
"moduleResolution": "Node16",
"declaration": true,
"sourceMap": true,
"outFile": "./build/index.js",
"inlineSources": true,
"sourceRoot": "/common/models/wordbreakers/src",
"strict": true,
"allowJs": false,
"baseUrl": "./",
"outDir": "build/obj",
"tsBuildInfoFile": "build/obj/tsconfig.tsbuildinfo",
"rootDir": "./src"
},
"references": [
{ "path": "../types" }

1
common/predictive-text/.gitignore vendored
View file

@ -68,6 +68,7 @@ typings/
**/*.js
# Intentional JavaScript files.
!build-bundler.js
!testing/**/*.js
!unit_tests/**/*.js
!polyfills/**/*.js

31
common/predictive-text/browser.tsconfig.json
View file

@ -1,31 +0,0 @@
{
"extends": "../../tsconfig-base.json",
"compilerOptions": {
"allowJs": true,
"module": "none",
"outFile": "./build/index.js",
"inlineSources": true,
"sourceMap": true,
"target": "es5",
"lib": ["es6", "dom"]
},
"include" : ["./*.ts"],
"references": [
{ "path": "../models/types" },
{ "path": "../models/wordbreakers" },
{ "path": "../models/templates" },
{ "path": "../web/lm-message-types" }
],
"exclude" : [
"node-defaultWorker.ts",
"virtualizedWorker.ts",
"index.d.ts",
"worker-interface.d.ts",
"node_modules",
"testing"
],
"files": [
"../web/lm-worker/build/index.wrapped-for-bundle.ts"
]
}

57
common/predictive-text/build-bundler.js Normal file
View file

@ -0,0 +1,57 @@
/*
* Note: while this file is not meant to exist long-term, it provides a nice
* low-level proof-of-concept for esbuild bundling of the various Web submodules.
*
* Add some extra code at the end of src/index.ts and run it to verify successful bundling!
*/
import esbuild from 'esbuild';
import { spawn } from 'child_process';
await esbuild.build({
bundle: true,
sourcemap: true,
format: "esm",
nodePaths: ['../../node_modules'],
entryPoints: {
'index': 'build/obj/web/index.js',
},
external: ['fs', 'vm'],
outdir: 'build/lib/web',
outExtension: { '.js': '.mjs' },
tsconfig: 'src/web/tsconfig.json',
target: "es5"
});
// // Direct-use version
// esbuild.buildSync({
// bundle: true,
// sourcemap: true,
// format: "iife",
// nodePaths: ['..'],
// entryPoints: {
// 'worker-main': 'build/obj/worker-main.js'
// },
// outdir: 'build/lib',
// tsconfig: 'tsconfig.json',
// target: "es5"
// });
const dtsBundleCommand1 = spawn('npx dts-bundle-generator --project src/web/tsconfig.json -o build/lib/web/index.d.ts src/web/index.ts', {
shell: true
});
dtsBundleCommand1.stdout.on('data', data => console.log(data.toString()));
dtsBundleCommand1.stderr.on('data', data => console.error(data.toString()));
// Forces synchronicity; done mostly so that the logs don't get jumbled up.
await new Promise((resolve, reject) => {
dtsBundleCommand1.on('exit', () => {
if(dtsBundleCommand1.exitCode != 0) {
reject();
process.exit(dtsBundleCommand1.exitCode);
}
resolve();
});
});

44
common/predictive-text/build.sh
View file

@ -22,10 +22,7 @@ cd "$(dirname "$THIS_SCRIPT")"
################################ Main script ################################
# TODO: once these modules are builder-based, reference here too:
# "@../models/templates" \
# "@../models/types" \
# "@../models/wordbreakers"
# "@../models/types" \ # is just a .d.ts, so there's nothing to actually BUILD.
builder_describe "Builds the lm-layer module" \
"@/common/web/keyman-version" \
@ -34,16 +31,11 @@ builder_describe "Builds the lm-layer module" \
"configure" \
"build" \
"test" \
":headless A headless, Node-oriented version of the module useful for unit tests" \
":browser The standard version of the module for in-browser use" \
"--ci Sets $(builder_term test) action to use CI-based test configurations & reporting"
builder_describe_outputs \
configure /node_modules \
configure:headless /node_modules \
configure:browser /node_modules \
build:headless /common/predictive-text/build/headless.js \
build:browser /common/predictive-text/build/index.js
configure /node_modules \
build /common/predictive-text/build/lib/web/index.mjs # is built by the final step.
builder_parse "$@"
@ -63,18 +55,14 @@ fi
### BUILD ACTIONS
# Builds the top-level JavaScript file for use in browsers
if builder_start_action build:browser; then
npm run tsc -- -b ./browser.tsconfig.json
builder_finish_action success build:browser
fi
# Builds the top-level JavaScript file for use on Node
if builder_start_action build:headless; then
npm run tsc -- -b ./tsconfig.json
if builder_start_action build; then
npm run tsc -- -b ./tsconfig.all.json
builder_finish_action success build:headless
# esbuild-bundled products at this level are not intended to be used for anything but testing.
node build-bundler.js
builder_finish_action success build
fi
### TEST ACTIONS
@ -86,16 +74,10 @@ if builder_has_option --ci; then
TEST_OPTIONS=--ci
fi
if builder_start_action test:headless; then
if builder_start_action test; then
# We'll test the included libraries here for now, at least until we have
# converted their builds to builder scripts
./unit_tests/test.sh test:libraries test:headless $TEST_OPTIONS
./unit_tests/test.sh test:libraries test:headless test:browser $TEST_OPTIONS
builder_finish_action success test:headless
fi
if builder_start_action test:browser; then
./unit_tests/test.sh test:browser $TEST_OPTIONS
builder_finish_action success test:browser
fi
builder_finish_action success test
fi

73
common/predictive-text/index.d.ts vendored
View file

@ -1,73 +0,0 @@
/**
* Defines TS method typing for the LMLayer's public API.
*
* Unfortunately, it is not possible to do this programmatically at this time, as TS won't output declarations
* if any JS files are allowed in compilation. We can't convert the embedded worker's intermediate file to TS
* safely due to DedicatedWorkerGlobalScope and a few other typing issues.
*/
/// <reference types="@keymanapp/lm-message-types" />
/// <reference types="@keymanapp/models-types" />
declare namespace com.keyman.text.prediction {
class LMLayer {
/**
* Construct the top-level LMLayer interface. This also starts the underlying Worker.
* Make sure to call .load() when using the default Worker.
*
* @param uri URI of the underlying LMLayer worker code. This will usually be a blob:
* or file: URI. If uri is not provided, this will start the default Worker.
*/
constructor(capabilities: Capabilities, worker?: Worker);
/**
* Initializes the LMLayer worker with the keyboard/platform's capabilities,
* as well as a description of the model required.
*/
loadModel(model: string): Promise<Configuration>;
/**
* Prepares the LMLayer for reinitialization with a different model/capability set.
*/
unloadModel();
predict(transform: Transform, context: Context): Promise<Suggestion[]>;
// TODO: asynchronous close() method.
// Worker code must recognize message and call self.close().
private onMessage(event: MessageEvent): void;
/**
* Given a function, this utility returns the source code within it, as a string.
* This is intended to unwrap the "wrapped" source code created in the LMLayerWorker
* build process.
*
* @param fn The function whose body will be returned.
*/
static unwrap(fn: Function): string;
/**
* Converts the INSIDE of a function into a blob URI that can
* be passed as a valid URI for a Worker.
* @param fn Function whose body will be referenced by a URI.
*
* This function makes the following possible:
*
* let worker = new Worker(LMLayer.asBlobURI(function myWorkerCode () {
* postMessage('inside Web Worker')
* function onmessage(event) {
* // handle message inside Web Worker.
* }
* }));
*/
static asBlobURI(fn: Function): string;
/**
* Clears out any computational resources in use by the LMLayer, including shutting
* down any internal WebWorkers.
*/
public shutdown(): void;
}
}

254
common/predictive-text/index.ts
View file

@ -1,254 +0,0 @@
/*
* Copyright (c) 2018 National Research Council Canada (author: Eddie A. Santos)
* Copyright (c) 2018 SIL International
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to
* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
* the Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
/// <reference path="promise-store.ts" />
/// <reference types="@keymanapp/lm-message-types" />
/// <reference path="worker-interface.d.ts" />
/**
* Top-level interface to the Language Modelling layer, or "LMLayer" for short.
*
* The Language Modelling layer provides a way for keyboards to offer prediction and
* correction functionalities. The LMLayer proper runs within a Web Worker, however,
* this class is intended to run in the main thread, and automatically spawn a Web
* Worker, capable of offering predictions.
*
* Since the Worker runs in a different thread, the public methods of this class are
* asynchronous. Methods of note include:
*
* - #loadModel() -- loads a specified model file
* - #predict() -- ask the LMLayer to offer suggestions (predictions or corrections) for
* the input event
* - #unloadModel() -- unloads the LMLayer's currently loaded model, preparing it to
* receive (load) a new model
*
* The top-level LMLayer will automatically starts up its own Web Worker.
*/
namespace com.keyman.text.prediction {
export class LMLayer {
/**
* The underlying worker instance. By default, this is the LMLayerWorker.
*/
private _worker: Worker;
/** Call this when the LMLayer has sent us the 'ready' message! */
private _declareLMLayerReady: (conf: Configuration) => void;
private _predictPromises: PromiseStore<Suggestion[]>;
private _wordbreakPromises: PromiseStore<USVString>;
private _acceptPromises: PromiseStore<Reversion>;
private _revertPromises: PromiseStore<Suggestion[]>;
private _nextToken: number;
private capabilities: Capabilities;
/**
* Construct the top-level LMLayer interface. This also starts the underlying Worker.
*
* @param uri URI of the underlying LMLayer worker code. This will usually be a blob:
* or file: URI. If uri is not provided, this will start the default Worker.
*/
constructor(capabilities: Capabilities, worker?: Worker, testMode?: boolean) {
// Either use the given worker, or instantiate the default worker.
this._worker = worker || DefaultWorker.constructInstance();
this._worker.onmessage = this.onMessage.bind(this)
this._declareLMLayerReady = null;
this._predictPromises = new PromiseStore();
this._wordbreakPromises = new PromiseStore();
this._acceptPromises = new PromiseStore();
this._revertPromises = new PromiseStore();
this._nextToken = Number.MIN_SAFE_INTEGER;
this.sendConfig(capabilities, !!testMode);
}
/**
* Initializes the LMLayer worker with the host platform's capability set.
*
* @param capabilities The host platform's capability spec - a model cannot assume access to more context
* than specified by this parameter.
*/
private sendConfig(capabilities: Capabilities, testMode: boolean) {
this._worker.postMessage({
message: 'config',
capabilities: capabilities,
testMode: testMode
});
}
/**
* Initializes the LMLayer worker with a path to the desired model file.
*/
loadModel(modelSource: string, loadType: 'file' | 'raw' = 'file'): Promise<Configuration> {
return new Promise((resolve, _reject) => {
// Sets up so the promise is resolved in the onMessage() callback, when it receives
// the 'ready' message.
this._declareLMLayerReady = resolve;
let modelSourceSpec: any = {
type: loadType
};
if(loadType == 'file') {
modelSourceSpec.file = modelSource;
} else {
modelSourceSpec.code = modelSource;
}
this._worker.postMessage({
message: 'load',
source: modelSourceSpec
});
});
}
/**
* Unloads the previously-active model from memory, resetting the LMLayer to prep
* for transition to use of a new model.
*/
public unloadModel() {
this._worker.postMessage({
message: 'unload'
});
}
predict(transform: Transform | Distribution<Transform>, context: Context): Promise<Suggestion[]> {
let token = this._nextToken++;
return new Promise((resolve, reject) => {
this._predictPromises.make(token, resolve, reject);
this._worker.postMessage({
message: 'predict',
token: token,
transform: transform,
context: context,
});
});
}
wordbreak(context: Context): Promise<USVString> {
let token = this._nextToken++;
return new Promise((resolve, reject) => {
this._wordbreakPromises.make(token, resolve, reject);
this._worker.postMessage({
message: 'wordbreak',
token: token,
context: context
})
});
}
acceptSuggestion(suggestion: Suggestion, context: Context, postTransform: Transform): Promise<Reversion> {
let token = this._nextToken++;
return new Promise((resolve, reject) => {
this._acceptPromises.make(token, resolve, reject);
this._worker.postMessage({
message: 'accept',
token: token,
suggestion: suggestion,
context: context,
postTransform: postTransform
});
});
}
revertSuggestion(reversion: Reversion, context: Context): Promise<Suggestion[]> {
let token = this._nextToken++;
return new Promise((resolve, reject) => {
this._revertPromises.make(token, resolve, reject);
this._worker.postMessage({
message: 'revert',
token: token,
reversion: reversion,
context: context
})
});
}
resetContext(context: Context) {
this._worker.postMessage({
message: 'reset-context',
context: context
});
}
// TODO: asynchronous close() method.
// Worker code must recognize message and call self.close().
private onMessage(event: MessageEvent): void {
let payload: OutgoingMessage = event.data;
if (payload.message === 'error') {
console.error(payload.log);
if(payload.error) {
console.error(payload.error);
}
}
else if (payload.message === 'ready') {
this._declareLMLayerReady(event.data.configuration);
} else if (payload.message === 'suggestions') {
this._predictPromises.keep(payload.token, payload.suggestions);
} else if (payload.message === 'currentword') {
this._wordbreakPromises.keep(payload.token, payload.word);
} else if (payload.message === 'postaccept') {
this._acceptPromises.keep(payload.token, payload.reversion);
} else if (payload.message === 'postrevert') {
this._revertPromises.keep(payload.token, payload.suggestions);
} else {
// This branch should never execute, but just in case...
//@ts-ignore
throw new Error(`Message not implemented: ${payload.message}`);
}
}
/**
* Clears out any computational resources in use by the LMLayer, including shutting
* down any internal WebWorkers.
*/
public shutdown() {
this._worker.terminate();
}
/**
* Given a function, this utility returns the source code within it, as a string.
* This is intended to unwrap the "wrapped" source code created in the LMLayerWorker
* build process.
*
* @param fn The function whose body will be returned.
*/
static unwrap(fn: Function): string {
let wrapper = fn.toString();
let match = wrapper.match(/function[^{]+{((?:.|\r|\n)+)}[^}]*$/);
return match[1];
}
}
}
(function () {
let ns = com.keyman.text.prediction;
// Let LMLayer be available both in the browser and in Node.
if (typeof module !== 'undefined' && typeof module.exports !== 'undefined') {
module.exports = ns.LMLayer;
//@ts-ignore
ns.LMLayer.PromiseStore = ns.PromiseStore;
} else {
//@ts-ignore
window.LMLayer = ns.LMLayer;
}
}());

12
common/predictive-text/node-defaultWorker.ts
View file

@ -1,12 +0,0 @@
///<reference path="virtualizedWorker.ts" />
namespace com.keyman.text.prediction {
export class DefaultWorker {
static constructInstance(): Worker {
let scriptStr = LMLayer.unwrap(LMLayerWorkerCode);
let worker = new VirtualizedWorker(scriptStr);
return worker;
}
}
}

24
common/predictive-text/package.json
View file

@ -1,7 +1,29 @@
{
"name": "@keymanapp/lexical-model-layer",
"description": "Keyman/Predictive Text integration layer",
"main": "build/index.js",
"main": "build/obj/node/index.js",
"exports": {
".": {
"node": {
"types": "./build/obj/node/index.d.ts",
"import": "./build/obj/index.js"
},
"default": {
"types": "./build/obj/web/index.d.ts",
"import": "./build/obj/web/index.js"
}
},
"./node": "./build/obj/node/index.js",
"./web": "./build/obj/web/index.js",
"./build/obj/node/index.js": "./build/obj/node/index.js",
"./build/obj/web/index.js": "./build/obj/web/index.js"
},
"imports": {
"#./*.js": "./build/obj/*.js",
"#./node/*.js": "./build/obj/node/*.js",
"#./web/*.js": "./build/obj/web/*.js"
},
"type": "module",
"scripts": {
"karma": "karma",
"mocha": "mocha",

90
common/predictive-text/promise-store.ts
View file

@ -1,90 +0,0 @@
/*
* Copyright (c) 2018 National Research Council Canada (author: Eddie A. Santos)
* Copyright (c) 2018 SIL International
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to
* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
* the Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
type Resolve<T> = (value?: T | PromiseLike<T>) => void;
type Reject = (reason?: any) => void;
interface PromiseCallbacks<T> {
resolve: Resolve<T>;
reject: Reject;
}
namespace com.keyman.text.prediction {
/**
* Associate tokens with promises.
*
* First, .make() a promise -- associate a token with resolve/reject callbacks.
*
* You can either .keep() a promise -- resolve() and forget it;
* Or you may also .break() a promise -- reject() and forget it.
*
* <T> is the type of resolved value (value yielded successfully by promise).
*/
export class PromiseStore<T> {
// IE11 offers partial support for new Map().
// Assume only .get(), .set(), .has(), .delete(), and .size work.
// See: http://kangax.github.io/compat-table/es6/#test-Map
private _promises: Map<Token, PromiseCallbacks<T>>;
constructor() {
this._promises = new Map();
}
/**
* How many promises are currently being tracked?
*/
get length(): number {
return this._promises.size;
}
/**
* Associate a token with its respective resolve and reject callbacks.
*/
make(token: Token, resolve: Resolve<T>, reject: Reject): void {
if (this._promises.has(token)) {
return reject(`Existing request with token ${token}`);
}
this._promises.set(token, { reject, resolve });
}
/**
* Resolve the promise associated with a token (with a value!).
* Once the promise is resolved, the token is removed..
*/
keep(token: Token, value: T) {
let callbacks = this._promises.get(token);
if (!callbacks) {
throw new Error(`No promise associated with token: ${token}`);
}
let accept = callbacks.resolve;
this._promises.delete(token);
return accept(value);
}
/**
* Instantly reject and forget a promise associated with the token.
*/
break(token: Token, reason?: any): void {
let callbacks = this._promises.get(token);
if (!callbacks) {
throw new Error(`No promise associated with token: ${token}`);
}
this._promises.delete(token);
callbacks.reject(reason);
}
}
}

3
common/predictive-text/src/index.ts Normal file
View file

@ -0,0 +1,3 @@
export { default as LMLayer } from './lmlayer.js';
export { default as NodeWorker } from './node/worker.js';
export { default as WebWorker } from './web/worker.js';

226
common/predictive-text/src/lmlayer.ts Normal file
View file

@ -0,0 +1,226 @@
/*
* Copyright (c) 2018 National Research Council Canada (author: Eddie A. Santos)
* Copyright (c) 2018 SIL International
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to
* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
* the Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
import PromiseStore from "./promise-store.js";
/// <reference types="@keymanapp/lm-message-types" />
/// <reference types="worker-interface.d.ts" />
/**
* Top-level interface to the Language Modelling layer, or "LMLayer" for short.
*
* The Language Modelling layer provides a way for keyboards to offer prediction and
* correction functionalities. The LMLayer proper runs within a Web Worker, however,
* this class is intended to run in the main thread, and automatically spawn a Web
* Worker, capable of offering predictions.
*
* Since the Worker runs in a different thread, the public methods of this class are
* asynchronous. Methods of note include:
*
* - #loadModel() -- loads a specified model file
* - #predict() -- ask the LMLayer to offer suggestions (predictions or corrections) for
* the input event
* - #unloadModel() -- unloads the LMLayer's currently loaded model, preparing it to
* receive (load) a new model
*
* The top-level LMLayer will automatically starts up its own Web Worker.
*/
export default class LMLayer {
/**
* The underlying worker instance. By default, this is the LMLayerWorker.
*/
private _worker: Worker;
/** Call this when the LMLayer has sent us the 'ready' message! */
private _declareLMLayerReady: (conf: Configuration) => void;
private _predictPromises: PromiseStore<Suggestion[]>;
private _wordbreakPromises: PromiseStore<USVString>;
private _acceptPromises: PromiseStore<Reversion>;
private _revertPromises: PromiseStore<Suggestion[]>;
private _nextToken: number;
private capabilities: Capabilities;
/**
* Construct the top-level LMLayer interface. This also starts the underlying Worker.
*
* @param uri URI of the underlying LMLayer worker code. This will usually be a blob:
* or file: URI. If uri is not provided, this will start the default Worker.
*/
constructor(capabilities: Capabilities, worker: Worker, testMode?: boolean) {
// Either use the given worker, or instantiate the default worker.
this._worker = worker;
this._worker.onmessage = this.onMessage.bind(this)
this._declareLMLayerReady = null;
this._predictPromises = new PromiseStore();
this._wordbreakPromises = new PromiseStore();
this._acceptPromises = new PromiseStore();
this._revertPromises = new PromiseStore();
this._nextToken = Number.MIN_SAFE_INTEGER;
this.sendConfig(capabilities, !!testMode);
}
/**
* Initializes the LMLayer worker with the host platform's capability set.
*
* @param capabilities The host platform's capability spec - a model cannot assume access to more context
* than specified by this parameter.
*/
private sendConfig(capabilities: Capabilities, testMode: boolean) {
this._worker.postMessage({
message: 'config',
capabilities: capabilities,
testMode: testMode
});
}
/**
* Initializes the LMLayer worker with a path to the desired model file.
*/
loadModel(modelSource: string, loadType: 'file' | 'raw' = 'file'): Promise<Configuration> {
return new Promise((resolve, _reject) => {
// Sets up so the promise is resolved in the onMessage() callback, when it receives
// the 'ready' message.
this._declareLMLayerReady = resolve;
let modelSourceSpec: any = {
type: loadType
};
if(loadType == 'file') {
modelSourceSpec.file = modelSource;
} else {
modelSourceSpec.code = modelSource;
}
this._worker.postMessage({
message: 'load',
source: modelSourceSpec
});
});
}
/**
* Unloads the previously-active model from memory, resetting the LMLayer to prep
* for transition to use of a new model.
*/
public unloadModel() {
this._worker.postMessage({
message: 'unload'
});
}
predict(transform: Transform | Distribution<Transform>, context: Context): Promise<Suggestion[]> {
let token = this._nextToken++;
return new Promise((resolve, reject) => {
this._predictPromises.make(token, resolve, reject);
this._worker.postMessage({
message: 'predict',
token: token,
transform: transform,
context: context,
});
});
}
wordbreak(context: Context): Promise<USVString> {
let token = this._nextToken++;
return new Promise((resolve, reject) => {
this._wordbreakPromises.make(token, resolve, reject);
this._worker.postMessage({
message: 'wordbreak',
token: token,
context: context
})
});
}
acceptSuggestion(suggestion: Suggestion, context: Context, postTransform: Transform): Promise<Reversion> {
let token = this._nextToken++;
return new Promise((resolve, reject) => {
this._acceptPromises.make(token, resolve, reject);
this._worker.postMessage({
message: 'accept',
token: token,
suggestion: suggestion,
context: context,
postTransform: postTransform
});
});
}
revertSuggestion(reversion: Reversion, context: Context): Promise<Suggestion[]> {
let token = this._nextToken++;
return new Promise((resolve, reject) => {
this._revertPromises.make(token, resolve, reject);
this._worker.postMessage({
message: 'revert',
token: token,
reversion: reversion,
context: context
})
});
}
resetContext(context: Context) {
this._worker.postMessage({
message: 'reset-context',
context: context
});
}
// TODO: asynchronous close() method.
// Worker code must recognize message and call self.close().
private onMessage(event: MessageEvent): void {
let payload: OutgoingMessage = event.data;
if (payload.message === 'error') {
console.error(payload.log);
if(payload.error) {
console.error(payload.error);
}
}
else if (payload.message === 'ready') {
this._declareLMLayerReady(event.data.configuration);
} else if (payload.message === 'suggestions') {
this._predictPromises.keep(payload.token, payload.suggestions);
} else if (payload.message === 'currentword') {
this._wordbreakPromises.keep(payload.token, payload.word);
} else if (payload.message === 'postaccept') {
this._acceptPromises.keep(payload.token, payload.reversion);
} else if (payload.message === 'postrevert') {
this._revertPromises.keep(payload.token, payload.suggestions);
} else {
// This branch should never execute, but just in case...
//@ts-ignore
throw new Error(`Message not implemented: ${payload.message}`);
}
}
/**
* Clears out any computational resources in use by the LMLayer, including shutting
* down any internal WebWorkers.
*/
public shutdown() {
this._worker.terminate();
}
}

3
common/predictive-text/src/node/index.ts Normal file
View file

@ -0,0 +1,3 @@
export { default as LMLayer } from '../lmlayer.js';
export { default as Worker } from './worker.js';
export { default as SourcemappedWorker } from './sourcemappedWorker.js';

19
common/predictive-text/src/node/sourcemappedWorker.ts Normal file
View file

@ -0,0 +1,19 @@
import VirtualizedWorker from "./virtualizedWorker.js";
import unwrap from '../unwrap.js';
import { LMLayerWorkerCode, LMLayerWorkerSourcemapComment } from "@keymanapp/lm-worker/worker-main.wrapped.js";
export default class SourcemappedWorker {
static constructInstance(): Worker {
let scriptStr = unwrap(LMLayerWorkerCode);
// If this is definitively set to either true or false, tree-shaking can take effect.
// An imported const variable doesn't seem to do it, though.
// if(false) {
scriptStr += '\n' + LMLayerWorkerSourcemapComment;
// }
let worker = new VirtualizedWorker(scriptStr);
return worker as any as Worker;
}
}

26
common/predictive-text/src/node/tsconfig.json Normal file
View file

@ -0,0 +1,26 @@
{
"extends": "../../../../tsconfig-base.json",
"compilerOptions": {
"allowJs": false,
"declaration": true,
"module": "es6",
"moduleResolution": "Node16",
"inlineSources": true,
"sourceMap": true,
"sourceRoot": "keyman",
"target": "es5",
"lib": ["es6"],
"types": ["node"],
"baseUrl": "../",
"outDir": "../../build/obj",
"tsBuildInfoFile": "../../build/obj/node/tsconfig.tsbuildinfo",
"rootDir": "../"
},
"include": [
"*.ts"
],
"references": [
{ "path": "../tsconfig.json" },
{ "path": "../../../web/lm-worker" }
]
}

84
common/predictive-text/src/node/virtualizedWorker.ts Normal file
View file

@ -0,0 +1,84 @@
// We use a subset of the Worker interface here; compiling directly against the true
// WebWorker type definitions would require us to implement more methods than we do.
/// <reference path="../worker-interface.d.ts" />
import * as fs from 'fs';
import * as vm from 'vm';
class VirtualizedWorkerContext {
// The LMLayerWorker installs itself to 'self', the expected Worker global, so we provide an alias.
self: VirtualizedWorkerContext;
constructor() {
this.self = this;
}
postMessage: (message: any) => void;
importScripts(...scriptNames: string[]) {
/* Use of vm.createContext and script.runInContext allow us to avoid
* polluting the global scope with imports. When we throw away the
* context object, imported scripts will be automatically GC'd.
*/
for(let script of scriptNames) {
this.__importScriptString(fs.readFileSync(script, "UTF-8"));
}
}
__importScriptString(scriptStr: string) {
let context = vm.createContext(this);
var script = new vm.Script(scriptStr);
script.runInContext(context);
}
}
/**
* Note: this does not create an actual Worker, separate process, or thread. Everything will
* be executed in-line on a virtualized context.
*
* In the future, it might be nice to use Node's Worker Threads implementation.
*/
export default class VirtualizedWorker implements Worker {
private _workerContext: VirtualizedWorkerContext;
constructor(scriptStr: string) {
this._workerContext = new VirtualizedWorkerContext();
// Needs to exist before setting up the worker; must exist by `.install()`.
this._workerContext.postMessage = this.workerPostMessage.bind(this);
// Initialize the "worker".
this._workerContext.__importScriptString(scriptStr);
}
// Sends the worker's postMessage messages to the appropriate `onmessage` handler.
private workerPostMessage(message: unknown) {
if(this.onmessage) {
this.onmessage({data: message} as any as MessageEvent);
}
}
/**
* Accepts a callback function that will receive messages sent from the `VirtualizedWorker`'s `postMessage` function,
* much like the standard `Worker.onmessage`.
*/
onmessage: (this: Worker, ev: MessageEvent) => any;
postMessage(message: any) {
let msgObj = {data: message};
let msgJSON = JSON.stringify(msgObj);
/*
* Execute the command within the virtualized worker's scope. The worker's returned
* `postMessage` calls will still reach outside, as they have a reference to `this` via
* `postMessage` (which we've set to a bound `this.workerPostMessage`).
*
* Among other things, this will allow the worker to use its internal namespaces without issue.
*/
let msgCommand = "onmessage(" + msgJSON + ")";
this._workerContext.__importScriptString(msgCommand);
}
terminate(): void {
this._workerContext = null;
}
}

15
common/predictive-text/src/node/worker.ts Normal file
View file

@ -0,0 +1,15 @@
import VirtualizedWorker from "./virtualizedWorker.js";
import unwrap from '../unwrap.js';
import { LMLayerWorkerCode, LMLayerWorkerSourcemapComment } from "@keymanapp/lm-worker/worker-main.wrapped.min.js";
export default class Worker {
static constructInstance(): Worker {
let scriptStr = unwrap(LMLayerWorkerCode);
scriptStr += '\n' + LMLayerWorkerSourcemapComment;
let worker = new VirtualizedWorker(scriptStr);
return worker as any as Worker;
}
}

90
common/predictive-text/src/promise-store.ts Normal file
View file

@ -0,0 +1,90 @@
/*
* Copyright (c) 2018 National Research Council Canada (author: Eddie A. Santos)
* Copyright (c) 2018 SIL International
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to
* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
* the Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
/// <reference types="@keymanapp/lm-message-types" />
type Resolve<T> = (value?: T | PromiseLike<T>) => void;
type Reject = (reason?: any) => void;
interface PromiseCallbacks<T> {
resolve: Resolve<T>;
reject: Reject;
}
/**
* Associate tokens with promises.
*
* First, .make() a promise -- associate a token with resolve/reject callbacks.
*
* You can either .keep() a promise -- resolve() and forget it;
* Or you may also .break() a promise -- reject() and forget it.
*
* <T> is the type of resolved value (value yielded successfully by promise).
*/
export default class PromiseStore<T> {
// IE11 offers partial support for new Map().
// Assume only .get(), .set(), .has(), .delete(), and .size work.
// See: http://kangax.github.io/compat-table/es6/#test-Map
private _promises: Map<Token, PromiseCallbacks<T>>;
constructor() {
this._promises = new Map();
}
/**
* How many promises are currently being tracked?
*/
get length(): number {
return this._promises.size;
}
/**
* Associate a token with its respective resolve and reject callbacks.
*/
make(token: Token, resolve: Resolve<T>, reject: Reject): void {
if (this._promises.has(token)) {
return reject(`Existing request with token ${token}`);
}
this._promises.set(token, { reject, resolve });
}
/**
* Resolve the promise associated with a token (with a value!).
* Once the promise is resolved, the token is removed..
*/
keep(token: Token, value: T) {
let callbacks = this._promises.get(token);
if (!callbacks) {
throw new Error(`No promise associated with token: ${token}`);
}
let accept = callbacks.resolve;
this._promises.delete(token);
return accept(value);
}
/**
* Instantly reject and forget a promise associated with the token.
*/
break(token: Token, reason?: any): void {
let callbacks = this._promises.get(token);
if (!callbacks) {
throw new Error(`No promise associated with token: ${token}`);
}
this._promises.delete(token);
callbacks.reject(reason);
}
}

33
common/predictive-text/src/tsconfig.json Normal file
View file

@ -0,0 +1,33 @@
{
"extends": "../../../tsconfig-base.json",
"compilerOptions": {
"allowJs": false,
"declaration": true,
"module": "es6",
"moduleResolution": "Node16",
"inlineSources": true,
"sourceMap": true,
"sourceRoot": "keyman",
"target": "es5",
"lib": ["es6"],
"types": ["node"],
"baseUrl": "./",
"outDir": "../build/obj",
"tsBuildInfoFile": "../build/obj/tsconfig.tsbuildinfo",
"rootDir": "./"
},
"references": [
{ "path": "../../web/utils" },
{ "path": "../../models/types"},
{ "path": "../../models/wordbreakers"},
{ "path": "../../web/lm-message-types"},
{ "path": "../../models/templates"}
],
"include" : [ "./*.ts" ],
"exclude" : [
"node",
"web",
"index.ts" // It's handled by the final layer.
]
}

11
common/predictive-text/src/unwrap.ts Normal file
View file

@ -0,0 +1,11 @@
/**
* Given a function, this utility returns the source code within it, as a string.
* This is intended to unwrap the "wrapped" source code created in the LMLayerWorker
* build process.
*
* @param fn The function whose body will be returned.
*/
export default function unwrap(encodedSrc: string): string {
let wrapper = decodeURIComponent(encodedSrc);
return wrapper;
}

3
common/predictive-text/src/web/index.ts Normal file
View file

@ -0,0 +1,3 @@
export { default as LMLayer } from '../lmlayer.js';
export { default as Worker } from './worker.js';
export { default as SourcemappedWorker } from './sourcemappedWorker.js';

34
common/predictive-text/src/web/sourcemappedWorker.ts Normal file
View file

@ -0,0 +1,34 @@
import unwrap from '../unwrap.js';
import { LMLayerWorkerCode, LMLayerWorkerSourcemapComment } from "@keymanapp/lm-worker/worker-main.wrapped.js";
export default class SourcemappedWorker {
static constructInstance(): Worker {
return new Worker(this.asBlobURI(LMLayerWorkerCode));
}
/**
* Converts the INSIDE of a function into a blob URI that can
* be passed as a valid URI for a Worker.
* @param fn Function whose body will be referenced by a URI.
*
* This function makes the following possible:
*
* let worker = new Worker(LMLayer.asBlobURI(function myWorkerCode () {
* postMessage('inside Web Worker')
* function onmessage(event) {
* // handle message inside Web Worker.
* }
* }));
*/
static asBlobURI(encodedSrc: string): string {
let code = unwrap(encodedSrc);
// If this is definitively set to either true or false, tree-shaking can take effect.
// An imported const variable doesn't seem to do it, though.
// if(false) {
code += '\n' + LMLayerWorkerSourcemapComment;
// }
let blob = new Blob([code], { type: 'text/javascript' });
return URL.createObjectURL(blob);
}
}

25
common/predictive-text/src/web/tsconfig.json Normal file
View file

@ -0,0 +1,25 @@
{
"extends": "../../../../tsconfig-base.json",
"compilerOptions": {
"allowJs": false,
"declaration": true,
"module": "es6",
"moduleResolution": "Node16",
"inlineSources": true,
"sourceMap": true,
"sourceRoot": "keyman",
"target": "es5",
"lib": ["es6", "DOM"],
"types": ["node"],
"baseUrl": "../",
"outDir": "../../build/obj",
"tsBuildInfoFile": "../../build/obj/web/tsconfig.tsbuildinfo",
"rootDir": "../"
},
"include": [
"*.ts"
],
"references": [
{ "path": "../tsconfig.json" }
]
}

34
common/predictive-text/src/web/worker.ts Normal file
View file

@ -0,0 +1,34 @@
import unwrap from '../unwrap.js';
import { LMLayerWorkerCode, LMLayerWorkerSourcemapComment } from "@keymanapp/lm-worker/worker-main.wrapped.min.js";
export default class DefaultWorker {
static constructInstance(): Worker {
return new Worker(this.asBlobURI(LMLayerWorkerCode));
}
/**
* Converts the INSIDE of a function into a blob URI that can
* be passed as a valid URI for a Worker.
* @param fn Function whose body will be referenced by a URI.
*
* This function makes the following possible:
*
* let worker = new Worker(LMLayer.asBlobURI(function myWorkerCode () {
* postMessage('inside Web Worker')
* function onmessage(event) {
* // handle message inside Web Worker.
* }
* }));
*/
static asBlobURI(encodedSrc: string): string {
let code = unwrap(encodedSrc);
// If this is definitively set to either true or false, tree-shaking can take effect.
// An imported const variable doesn't seem to do it, though.
// if(false) {
code += '\n' + LMLayerWorkerSourcemapComment;
// }
let blob = new Blob([code], { type: 'text/javascript' });
return URL.createObjectURL(blob);
}
}

0
common/predictive-text/worker-interface.d.ts → common/predictive-text/src/worker-interface.d.ts vendored
View file

34
common/predictive-text/tsconfig.all.json Normal file
View file

@ -0,0 +1,34 @@
{
// Due to the heterogenous target mix supported by the LMLayer, the two different "worker" types
// require separate sub-tsconfigs, both of which require common references. The multi-config
// setup in this project and its subfolders ensure all modules can compile cleanly and in the
// necessary compilation order.
"extends": "../../tsconfig-base.json",
"compilerOptions": {
"allowJs": false,
"declaration": true,
"module": "es6",
"moduleResolution": "Node16",
"inlineSources": true,
"sourceMap": true,
"sourceRoot": "keyman",
"target": "es5",
"lib": ["es6"],
"types": ["node"],
"baseUrl": "./",
"outDir": "build/obj",
"tsBuildInfoFile": "build/obj/tsconfig.all.tsbuildinfo",
"rootDir": "./src"
},
"files": [],
"references": [
{ "path": "../web/utils" },
{ "path": "../models/types"},
{ "path": "../models/wordbreakers"},
{ "path": "../web/lm-message-types"},
{ "path": "../models/templates"},
{ "path": "src/node" },
{ "path": "src/web" }
],
}

32
common/predictive-text/tsconfig.json
View file

@ -1,32 +0,0 @@
{
"extends": "../../tsconfig-base.json",
"compilerOptions": {
"allowJs": true,
"outFile": "build/headless.js",
"module": "none",
"inlineSources": true,
"sourceMap": true,
"target": "es5",
"lib": ["es6"]
},
"references": [
{ "path": "../web/utils" },
{ "path": "../models/types"},
{ "path": "../models/wordbreakers"},
{ "path": "../web/lm-message-types"},
{ "path": "../models/templates"},
],
"include" : [ "./*.ts" ],
"exclude" : [
"web-defaultWorker.ts",
"node_modules",
"testing"
],
"files": [
// By renaming the output JS as TS, and including the @ts-nocheck directive, we can
// bypass wrapped-worker typing issues while still bundling the file as part of the
// tsc build!
"../web/lm-worker/build/index.wrapped-for-bundle.ts"
]
}

7
common/predictive-text/unit_tests/headless/promise-store.js
View file

@ -1,7 +1,8 @@
var assert = require('chai').assert;
var sinon = require('sinon');
import { assert } from 'chai';
import sinon from 'sinon';
let PromiseStore = require('../../build/headless').PromiseStore;
import PromiseStore from '#./promise-store.js';
import { randomToken } from '@keymanapp/common-test-resources/model-helpers.mjs';
describe('PromiseStore', function () {
describe('.make()', function () {

56
common/predictive-text/unit_tests/headless/top-level-lmlayer.js
View file

@ -1,11 +1,29 @@
var assert = require('chai').assert;
var sinon = require('sinon');
import { assert } from 'chai';
import sinon from 'sinon';
let LMLayer = require('../../build/headless');
import { createRequire } from 'module';
const require = createRequire(import.meta.url);
import { LMLayer } from '#./node/index.js';
import { capabilities } from '@keymanapp/common-test-resources/model-helpers.mjs';
// Test the top-level LMLayer interface.
// Note: these tests can only be run after BOTH stages of compilation are completed.
describe('LMLayer', function() {
/**
* Returns an object implementing *enough* of the Worker
* interface to fool the LMLayer into thinking it's
* communicating with a bona fide Web Worker.
*
* @returns {Worker} an object with sinon.fake() instances.
*/
function createFakeWorker(postMessage) {
return {
postMessage: postMessage ? sinon.fake(postMessage) : sinon.fake(),
onmessage: null
};
}
describe('[[constructor]]', function () {
it('should accept a Worker to instantiate', function () {
new LMLayer(capabilities(), createFakeWorker());
@ -29,7 +47,7 @@ describe('LMLayer', function() {
let fakeWorker = createFakeWorker();
let lmLayer = new LMLayer(capabilities(), fakeWorker);
lmLayer.loadModel("./unit_tests/in_browser/resources/models/simple-dummy.js");
lmLayer.loadModel(require.resolve("@keymanapp/common-test-resources/models/simple-dummy.js"));
assert.isFunction(fakeWorker.onmessage, 'LMLayer failed to set a callback!');
});
@ -37,7 +55,7 @@ describe('LMLayer', function() {
it('should send the `load` message to the LMLayer', async function () {
let fakeWorker = createFakeWorker(fakePostMessage);
let lmLayer = new LMLayer(capabilities(), fakeWorker);
let configuration = await lmLayer.loadModel("./unit_tests/in_browser/resources/models/simple-dummy.js");
let configuration = await lmLayer.loadModel(require.resolve("@keymanapp/common-test-resources/models/simple-dummy.js"));
assert.propertyVal(fakeWorker.postMessage, 'callCount', 2);
// In the "Worker", assert the message looks right and
@ -95,34 +113,6 @@ describe('LMLayer', function() {
})
});
// Since the Blob API is limited to browsers, look for those
// tests for .asBlobURI() in the in_browser tests.
describe('.unwrap', function () {
it('should return the inner code of a function', function () {
// Create a multi-line function body we can match in a RegExp.
let text = LMLayer.unwrap(function hello() {
var hello;
var world;
});
// Unwrap should give us back ONLY the body. Whitespace isn't really important.
assert.match(text, /^\s*var\s+hello;\s*var\s+world;\s*$/);
});
});
/**
* Returns an object implementing *enough* of the Worker
* interface to fool the LMLayer into thinking it's
* communicating with a bona fide Web Worker.
*
* @returns {Worker} an object with sinon.fake() instances.
*/
function createFakeWorker(postMessage) {
return {
postMessage: postMessage ? sinon.fake(postMessage) : sinon.fake(),
onmessage: null
};
}
/**
* Call a function in the future, i.e., later in the event loop.
* The call does NOT block the current execution.

24
common/predictive-text/unit_tests/headless/worker-dummy-integration.js
View file

@ -1,9 +1,11 @@
var assert = require('chai').assert;
var fs = require('fs');
let LMLayer = require('../../build/headless');
import { assert } from 'chai';
import fs from 'fs';
// Load the LMLayerWorkerCode function into this context
require('node:vm').runInThisContext(fs.readFileSync(__dirname + '/../../../web/lm-worker/build/index.js', 'utf-8'));
import { createRequire } from 'module';
const require = createRequire(import.meta.url);
import { LMLayer, SourcemappedWorker as Worker } from '#./node/index.js';
import { capabilities, iGotDistractedByHazel } from '@keymanapp/common-test-resources/model-helpers.mjs';
/*
* Shows off the LMLayer API, using the full prediction interface.
@ -16,7 +18,7 @@ require('node:vm').runInThisContext(fs.readFileSync(__dirname + '/../../../web/l
describe('LMLayer using dummy model', function () {
describe('Prediction', function () {
it('will predict future suggestions (loaded from file)', function () {
var lmLayer = new LMLayer(capabilities());
var lmLayer = new LMLayer(capabilities(), Worker.constructInstance());
var stripIDs = function(suggestions) {
suggestions.forEach(function(suggestion) {
@ -29,7 +31,7 @@ describe('LMLayer using dummy model', function () {
// Not done yet, as this test case is a slightly-edited copy of the in-browser version.
return lmLayer.loadModel(
// We're running headlessly, so the path can be relative to the npm root directory.
"unit_tests/in_browser/resources/models/simple-dummy.js"
require.resolve("@keymanapp/common-test-resources/models/simple-dummy.js")
).then(function (actualConfiguration) {
return Promise.resolve();
}).then(function () {
@ -55,7 +57,7 @@ describe('LMLayer using dummy model', function () {
});
it('will predict future suggestions (loaded from raw source)', function () {
var lmLayer = new LMLayer(capabilities());
var lmLayer = new LMLayer(capabilities(), Worker.constructInstance());
var stripIDs = function(suggestions) {
suggestions.forEach(function(suggestion) {
@ -64,7 +66,7 @@ describe('LMLayer using dummy model', function () {
};
// We're running headlessly, so the path can be relative to the npm root directory.
let modelCode = fs.readFileSync("./unit_tests/in_browser/resources/models/simple-dummy.js").toString();
let modelCode = fs.readFileSync(require.resolve("@keymanapp/common-test-resources/models/simple-dummy.js")).toString();
// We're testing many as asynchronous messages in a row.
// this would be cleaner using async/await syntax.
@ -98,14 +100,14 @@ describe('LMLayer using dummy model', function () {
describe('Wordbreaking', function () {
it('will perform (default) wordbreaking and return word at caret', function () {
var lmLayer = new LMLayer(capabilities());
var lmLayer = new LMLayer(capabilities(), Worker.constructInstance());
// We're testing many as asynchronous messages in a row.
// this would be cleaner using async/await syntax.
// Not done yet, as this test case is a slightly-edited copy of the in-browser version.
return lmLayer.loadModel(
// We're running headlessly, so the path can be relative to the npm root directory.
"unit_tests/in_browser/resources/models/simple-dummy.js"
require.resolve("@keymanapp/common-test-resources/models/simple-dummy.js")
).then(function (actualConfiguration) {
return Promise.resolve();
}).then(function () {

17
common/predictive-text/unit_tests/headless/worker-trie-integration.js
View file

@ -1,5 +1,10 @@
var assert = require('chai').assert;
let LMLayer = require('../../build/headless');
import { assert } from 'chai';
import { createRequire } from 'module';
const require = createRequire(import.meta.url);
import { LMLayer, SourcemappedWorker as Worker } from '#./node/index.js';
import { capabilities } from '@keymanapp/common-test-resources/model-helpers.mjs';
/*
* How to run the worlist
@ -8,14 +13,14 @@ describe('LMLayer using the trie model', function () {
describe('Prediction', function () {
var EXPECTED_SUGGESTIONS = 3;
it('will predict an empty buffer', function () {
var lmLayer = new LMLayer(capabilities());
var lmLayer = new LMLayer(capabilities(), Worker.constructInstance());
// We're testing many as asynchronous messages in a row.
// this would be cleaner using async/await syntax.
// Not done yet, as this test case is a slightly-edited copy of the in-browser version.
return lmLayer.loadModel(
// We're running headlessly, so the path can be relative to the npm root directory.
"unit_tests/in_browser/resources/models/simple-trie.js"
require.resolve("@keymanapp/common-test-resources/models/simple-trie.js")
).then(function (_actualConfiguration) {
return Promise.resolve();
}).then(function () {
@ -50,11 +55,11 @@ describe('LMLayer using the trie model', function () {
//
// https://community.software.sil.org/t/search-term-to-key-in-lexical-model-not-working-both-ways-by-default/3133
it('should use the default searchTermToKey()', function () {
var lmLayer = new LMLayer(capabilities());
var lmLayer = new LMLayer(capabilities(), Worker.constructInstance());
return lmLayer.loadModel(
// We're running headlessly, so the path can be relative to the npm root directory.
"unit_tests/in_browser/resources/models/naive-trie.js"
require.resolve("@keymanapp/common-test-resources/models/naive-trie.js")
).then(function (_actualConfiguration) {
return Promise.resolve();
}).then(function () {

139
common/predictive-text/unit_tests/helpers.js
View file

@ -1,139 +0,0 @@
/**
* @file helpers
*
* Globally-defined helper functions for use in in Mocha tests.
*/
var fs = require("fs");
var vm = require("vm");
// Choose the appropriate global object. Either `global` in
// Node, or `window` in browsers.
var _ = global || window;
/**
* Creates a MessageEvent (for inter-worker communication), with the given data payload.
*
* @param {*} data
*/
_.createMessageEventWithData = function createMessageEventWithData(data) {
return { data };
}
/**
* Creates a simple, default capabilities object for standard-case LMLayer init.
*/
_.capabilities = function capabilities() {
return {
maxLeftContextCodePoints: 64
}
}
/**
* Mimics a message from the outer LMLayer shell with a simple, default config object.
* Used for Worker tests.
*/
_.configWorker = function configWorker(worker) {
worker.onMessage(createMessageEventWithData({
message: 'config',
capabilities: _.capabilities()
}));
}
/**
* A valid model that suggests exactly what you want it to suggest.
*
* @returns {ModelDescription}
*/
_.dummyModel = function dummyModel(futureSuggestions) {
return {
type: 'dummy',
futureSuggestions: futureSuggestions || []
};
}
/**
* Capabilities of a keyboard that will ONLY send left-sided capabilities.
* The keyboard does not support deleting to the right.
*
* @returns {Capabilities}
*/
_.defaultCapabilities = function defaultCapabilities() {
return {
maxLeftContextCodeUnits: 64
};
}
/**
* Returns the Context of an empty buffer; no text, at both the start and
* end of the buffer.
*
* @returns {Context}
*/
_.emptyContext = function emptyContext() {
return {
left: '',
startOfBuffer: true,
endOfBuffer: true
};
}
/**
* Returns a Transform that, when applied, makes no changes to the buffer.
*
* @returns {Transform}
*/
_.zeroTransform = function zeroTransform() {
return {
insert: '',
deleteLeft: 0,
};
}
/**
* Returns a random token. NOT guaranteed to be unique.
*
* @returns {Token}
*/
_.randomToken = function randomToken() {
var range = Number.MAX_SAFE_INTEGER - Number.MIN_SAFE_INTEGER;
return Math.random() * range + Number.MIN_SAFE_INTEGER;
}
// Use fixtures used in browser tests IN NODE!
if (typeof require === 'function') {
_.iGotDistractedByHazel = function () {
return jsonFixture('future_suggestions/i_got_distracted_by_hazel');
}
/**
* Return a JSON fixture
*/
_.jsonFixture = function (name) {
// Assuming this file structure:
// .
// ├── helpers.js
// └── in_browser
// └── json
// ├── future_suggestions
// │ └── ...
// └── wordlists
// └── ...
return require('./in_browser/json/' + name);
}
// This worker-global function does not exist by default in Node!
_.importScriptsWith = function(context) {
return function() { // the constructed context's importScripts method.
/* Use of vm.createContext and script.runInContext allow us to avoid
* polluting the global scope with imports. When we throw away the
* context object, imported scripts will be automatically GC'd.
*/
for(var i=0; i < arguments.length; i++) {
context = vm.createContext(context);
var script = new vm.Script(fs.readFileSync(arguments[i]));
script.runInContext(context);
}
}
}
}

4
common/predictive-text/unit_tests/in_browser/CI.conf.cjs Normal file
View file

@ -0,0 +1,4 @@
var BASE_CONFIG = require("./base.conf.cjs");
var ci_config_adapter = require("../../../test/resources/karma-browserstack-config.cjs");
module.exports = ci_config_adapter(BASE_CONFIG, "full");

4
common/predictive-text/unit_tests/in_browser/CI.conf.js
View file

@ -1,4 +0,0 @@
var BASE_CONFIG = require("./base.conf.js");
var ci_config_adapter = require("../../../../common/test/resources/karma-browserstack-config.js");
module.exports = ci_config_adapter(BASE_CONFIG, "full");

37
common/predictive-text/unit_tests/in_browser/base.conf.js → common/predictive-text/unit_tests/in_browser/base.conf.cjs
View file

@ -4,7 +4,7 @@
module.exports = {
// base path that will be used to resolve all patterns (eg. files, exclude)
basePath: './',
basePath: '../../../../', // Root directory of the repository(!)
client: {
/* `client.args` here is passed to the test runner page as `__karma__.config.args`.
@ -31,16 +31,27 @@ module.exports = {
// list of files / patterns to load in the browser
files: [
// Include the generated worker code. Make sure it's linked before any of the test cases.
'../../../web/lm-worker/build/index.js',
'../../build/index.js',
'../../../../common/test/resources/timeout-adapter.js',
'helpers.js', // Provides utility helpers and objects for tests.
'cases/**/*.js', // Where the tests actually reside.
// { pattern: 'node_modules/**/*.js', served: true, watched: false, included: false },
// { pattern: 'node_modules/**/*.mjs', served: true, watched: false, included: false, type: "module" },
// { pattern: 'node_modules/**/*.cjs', served: true, watched: false, included: false },
// Provides utility helpers and objects for tests.
{ pattern: 'common/predictive-text/unit_tests/in_browser/helpers.mjs', served: true, watched: true, included: false},
// Where the tests actually reside.
{ pattern: 'common/predictive-text/unit_tests/in_browser/cases/**/*.js', type: 'module' },
'common/test/resources/*.js',
'common/test/resources/json/models/**/*.json',
{ pattern: 'common/web/lm-worker/build/lib/*.js', watched: true, served: true, included: false},
{ pattern: 'common/web/lm-worker/build/lib/*.js.map', watched: true, served: true, included: false},
{ pattern: 'common/predictive-text/build/obj/**/*.*', watched: true, served: true, included: false },
{ pattern: 'common/predictive-text/build/obj/**/*.js.map', watched: true, served: true, included: false },
{ pattern: 'common/predictive-text/build/lib/**/*.*', watched: true, served: true, included: false },
{ pattern: 'common/predictive-text/build/lib/**/*.js.map', watched: true, served: true, included: false },
// We don't have anything in these locations... yet. But they'll be useful for test resources.
'json/**/*.json', // Where pre-loaded JSON resides.
{pattern: 'resources/**/*.*', watched: true, served: true, included: false}, // General testing resources.
{pattern: 'common/test/resources/**/*.*', watched: true, served: true, included: false}, // General testing resources.
],
// list of files / patterns to exclude
@ -50,17 +61,19 @@ module.exports = {
// preprocess matching files before serving them to the browser
// available preprocessors: https://npmjs.org/browse/keyword/karma-preprocessor
preprocessors: {
'json/**/*.json' : ['json_fixtures']
'common/test/resources/json/models/**/*.json' : ['json_fixtures']
},
// Settings to properly configure how JSON fixtures are automatically loaded by Karma.
jsonFixturesPreprocessor: {
stripPrefix: 'json/',
stripPrefix: 'common/test/resources/json/',
variableName: '__json__'
},
proxies: {
"/resources/": "/base/resources/"
"/resources/": "/base/common/test/resources/",
// "/node_modules/": "/base/node_modules/",
// "/@keymanapp/lm-worker/": "/base/node_modules/@keymanapp/lm-worker/"
},
// web server port

11
common/predictive-text/unit_tests/in_browser/cases/top-level-lmlayer.js
View file

@ -1,22 +1,23 @@
var assert = chai.assert;
var LMLayer = com.keyman.text.prediction.LMLayer;
var DefaultWorker = com.keyman.text.prediction.DefaultWorker;
import { LMLayer, Worker as WorkerBuilder } from "../../../build/lib/web/index.mjs";
import * as helpers from "../helpers.mjs";
describe('LMLayer', function () {
this.timeout(testconfig.timeouts.standard);
describe('[[constructor]]', function () {
it('should construct with a single argument', function () {
let lmLayer = new LMLayer(helpers.defaultCapabilities, null, true);
let lmLayer = new LMLayer(helpers.defaultCapabilities, WorkerBuilder.constructInstance(), true);
assert.instanceOf(lmLayer, LMLayer);
lmLayer.shutdown();
});
});
describe('#asBlobURI()', function () {
describe.skip('#asBlobURI()', function () {
// #asBlobURI() requires browser APIs, hence why it cannot be tested headless in Node.
it('should take a function and convert it into a blob function', function (done) {
let uri = DefaultWorker.asBlobURI(function dummyHandler() {
let uri = WorkerBuilder.asBlobURI(function dummyHandler() {
// Post something weird, so we can be reasonably certain the Web Worker is...
// well, working.
// WARNING: Do NOT refactor this string as a variable. It **MUST** remain a string

10
common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.js
View file

@ -1,5 +1,7 @@
var assert = chai.assert;
var LMLayer = com.keyman.text.prediction.LMLayer;
import { LMLayer, Worker } from "../../../build/lib/web/index.mjs";
import * as helpers from "../helpers.mjs";
/*
* Shows off the LMLayer API, using the full prediction interface.
@ -17,7 +19,7 @@ describe('LMLayer using dummy model', function () {
this.timeout(testconfig.timeouts.standard * 3); // This one makes multiple subsequent calls across
// the WebWorker boundary, so we should be generous here.
var lmLayer = new LMLayer(helpers.defaultCapabilities, null, true);
var lmLayer = new LMLayer(helpers.defaultCapabilities, Worker.constructInstance(), true);
var stripIDs = function(suggestions) {
suggestions.forEach(function(suggestion) {
@ -60,7 +62,7 @@ describe('LMLayer using dummy model', function () {
it('will perform (default) wordbreaking and return word at caret', function () {
this.timeout(testconfig.timeouts.standard * 3); // This one makes multiple subsequent calls across
// the WebWorker boundary, so we should be generous here.
var lmLayer = new LMLayer(helpers.defaultCapabilities);
var lmLayer = new LMLayer(helpers.defaultCapabilities, Worker.constructInstance());
// We're testing many as asynchronous messages in a row.
// this would be cleaner using async/await syntax, but
@ -94,6 +96,6 @@ describe('LMLayer using dummy model', function () {
}
function iGotDistractedByHazel() {
return __json__['future_suggestions/i_got_distracted_by_hazel'];
return __json__['models/future_suggestions/i_got_distracted_by_hazel'];
}
});

8
common/predictive-text/unit_tests/in_browser/cases/worker-trie-integration.js
View file

@ -1,5 +1,7 @@
var assert = chai.assert;
var LMLayer = com.keyman.text.prediction.LMLayer;
import { LMLayer, Worker } from "../../../build/lib/web/index.mjs";
import * as helpers from "../helpers.mjs";
/*
* How to run the worlist
@ -16,7 +18,7 @@ describe('LMLayer using the trie model', function () {
// Parameter 3 = true: enables 'test mode', disables correction-search timeout.
// This helps prevent the correction-search timeout from flaking out periodically during unit tests in
// CI, since remote servers / devices are involved.
var lmLayer = new LMLayer(helpers.defaultCapabilities, null, true);
var lmLayer = new LMLayer(helpers.defaultCapabilities, Worker.constructInstance(), true);
// We're testing many as asynchronous messages in a row.
// this would be cleaner using async/await syntax, but
@ -58,7 +60,7 @@ describe('LMLayer using the trie model', function () {
//
// https://community.software.sil.org/t/search-term-to-key-in-lexical-model-not-working-both-ways-by-default/3133
it('should use the default searchTermToKey()', function () {
var lmLayer = new LMLayer(helpers.defaultCapabilities);
var lmLayer = new LMLayer(helpers.defaultCapabilities, Worker.constructInstance());
return lmLayer.loadModel(
// We need to provide an absolute path since the worker is based within a blob.

15
common/predictive-text/unit_tests/in_browser/cases/worker.js
View file

@ -1,6 +1,8 @@
var assert = chai.assert;
var LMLayer = com.keyman.text.prediction.LMLayer;
var DefaultWorker = com.keyman.text.prediction.DefaultWorker;
import { Worker as WorkerBuilder } from "../../../build/lib/web/index.mjs";
import { LMLayerWorkerCode } from "/base/common/web/lm-worker/build/lib/worker-main.wrapped.min.js";
import * as helpers from "../helpers.mjs";
describe('LMLayerWorker', function () {
// This one makes multiple subsequent calls across the WebWorker boundary, so we should be generous here.
@ -8,15 +10,16 @@ describe('LMLayerWorker', function () {
describe('LMLayerWorkerCode', function() {
it('should exist!', function() {
assert.isFunction(LMLayerWorkerCode,
'Could not find LMLayerWorkerCode! Does embedded_worker.js exist?'
);
// assert.isFunction(LMLayerWorkerCode,
// 'Could not find LMLayerWorkerCode! Does embedded_worker.js exist?'
// );
assert.isString(LMLayerWorkerCode);
});
});
describe('Usage within a Web Worker', function () {
it('should install itself in the worker context', function (done) {
let uri = DefaultWorker.asBlobURI(LMLayerWorkerCode);
let uri = WorkerBuilder.asBlobURI(LMLayerWorkerCode);
let worker = new Worker(uri);
worker.onmessage = function thisShouldBeCalled(message) {
done();

8
common/predictive-text/unit_tests/in_browser/helpers.js
View file

@ -1,8 +0,0 @@
var helpers;
// Establishes the equivalent of a TS namespace.
(function(helpers){
helpers.defaultCapabilities = {
maxLeftContextCodeUnits: 64
};
})(helpers || (helpers = {}));

3
common/predictive-text/unit_tests/in_browser/helpers.mjs Normal file
View file

@ -0,0 +1,3 @@
export let defaultCapabilities = {
maxLeftContextCodeUnits: 64
};

2
common/predictive-text/unit_tests/in_browser/manual.conf.js → common/predictive-text/unit_tests/in_browser/manual.conf.cjs
View file

@ -1,5 +1,5 @@
module.exports = function(config) {
var base = require("./base.conf.js");
var base = require("./base.conf.cjs");
var specifics = {
// test results reporter to use

16
common/predictive-text/unit_tests/test.sh
View file

@ -15,7 +15,7 @@ cd "$THIS_SCRIPT_PATH"
################################ Main script ################################
# Defaults
FLAGS="--require ./unit_tests/helpers"
FLAGS=""
builder_describe "Runs all tests for the language-modeling / predictive-text layer module" \
"configure" \
@ -74,6 +74,12 @@ if builder_start_action test:libraries; then
npm run test
popd
pushd "$KEYMAN_ROOT/common/web/lm-worker"
echo
echo "### Running ${BUILDER_TERM_START}common/web/lm-worker${BUILDER_TERM_END} tests"
./build.sh test
popd
builder_finish_action success test:libraries
fi
@ -122,13 +128,13 @@ if builder_start_action test:browser; then
if builder_has_option --ci; then
KARMA_FLAGS="$KARMA_FLAGS --reporters teamcity,BrowserStack"
KARMA_CONFIG="CI.conf.js"
KARMA_CONFIG="CI.conf.cjs"
KARMA_INFO_LEVEL="--log-level=debug"
else
KARMA_CONFIG="manual.conf.js"
KARMA_CONFIG="manual.conf.cjs"
if builder_is_debug_build; then
KARMA_FLAGS="$KARMA_FLAGS --no-single-run"
KARMA_CONFIG="manual.conf.js"
KARMA_CONFIG="manual.conf.cjs"
KARMA_INFO_LEVEL="--log-level=debug"
echo
@ -137,7 +143,7 @@ if builder_start_action test:browser; then
fi
fi
if [[ KARMA_CONFIG == "manual.conf.js" ]]; then
if [[ KARMA_CONFIG == "manual.conf.cjs" ]]; then
get_browser_set_for_OS
else
BROWSERS=

88
common/predictive-text/virtualizedWorker.ts
View file

@ -1,88 +0,0 @@
/// <reference path="embedded_worker.d.ts" />
/// <reference path="worker-interface.d.ts" />
namespace com.keyman.text.prediction {
// Unfortunately, importing like this loses all type information for the modules.
// It's that, or massively overhaul lm-layer to be module-based instead of
// namespace-based.
var fs = require("fs");
var vm = require("vm");
class VirtualizedWorkerContext {
// The LMLayerWorker installs itself to 'self', the expected Worker global, so we provide an alias.
self: VirtualizedWorkerContext;
constructor() {
this.self = this;
}
postMessage: (message: any) => void;
importScripts(...scriptNames: string[]) {
/* Use of vm.createContext and script.runInContext allow us to avoid
* polluting the global scope with imports. When we throw away the
* context object, imported scripts will be automatically GC'd.
*/
for(let script of scriptNames) {
this.__importScriptString(fs.readFileSync(script, "UTF-8"));
}
}
__importScriptString(scriptStr: string) {
let context = vm.createContext(this);
var script = new vm.Script(scriptStr);
script.runInContext(context);
}
}
/**
* Note: this does not create an actual Worker, separate process, or thread. Everything will
* be executed in-line on a virtualized context.
*
* In the future, it might be nice to use Node's Worker Threads implementation.
*/
export class VirtualizedWorker implements Worker {
private _workerContext: VirtualizedWorkerContext;
constructor(scriptStr: string) {
this._workerContext = new VirtualizedWorkerContext();
// Needs to exist before setting up the worker; must exist by `.install()`.
this._workerContext.postMessage = this.workerPostMessage.bind(this);
// Initialize the "worker".
this._workerContext.__importScriptString(scriptStr);
}
// Sends the worker's postMessage messages to the appropriate `onmessage` handler.
private workerPostMessage(message: unknown) {
if(this.onmessage) {
this.onmessage({data: message} as any as MessageEvent);
}
}
/**
* Accepts a callback function that will receive messages sent from the `VirtualizedWorker`'s `postMessage` function,
* much like the standard `Worker.onmessage`.
*/
onmessage: (this: Worker, ev: MessageEvent) => any;
postMessage(message: any) {
let msgObj = {data: message};
let msgJSON = JSON.stringify(msgObj);
/*
* Execute the command within the virtualized worker's scope. The worker's returned
* `postMessage` calls will still reach outside, as they have a reference to `this` via
* `postMessage` (which we've set to a bound `this.workerPostMessage`).
*
* Among other things, this will allow the worker to use its internal namespaces without issue.
*/
let msgCommand = "onmessage(" + msgJSON + ")";
this._workerContext.__importScriptString(msgCommand);
}
terminate(): void {
this._workerContext = null;
}
}
}

27
common/predictive-text/web-defaultWorker.ts
View file

@ -1,27 +0,0 @@
namespace com.keyman.text.prediction {
export class DefaultWorker {
static constructInstance(): Worker {
return new Worker(this.asBlobURI(LMLayerWorkerCode));
}
/**
* Converts the INSIDE of a function into a blob URI that can
* be passed as a valid URI for a Worker.
* @param fn Function whose body will be referenced by a URI.
*
* This function makes the following possible:
*
* let worker = new Worker(LMLayer.asBlobURI(function myWorkerCode () {
* postMessage('inside Web Worker')
* function onmessage(event) {
* // handle message inside Web Worker.
* }
* }));
*/
static asBlobURI(fn: Function): string {
let code = LMLayer.unwrap(fn);
let blob = new Blob([code], { type: 'text/javascript' });
return URL.createObjectURL(blob);
}
}
}

3
common/test/resources/index.mjs Normal file
View file

@ -0,0 +1,3 @@
// Exists to facilitate relative path resolution via Node's require.resolve function.
export { };

0
common/predictive-text/unit_tests/in_browser/json/future_suggestions/i_got_distracted_by_hazel.json → common/test/resources/json/models/future_suggestions/i_got_distracted_by_hazel.json
View file

0
common/predictive-text/unit_tests/in_browser/json/tries/accented.json → common/test/resources/json/models/tries/accented.json
View file

0
common/predictive-text/unit_tests/in_browser/json/tries/english-1000.json → common/test/resources/json/models/tries/english-1000.json
View file

0
common/test/resources/karma-browserstack-config.js → common/test/resources/karma-browserstack-config.cjs
View file

6
common/test/resources/keyboards/khmer_angkor.js
View file

@ -6,8 +6,8 @@ KeymanWeb.KR(new Keyboard_khmer_angkor());
}
function Keyboard_khmer_angkor()
{
var modCodes = com.keyman.text.Codes.modifierCodes;
var keyCodes = com.keyman.text.Codes.keyCodes;
var modCodes = keyman.osk.modifierCodes;
var keyCodes = keyman.osk.keyCodes;
this.KI="Keyboard_khmer_angkor";
this.KN="Khmer Angkor";
@ -2804,7 +2804,7 @@ function Keyboard_khmer_angkor()
k.KO(-1,t,"»");
}
if(m) {
k.KDC(-1,t);
r=this.g_normalise(t,e);
}

6
common/test/resources/keyboards/test_deadkeys.js
View file

@ -6,8 +6,8 @@ KeymanWeb.KR(new Keyboard_test_deadkeys());
}
function Keyboard_test_deadkeys()
{
var modCodes = com.keyman.text.Codes.modifierCodes;
var keyCodes = com.keyman.text.Codes.keyCodes;
var modCodes = keyman.osk.modifierCodes;
var keyCodes = keyman.osk.keyCodes;
this.KI="Keyboard_test_deadkeys";
this.KN="Keyman Deadkey Stress-Tester";
@ -325,7 +325,7 @@ function Keyboard_test_deadkeys()
k.KDO(-1,t,17);
}
if(m) {
k.KDC(-1,t);
r=this.g_dead_reorder(t,e);
}

160
common/test/resources/model-helpers.mjs Normal file
View file

@ -0,0 +1,160 @@
import path from 'path';
import { assert } from 'chai';
import fs from 'fs';
import vm from 'vm';
/**
* @file model-helpers.mjs
*
* Defines a common set of helper functions for use with predictive-text-related
* unit tests.
*/
// Ensure that we can successfully load the module & apply kmwLength, as it's
// needed for some of the unit tests.
// // Verify that the KMW string extensions are loaded via side-effect.
import { extendString } from '@keymanapp/web-utils';
import { createRequire } from "module";
import { fileURLToPath } from 'url';
// Ensure that our KMW string-extensions activate.
extendString();
assert.ok('💩'.kmwLength);
/**
* Creates a MessageEvent (for inter-worker communication), with the given data payload.
*
* @param {*} data
*/
export function createMessageEventWithData(data) {
return { data };
}
/**
* Creates a simple, default capabilities object for standard-case LMLayer init.
*/
export function capabilities() {
return {
maxLeftContextCodePoints: 64
}
}
/**
* Mimics a message from the outer LMLayer shell with a simple, default config object.
* Used for Worker tests.
*/
export function configWorker(worker) {
worker.onMessage(createMessageEventWithData({
message: 'config',
capabilities: capabilities()
}));
}
/**
* A valid model that suggests exactly what you want it to suggest.
*
* @returns {ModelDescription}
*/
export function dummyModel(futureSuggestions) {
return {
type: 'dummy',
futureSuggestions: futureSuggestions || []
};
}
/**
* Capabilities of a keyboard that will ONLY send left-sided capabilities.
* The keyboard does not support deleting to the right.
*
* @returns {Capabilities}
*/
export function defaultCapabilities() {
return {
maxLeftContextCodeUnits: 64
};
}
/**
* Returns the Context of an empty buffer; no text, at both the start and
* end of the buffer.
*
* @returns {Context}
*/
export function emptyContext() {
return {
left: '',
startOfBuffer: true,
endOfBuffer: true
};
}
/**
* Returns a Transform that, when applied, makes no changes to the buffer.
*
* @returns {Transform}
*/
export function zeroTransform() {
return {
insert: '',
deleteLeft: 0,
};
}
/**
* Returns a random token. NOT guaranteed to be unique.
*
* @returns {Token}
*/
export function randomToken() {
var range = Number.MAX_SAFE_INTEGER - Number.MIN_SAFE_INTEGER;
return Math.random() * range + Number.MIN_SAFE_INTEGER;
}
export function iGotDistractedByHazel() {
return jsonFixture('models/future_suggestions/i_got_distracted_by_hazel');
}
export function jsonFixture(name, root, import_root) {
// Assuming this file structure:
// root
// └── json
// ├── future_suggestions
// │ └── ...
// └── wordlists
// └── ...
// Default root: this folder.
if(!root) {
root = 'json';
}
if(!import_root) {
import_root = import.meta.url;
}
// The most straight-forward way... is to use CommonJS-style require to load JSON.
// Fortunately, Node provides the tools needed to recreate it.
const require = createRequire(import_root);
// ES-module mode also leaves out `__dirname`, so we rebuild that too.
const __filename = fileURLToPath(import_root);
const __dirname = path.dirname(__filename);
return require(path.join(__dirname, root, `${name}.json`));
}
export function importScriptsWith(context) {
return function() { // the constructed context's importScripts method.
/* Use of vm.createContext and script.runInContext allow us to avoid
* polluting the global scope with imports. When we throw away the
* context object, imported scripts will be automatically GC'd.
*/
for(var i=0; i < arguments.length; i++) {
context = vm.createContext(context);
var script = new vm.Script(fs.readFileSync(arguments[i]));
script.runInContext(context);
}
}
}

0
common/predictive-text/unit_tests/in_browser/resources/models/naive-trie.js → common/test/resources/models/naive-trie.js
View file

0
common/predictive-text/unit_tests/in_browser/resources/models/simple-dummy.js → common/test/resources/models/simple-dummy.js
View file

0
common/predictive-text/unit_tests/in_browser/resources/models/simple-trie.js → common/test/resources/models/simple-trie.js
View file

12
common/test/resources/package.json Normal file
View file

@ -0,0 +1,12 @@
{
"name": "@keymanapp/common-test-resources",
"description": "Common test resources for Keyman's JS & TS-based components & products",
"type": "module",
"license": "MIT",
"devDependencies": {
"@keymanapp/resources-gosh": "*",
"@types/node": "^10.17.21",
"chai": "^4.3.4",
"typescript": "^4.9.5"
}
}

2
common/test/resources/timeout-adapter.js
View file

@ -1,6 +1,6 @@
// Preprocessing of the Karma configuration's client.args parameter.
com = com || {};
var com = com || {};
com.keyman = com.keyman || {};
com.keyman.karma = com.keyman.karma || {};

22
common/tools/sourcemap-path-remapper/build.sh
View file

@ -1,7 +1,10 @@
#!/usr/bin/env bash
#
# Compile our sourcemap-path remapping module for use by Web builds, releases, etc.
# Compiles the JS sourcemap remapper tool used by some of our TS projects when complications
# arise with sourcemap handling.
#
# Exit on command failure and when using unset variables:
set -eu
## START STANDARD BUILD SCRIPT INCLUDE
@ -17,9 +20,6 @@ cd "$THIS_SCRIPT_PATH"
################################ Main script ################################
# TODO: for predictive-text, we only need :headless, perhaps we should be splitting modules?
# TODO: remove :tools once kmlmc is a dependency for test:module
builder_describe "Builds a sourcemap manipulation ES module for use in Web-related builds" \
"clean" \
"configure" \
@ -31,13 +31,6 @@ builder_describe_outputs \
builder_parse "$@"
### CONFIGURE ACTIONS
if builder_start_action configure; then
verify_npm_setup
builder_finish_action success configure
fi
### CLEAN ACTIONS
if builder_start_action clean; then
@ -45,6 +38,13 @@ if builder_start_action clean; then
builder_finish_action success clean
fi
### CONFIGURE ACTIONS
if builder_start_action configure; then
verify_npm_setup
builder_finish_action success configure
fi
### BUILD ACTIONS
if builder_start_action build; then

2
common/tools/sourcemap-path-remapper/package.json
View file

@ -10,7 +10,7 @@
"type": "git",
"url": "git+https://github.com/keymanapp/keyman.git"
},
"author": "Joshua A. Horton <jahorton@sil.org>",
"author": "Joshua A. Horton <joshua_horton@sil.org>",
"license": "MIT",
"bugs": {
"url": "https://github.com/keymanapp/keyman/issues"

22
common/tools/sourcemap-path-remapper/src/index.ts
View file

@ -1,5 +1,5 @@
import * as fs from 'fs';
import convertSourcemap from 'convert-source-map'; // Transforms sourcemaps among various common formats.
import convertSourceMap from 'convert-source-map'; // Transforms sourcemaps among various common formats.
// Base64, stringified-JSON, end-of-file comment...
// Because we're compiling the main project based on its TS build outputs, and our cross-module references
@ -16,33 +16,33 @@ export interface Mapping {
*/
class RemappingState {
constructor(srcmap: Object) {
this.sourcemap = srcmap;
this.sourceMap = srcmap;
this.resetChangedSourcepathTracking();
}
public readonly sourcemap: any;
private _unchangedSourcepaths: string[] = [];
public readonly sourceMap: any;
private _unchangedSourcePaths: string[] = [];
public get unchangedSourcepaths() {
return this._unchangedSourcepaths;
return this._unchangedSourcePaths;
}
public resetChangedSourcepathTracking() {
this._unchangedSourcepaths = [...this.sourcemap.sources];
this._unchangedSourcePaths = [...this.sourceMap.sources];
}
/**
* The sourcemap's `sourceRoot` property.
*/
public get sourceRoot() {
return this.sourcemap.sourceRoot;
return this.sourceMap.sourceRoot;
}
/**
* The sourcemap's `sourceRoot` property.
*/
public set sourceRoot(path: string) {
this.sourcemap.sourceRoot = path;
this.sourceMap.sourceRoot = path;
}
/**
@ -59,7 +59,7 @@ class RemappingState {
const unusedTransforms = [...transforms];
const sourcePaths = this.sourcemap.sources;
const sourcePaths = this.sourceMap.sources;
for(let i = 0; i < sourcePaths.length; i++) {
for(const transform of transforms) {
if(sourcePaths[i].match(transform.from)) {
@ -101,7 +101,7 @@ class RemappingState {
* @param file
*/
public toFile(file: fs.PathLike) {
fs.writeFileSync(file, convertSourcemap.fromObject(this.sourcemap).toJSON());
fs.writeFileSync(file, convertSourceMap.fromObject(this.sourceMap).toJSON());
}
}
@ -132,6 +132,6 @@ export default class SourcemapRemapper {
* @returns
*/
public static fromBuffer(buffer: Buffer) {
return this.fromObject(convertSourcemap.fromJSON(buffer).toObject());
return this.fromObject(convertSourceMap.fromJSON(buffer).toObject());
}
}

45
common/web/input-processor/build-bundler.js Normal file
View file

@ -0,0 +1,45 @@
/*
* Note: while this file is not meant to exist long-term, it provides a nice
* low-level proof-of-concept for esbuild bundling of the various Web submodules.
*
* Add some extra code at the end of src/index.ts and run it to verify successful bundling!
*/
import esbuild from 'esbuild';
import { spawn } from 'child_process';
// Bundled ES module version
esbuild.buildSync({
entryPoints: ['build/obj/index.js'],
bundle: true,
sourcemap: true,
external: ['fs', 'vm'],
format: "esm",
// Sets 'common/web' as a root folder for module resolution;
// this allows the keyman-version and utils imports to resolve.
//
// We also need to point it at the nested build output folder to resolve in-project
// imports when compiled - esbuild doesn't seem to pick up on the shifted base.
nodePaths: ['..', "build/obj"],
outfile: "build/lib/index.mjs",
tsconfig: 'tsconfig.json',
target: "es5"
});
// Bundled CommonJS (classic Node) module version
esbuild.buildSync({
entryPoints: ['build/obj/index.js'],
bundle: true,
sourcemap: true,
external: ['fs', 'vm'],
format: "cjs",
// Sets 'common/web' as a root folder for module resolution;
// this allows the keyman-version and utils imports to resolve.
//
// We also need to point it at the nested build output folder to resolve in-project
// imports when compiled - esbuild doesn't seem to pick up on the shifted base.
nodePaths: ['..', "build/obj"],
outfile: "build/lib/index.cjs",
tsconfig: 'tsconfig.json',
target: "es5"
});

36
common/web/input-processor/build.sh
View file

@ -16,13 +16,11 @@ cd "$(dirname "$THIS_SCRIPT")"
################################ Main script ################################
# TODO: for predictive-text, we only need :headless, perhaps we should be splitting modules?
# TODO: remove :tools once kmlmc is a dependency for test:module
builder_describe "Builds the standalone, headless form of Keyman Engine for Web's input-processor module" \
"@/common/web/keyman-version" \
"@/common/web/keyboard-processor" \
"@/common/predictive-text" \
"@/developer/src/kmc-model test" \
"clean" \
"configure" \
"build" \
@ -33,10 +31,7 @@ builder_describe "Builds the standalone, headless form of Keyman Engine for Web'
builder_describe_outputs \
configure /node_modules \
configure:module /node_modules \
configure:tools /node_modules \
build:module /common/web/input-processor/build/index.js \
build:tools /developer/src/kmc/build/src/kmlmc.js # TODO: remove this once kmlmc is a dependency
build /common/web/input-processor/build/lib/index.mjs \
builder_parse "$@"
@ -56,36 +51,25 @@ fi
### BUILD ACTIONS
if builder_start_action build:tools; then
# Used by test:module
# TODO: convert to a dependency once we have updated kmlmc to use builder script
pushd "$KEYMAN_ROOT/developer/src/kmc-model"
./build.sh
popd
pushd "$KEYMAN_ROOT/developer/src/kmc"
./build.sh
popd
if builder_start_action build; then
tsc -b ./tsconfig.json
node build-bundler.js
builder_finish_action success build:tools
fi
# Declaration bundling.
tsc --emitDeclarationOnly --outFile ./build/lib/index.d.ts
if builder_start_action build:module; then
tsc -b src/tsconfig.json
builder_finish_action success build:module
builder_finish_action success build
fi
# TEST ACTIONS
if builder_start_action test:module; then
if builder_start_action test; then
FLAGS=
if builder_has_option --ci; then
FLAGS="--reporter mocha-teamcity-reporter"
fi
# Build the leaf-style, bundled version of input-processor for use in testing.
tsc -b src/tsconfig.bundled.json
mocha --recursive $FLAGS ./tests/cases/
builder_finish_action success test:module
builder_finish_action success test
fi

15
common/web/input-processor/package.json
View file

@ -38,5 +38,20 @@
"@keymanapp/keyman-version": "*",
"@keymanapp/web-utils": "*",
"eventemitter3": "^4.0.0"
},
"type": "module",
"imports": {
"#./*.js": "./build/obj/*.js"
},
"main": "./build/obj/index.js",
"types": "./build/obj/index.d.ts",
"exports": {
".": "./build/obj/index.js",
"./lib": {
"types": "./build/lib/index.d.ts",
"import": "./build/lib/index.mjs",
"require": "./build/lib/index.cjs"
},
"./obj/*.js": "./build/obj/*.js"
}
}

20
common/web/input-processor/src/includes/events.ts
View file

@ -1,20 +0,0 @@
// Implements Node's EventEmitter class and related module components in a near
// browser-compatible way. (Just requires a blank 'module' object on the window.)
///<reference path="../../../../../node_modules/eventemitter3/index.js" />
// Unfortunately, I can't get it to recognize type information properly
// because we can't use require statements. So, a small-scale manual definition.
declare class EventEmitter {
/** Add a listener for a given event */
on(event: string, func: (...args: any[]) => boolean, context?: any);
/** Add a one-time listener for a given event */
once(event: string, func: (...args: any[]) => boolean, context?: any);
removeListener(event: string, func: (...args: any[]) => boolean, context?: any, once?: boolean);
// Defines their alternately-themed aliases.
addListener: typeof EventEmitter.prototype.on;
off: typeof EventEmitter.prototype.removeListener;
// Defines the actual event-raising function.
emit(eventName: string, ...args: any[]);
}

7
common/web/input-processor/src/index.ts Normal file
View file

@ -0,0 +1,7 @@
/// <reference path="../../../predictive-text/src/worker-interface.d.ts" />
export { default as InputProcessor } from './text/inputProcessor.js';
export { default as ContextWindow } from './text/contextWindow.js';
export { default as ModelSpec } from './text/prediction/modelSpec.js';
export { default as LanguageProcessor, StateChangeEnum } from './text/prediction/languageProcessor.js';
export { default as PredictionContext } from './text/prediction/predictionContext.js';

68
common/web/input-processor/src/text/contextWindow.ts
View file

@ -1,44 +1,44 @@
namespace com.keyman.text {
export class ContextWindow implements Context {
// Used to limit the range of context replicated for use of keyboard rules within
// the engine, as used for fat-finger prep / `Alternate` generation.
public static readonly ENGINE_RULE_WINDOW: Configuration = {
leftContextCodePoints: 64,
rightContextCodePoints: 32
};
import { Mock } from "@keymanapp/keyboard-processor";
left: string;
right?: string;
export default class ContextWindow implements Context {
// Used to limit the range of context replicated for use of keyboard rules within
// the engine, as used for fat-finger prep / `Alternate` generation.
public static readonly ENGINE_RULE_WINDOW: Configuration = {
leftContextCodePoints: 64,
rightContextCodePoints: 32
};
startOfBuffer: boolean;
endOfBuffer: boolean;
left: string;
right?: string;
casingForm?: CasingForm;
startOfBuffer: boolean;
endOfBuffer: boolean;
constructor(mock: Mock, config: Configuration, layerId: string) {
this.left = mock.getTextBeforeCaret();
this.startOfBuffer = this.left._kmwLength() <= config.leftContextCodePoints;
if(!this.startOfBuffer) {
// Our custom substring version will return the last n characters if param #1 is given -n.
this.left = this.left._kmwSubstr(-config.leftContextCodePoints);
}
casingForm?: CasingForm;
this.right = mock.getTextAfterCaret();
this.endOfBuffer = this.right._kmwLength() <= config.rightContextCodePoints;
if(!this.endOfBuffer) {
this.right = this.right._kmwSubstr(0, config.rightContextCodePoints);
}
this.casingForm =
layerId == 'shift' ? 'initial' :
layerId == 'caps' ? 'upper' :
null;
constructor(mock: Mock, config: Configuration, layerId: string) {
this.left = mock.getTextBeforeCaret();
this.startOfBuffer = this.left._kmwLength() <= config.leftContextCodePoints;
if(!this.startOfBuffer) {
// Our custom substring version will return the last n characters if param #1 is given -n.
this.left = this.left._kmwSubstr(-config.leftContextCodePoints);
}
public toMock(): Mock {
let caretPos = this.left._kmwLength();
return new Mock(this.left + (this.right || ""), caretPos);
this.right = mock.getTextAfterCaret();
this.endOfBuffer = this.right._kmwLength() <= config.rightContextCodePoints;
if(!this.endOfBuffer) {
this.right = this.right._kmwSubstr(0, config.rightContextCodePoints);
}
this.casingForm =
layerId == 'shift' ? 'initial' :
layerId == 'caps' ? 'upper' :
null;
}
public toMock(): Mock {
let caretPos = this.left._kmwLength();
return new Mock(this.left + (this.right || ""), caretPos);
}
}

663
common/web/input-processor/src/text/inputProcessor.ts
View file

@ -1,373 +1,372 @@
// Defines a 'polyfill' of sorts for NPM's events module
/// <reference path="../includes/events.ts" />
/// <reference path="contextWindow.ts" />
/// <reference path="prediction/languageProcessor.ts" />
/// <reference types="@keymanapp/models-types" />
namespace com.keyman.text {
export class InputProcessor {
public static readonly DEFAULT_OPTIONS: ProcessorInitOptions = {
baseLayout: 'us'
import ContextWindow from "./contextWindow.js";
import LanguageProcessor from "./prediction/languageProcessor.js";
import type ModelSpec from "./prediction/modelSpec.js";
import { globalObject, DeviceSpec } from "@keymanapp/web-utils";
import {
type Alternate,
Codes,
type Keyboard,
KeyboardInterface,
KeyboardProcessor,
type KeyEvent,
Mock,
type OutputTarget,
type ProcessorInitOptions,
RuleBehavior,
SystemStoreIDs,
type TextTransform
} from "@keymanapp/keyboard-processor";
export default class InputProcessor {
public static readonly DEFAULT_OPTIONS: ProcessorInitOptions = {
baseLayout: 'us'
}
/**
* Indicates the device (platform) to be used for non-keystroke events,
* such as those sent to `begin postkeystroke` and `begin newcontext`
* entry points.
*/
private contextDevice: DeviceSpec;
private kbdProcessor: KeyboardProcessor;
private lngProcessor: LanguageProcessor;
constructor(device: DeviceSpec, predictiveTextWorker: Worker, options?: ProcessorInitOptions) {
if(!device) {
throw new Error('device must be defined');
}
/**
* Indicates the device (platform) to be used for non-keystroke events,
* such as those sent to `begin postkeystroke` and `begin newcontext`
* entry points.
*/
private contextDevice: utils.DeviceSpec;
private kbdProcessor: KeyboardProcessor;
private lngProcessor: prediction.LanguageProcessor;
if(!options) {
options = InputProcessor.DEFAULT_OPTIONS;
}
constructor(device: utils.DeviceSpec, options?: ProcessorInitOptions) {
if(!device) {
throw new Error('device must be defined');
this.contextDevice = device;
this.kbdProcessor = new KeyboardProcessor(device, options);
this.lngProcessor = new LanguageProcessor(predictiveTextWorker);
}
public get languageProcessor(): LanguageProcessor {
return this.lngProcessor;
}
public get keyboardProcessor(): KeyboardProcessor {
return this.kbdProcessor;
}
public get keyboardInterface(): KeyboardInterface {
return this.keyboardProcessor.keyboardInterface;
}
public get activeKeyboard(): Keyboard {
return this.keyboardInterface.activeKeyboard;
}
public set activeKeyboard(keyboard: Keyboard) {
this.keyboardInterface.activeKeyboard = keyboard;
// All old deadkeys and keyboard-specific cache should immediately be invalidated
// on a keyboard change.
this.resetContext();
}
public get activeModel(): ModelSpec {
return this.languageProcessor.activeModel;
}
/**
* Tell the currently active keyboard that a new context has been selected,
* e.g. by focus change, selection change, keyboard change, etc.
*
* @param {Object} outputTarget The OutputTarget that has focus
* @returns {Object} A RuleBehavior object describing the cumulative effects of
* all matched keyboard rules
*/
processNewContextEvent(outputTarget: OutputTarget): RuleBehavior {
const ruleBehavior = this.keyboardProcessor.processNewContextEvent(this.contextDevice, outputTarget);
if(ruleBehavior) {
ruleBehavior.finalize(this.keyboardProcessor, outputTarget, true);
}
return ruleBehavior;
}
/**
* Simulate a keystroke according to the touched keyboard button element
*
* Handles default output and keyboard processing for both OSK and physical keystrokes.
*
* @param {Object} keyEvent The abstracted KeyEvent to use for keystroke processing
* @param {Object} outputTarget The OutputTarget receiving the KeyEvent
* @returns {Object} A RuleBehavior object describing the cumulative effects of
* all matched keyboard rules.
*/
processKeyEvent(keyEvent: KeyEvent, outputTarget: OutputTarget): RuleBehavior {
const kbdMismatch = keyEvent.srcKeyboard && this.activeKeyboard != keyEvent.srcKeyboard;
const trueActiveKeyboard = this.activeKeyboard;
try {
if(kbdMismatch) {
// Avoid force-reset of context per our setter above.
this.keyboardInterface.activeKeyboard = keyEvent.srcKeyboard;
}
if(!options) {
options = InputProcessor.DEFAULT_OPTIONS;
return this._processKeyEvent(keyEvent, outputTarget);
} finally {
if(kbdMismatch) {
// Restore our "current" activeKeyboard to its setting before the mismatching KeyEvent.
this.keyboardInterface.activeKeyboard = trueActiveKeyboard;
}
this.contextDevice = device;
this.kbdProcessor = new KeyboardProcessor(device, options);
this.lngProcessor = new prediction.LanguageProcessor();
}
}
public get languageProcessor(): prediction.LanguageProcessor {
return this.lngProcessor;
}
/**
* Acts as the core of `processKeyEvent` once we're comfortable asserting that the incoming
* keystroke matches the current `activeKeyboard`.
* @param keyEvent
* @param outputTarget
* @returns
*/
private _processKeyEvent(keyEvent: KeyEvent, outputTarget: OutputTarget): RuleBehavior {
let formFactor = keyEvent.device.formFactor;
let fromOSK = keyEvent.isSynthetic;
public get keyboardProcessor(): KeyboardProcessor {
return this.kbdProcessor;
}
public get keyboardInterface(): text.KeyboardInterface {
return this.keyboardProcessor.keyboardInterface;
}
public get activeKeyboard(): keyboards.Keyboard {
return this.keyboardInterface.activeKeyboard;
}
public set activeKeyboard(keyboard: keyboards.Keyboard) {
this.keyboardInterface.activeKeyboard = keyboard;
// All old deadkeys and keyboard-specific cache should immediately be invalidated
// on a keyboard change.
this.resetContext();
}
public get activeModel(): prediction.ModelSpec {
return this.languageProcessor.activeModel;
}
/**
* Tell the currently active keyboard that a new context has been selected,
* e.g. by focus change, selection change, keyboard change, etc.
*
* @param {Object} outputTarget The OutputTarget that has focus
* @returns {Object} A RuleBehavior object describing the cumulative effects of
* all matched keyboard rules
*/
processNewContextEvent(outputTarget: OutputTarget): RuleBehavior {
const ruleBehavior = this.keyboardProcessor.processNewContextEvent(this.contextDevice, outputTarget);
if(ruleBehavior) {
ruleBehavior.finalize(this.keyboardProcessor, outputTarget, true);
}
return ruleBehavior;
}
/**
* Simulate a keystroke according to the touched keyboard button element
*
* Handles default output and keyboard processing for both OSK and physical keystrokes.
*
* @param {Object} keyEvent The abstracted KeyEvent to use for keystroke processing
* @param {Object} outputTarget The OutputTarget receiving the KeyEvent
* @returns {Object} A RuleBehavior object describing the cumulative effects of
* all matched keyboard rules.
*/
processKeyEvent(keyEvent: KeyEvent, outputTarget: OutputTarget): RuleBehavior {
const kbdMismatch = keyEvent.srcKeyboard && this.activeKeyboard != keyEvent.srcKeyboard;
const trueActiveKeyboard = this.activeKeyboard;
try {
if(kbdMismatch) {
// Avoid force-reset of context per our setter above.
this.keyboardInterface.activeKeyboard = keyEvent.srcKeyboard;
}
return this._processKeyEvent(keyEvent, outputTarget);
} finally {
if(kbdMismatch) {
// Restore our "current" activeKeyboard to its setting before the mismatching KeyEvent.
this.keyboardInterface.activeKeyboard = trueActiveKeyboard;
}
// The default OSK layout for desktop devices does not include nextlayer info, relying on modifier detection here.
// It's the OSK equivalent to doModifierPress on 'desktop' form factors.
if((formFactor == DeviceSpec.FormFactor.Desktop || !this.activeKeyboard || this.activeKeyboard.usesDesktopLayoutOnDevice(keyEvent.device)) && fromOSK) {
// If it's a desktop OSK style and this triggers a layer change,
// a modifier key was clicked. No output expected, so it's safe to instantly exit.
if(this.keyboardProcessor.selectLayer(keyEvent)) {
return new RuleBehavior();
}
}
/**
* Acts as the core of `processKeyEvent` once we're comfortable asserting that the incoming
* keystroke matches the current `activeKeyboard`.
* @param keyEvent
* @param outputTarget
* @returns
*/
private _processKeyEvent(keyEvent: KeyEvent, outputTarget: OutputTarget): RuleBehavior {
let formFactor = keyEvent.device.formFactor;
let fromOSK = keyEvent.isSynthetic;
// The default OSK layout for desktop devices does not include nextlayer info, relying on modifier detection here.
// It's the OSK equivalent to doModifierPress on 'desktop' form factors.
if((formFactor == utils.FormFactor.Desktop || !this.activeKeyboard || this.activeKeyboard.usesDesktopLayoutOnDevice(keyEvent.device)) && fromOSK) {
// If it's a desktop OSK style and this triggers a layer change,
// a modifier key was clicked. No output expected, so it's safe to instantly exit.
if(this.keyboardProcessor.selectLayer(keyEvent)) {
return new RuleBehavior();
}
// Will handle keystroke-based non-layer change modifier & state keys, mapping them through the physical keyboard's version
// of state management. `doModifierPress` must always run.
if(this.keyboardProcessor.doModifierPress(keyEvent, outputTarget, !fromOSK)) {
// If run on a desktop platform, we know that modifier & state key presses may not
// produce output, so we may make an immediate return safely.
if(!fromOSK) {
return new RuleBehavior();
}
// Will handle keystroke-based non-layer change modifier & state keys, mapping them through the physical keyboard's version
// of state management. `doModifierPress` must always run.
if(this.keyboardProcessor.doModifierPress(keyEvent, outputTarget, !fromOSK)) {
// If run on a desktop platform, we know that modifier & state key presses may not
// produce output, so we may make an immediate return safely.
if(!fromOSK) {
return new RuleBehavior();
}
}
// If suggestions exist AND space is pressed, accept the suggestion and do not process the keystroke.
// If a suggestion was just accepted AND backspace is pressed, revert the change and do not process the backspace.
// We check the first condition here, while the prediction UI handles the second through the try__() methods below.
if(this.languageProcessor.isActive) {
// The following code relies on JS's logical operator "short-circuit" properties to prevent unwanted triggering of the second condition.
// Can the suggestion UI revert a recent suggestion? If so, do that and swallow the backspace.
if((keyEvent.kName == "K_BKSP" || keyEvent.Lcode == Codes.keyCodes["K_BKSP"]) && this.languageProcessor.tryRevertSuggestion()) {
return new RuleBehavior();
// Can the suggestion UI accept an existing suggestion? If so, do that and swallow the space character.
} else if((keyEvent.kName == "K_SPACE" || keyEvent.Lcode == Codes.keyCodes["K_SPACE"]) && this.languageProcessor.tryAcceptSuggestion('space')) {
return new RuleBehavior();
}
}
// // ...end I3363 (Build 301)
// Create a "mock" backup of the current outputTarget in its pre-input state.
// Current, long-existing assumption - it's DOM-backed.
let preInputMock = Mock.from(outputTarget, true);
const startingLayerId = this.keyboardProcessor.layerId;
// We presently need the true keystroke to run on the FULL context. That index is still
// needed for some indexing operations when comparing two different output targets.
let ruleBehavior = this.keyboardProcessor.processKeystroke(keyEvent, outputTarget);
// Swap layer as appropriate.
if(keyEvent.kNextLayer) {
this.keyboardProcessor.selectLayer(keyEvent);
}
// If it's a key that we 'optimize out' of our fat-finger correction algorithm,
// we MUST NOT trigger it for this keystroke.
let isOnlyLayerSwitchKey = text.Codes.isKnownOSKModifierKey(keyEvent.kName);
// Best-guess stopgap for possible custom modifier keys.
// If a key (1) does not affect the context and (2) shifts the active layer,
// we assume it's a modifier key. (Touch keyboards may define custom modifier keys.)
//
// Note: this will mean we won't generate alternates in the niche scenario where:
// 1. Keypress does not alter the actual context
// 2. It DOES emit a deadkey with an earlier processing rule.
// 3. The FINAL processing rule does not match.
// 4. The key ALSO signals a layer shift.
// If any of the four above conditions aren't met - no problem!
// So it's a pretty niche scenario.
if((ruleBehavior?.transcription?.transform as TextTransform)?.isNoOp() && keyEvent.kNextLayer) {
isOnlyLayerSwitchKey = true;
}
const keepRuleBehavior = ruleBehavior != null;
// Should we swallow any further processing of keystroke events for this keydown-keypress sequence?
if(keepRuleBehavior) {
// alternates are our fat-finger alternate outputs. We don't build these for keys we detect as
// layer switch keys
let alternates = isOnlyLayerSwitchKey ? null : this.buildAlternates(ruleBehavior, keyEvent, preInputMock);
// Now that we've done all the keystroke processing needed, ensure any extra effects triggered
// by the actual keystroke occur.
ruleBehavior.finalize(this.keyboardProcessor, outputTarget, false);
// -- All keystroke (and 'alternate') processing is now complete. Time to finalize everything! --
// Notify the ModelManager of new input - it's predictive text time!
if(alternates && alternates.length > 0) {
ruleBehavior.transcription.alternates = alternates;
}
} else {
// We need a dummy RuleBehavior for keys which have no output (e.g. Shift)
ruleBehavior = new RuleBehavior();
ruleBehavior.transcription = outputTarget.buildTranscriptionFrom(outputTarget, null, false);
ruleBehavior.triggersDefaultCommand = true;
}
// The keyboard may want to take an action after all other keystroke processing is
// finished, for example to switch layers. This action may not have any output
// but may change system store or variable store values. Given this, we don't need to
// save anything about the post behavior, after finalizing it
// We need to tell the keyboard if the layer has been changed, either by a keyboard rule itself,
// or by the touch layout 'nextlayer' control.
const hasLayerChanged = ruleBehavior.setStore[KeyboardInterface.TSS_LAYER] || keyEvent.kNextLayer;
this.keyboardProcessor.newLayerStore.set(hasLayerChanged ? this.keyboardProcessor.layerId : '');
this.keyboardProcessor.oldLayerStore.set(hasLayerChanged ? startingLayerId : '');
let postRuleBehavior = this.keyboardProcessor.processPostKeystroke(this.contextDevice, outputTarget);
if(postRuleBehavior) {
postRuleBehavior.finalize(this.keyboardProcessor, outputTarget, true);
}
// Yes, even for ruleBehavior.triggersDefaultCommand. Those tend to change the context.
ruleBehavior.predictionPromise = this.languageProcessor.predict(ruleBehavior.transcription, this.keyboardProcessor.layerId);
// Text did not change (thus, no text "input") if we tabbed or merely moved the caret.
if(!ruleBehavior.triggersDefaultCommand) {
// For DOM-aware targets, this will trigger a DOM event page designers may listen for.
outputTarget.doInputEvent();
}
return keepRuleBehavior ? ruleBehavior : null;
}
private buildAlternates(ruleBehavior: RuleBehavior, keyEvent: KeyEvent, preInputMock: Mock): Alternate[] {
let alternates: Alternate[];
// If suggestions exist AND space is pressed, accept the suggestion and do not process the keystroke.
// If a suggestion was just accepted AND backspace is pressed, revert the change and do not process the backspace.
// We check the first condition here, while the prediction UI handles the second through the try__() methods below.
if(this.languageProcessor.isActive) {
// The following code relies on JS's logical operator "short-circuit" properties to prevent unwanted triggering of the second condition.
// If we're performing a 'default command', it's not a standard 'typing' event - don't do fat-finger stuff.
// Also, don't do fat-finger stuff if predictive text isn't enabled.
if(this.languageProcessor.isActive && !ruleBehavior.triggersDefaultCommand) {
let keyDistribution = keyEvent.keyDistribution;
// Can the suggestion UI revert a recent suggestion? If so, do that and swallow the backspace.
if((keyEvent.kName == "K_BKSP" || keyEvent.Lcode == Codes.keyCodes["K_BKSP"]) && this.languageProcessor.tryRevertSuggestion()) {
return new RuleBehavior();
// Can the suggestion UI accept an existing suggestion? If so, do that and swallow the space character.
} else if((keyEvent.kName == "K_SPACE" || keyEvent.Lcode == Codes.keyCodes["K_SPACE"]) && this.languageProcessor.tryAcceptSuggestion('space')) {
return new RuleBehavior();
}
}
// We don't need to track absolute indexing during alternate-generation;
// only position-relative, so it's better to use a sliding window for context
// when making alternates. (Slightly worse for short text, matters greatly
// for long text.)
let contextWindow = new ContextWindow(preInputMock, ContextWindow.ENGINE_RULE_WINDOW, this.keyboardProcessor.layerId);
let windowedMock = contextWindow.toMock();
// // ...end I3363 (Build 301)
// Note - we don't yet do fat-fingering with longpress keys.
if(keyDistribution && keyEvent.kbdLayer) {
// Tracks a 'deadline' for fat-finger ops, just in case both context is long enough
// and device is slow enough that the calculation takes too long.
//
// Consider use of https://developer.mozilla.org/en-US/docs/Web/API/Performance/now instead?
// Would allow finer-tuned control.
let TIMEOUT_THRESHOLD: number = Number.MAX_VALUE;
let _globalThis = com.keyman.utils.getGlobalObject();
let timer: () => number;
// Create a "mock" backup of the current outputTarget in its pre-input state.
// Current, long-existing assumption - it's DOM-backed.
let preInputMock = Mock.from(outputTarget, true);
// Available by default on `window` in browsers, but _not_ on `global` in Node,
// surprisingly. Since we can't use code dependent on `require` statements
// at present, we have to condition upon it actually existing.
if(_globalThis['performance'] && _globalThis['performance']['now']) {
timer = function() {
return _globalThis['performance']['now']();
};
const startingLayerId = this.keyboardProcessor.layerId;
TIMEOUT_THRESHOLD = timer() + 16; // + 16ms.
} // else {
// We _could_ just use Date.now() as a backup... but that (probably) only matters
// when unit testing. So... we actually don't _need_ time thresholding when in
// a Node environment.
// }
// We presently need the true keystroke to run on the FULL context. That index is still
// needed for some indexing operations when comparing two different output targets.
let ruleBehavior = this.keyboardProcessor.processKeystroke(keyEvent, outputTarget);
// Tracks a minimum probability for keystroke probability. Anything less will not be
// included in alternate calculations.
//
// Seek to match SearchSpace.EDIT_DISTANCE_COST_SCALE from the predictive-text engine.
// Reasoning for the selected value may be seen there. Short version - keystrokes
// that _appear_ very precise may otherwise not even consider directly-neighboring keys.
let KEYSTROKE_EPSILON = Math.exp(-5);
// Swap layer as appropriate.
if(keyEvent.kNextLayer) {
this.keyboardProcessor.selectLayer(keyEvent);
}
// Sort the distribution into probability-descending order.
keyDistribution.sort((a, b) => b.p - a.p);
// If it's a key that we 'optimize out' of our fat-finger correction algorithm,
// we MUST NOT trigger it for this keystroke.
let isOnlyLayerSwitchKey = Codes.isKnownOSKModifierKey(keyEvent.kName);
let activeLayout = this.activeKeyboard.layout(keyEvent.device.formFactor);
alternates = [];
// Best-guess stopgap for possible custom modifier keys.
// If a key (1) does not affect the context and (2) shifts the active layer,
// we assume it's a modifier key. (Touch keyboards may define custom modifier keys.)
//
// Note: this will mean we won't generate alternates in the niche scenario where:
// 1. Keypress does not alter the actual context
// 2. It DOES emit a deadkey with an earlier processing rule.
// 3. The FINAL processing rule does not match.
// 4. The key ALSO signals a layer shift.
// If any of the four above conditions aren't met - no problem!
// So it's a pretty niche scenario.
if((ruleBehavior?.transcription?.transform as TextTransform)?.isNoOp() && keyEvent.kNextLayer) {
isOnlyLayerSwitchKey = true;
}
let totalMass = 0; // Tracks sum of non-error probabilities.
for(let pair of keyDistribution) {
if(pair.p < KEYSTROKE_EPSILON) {
totalMass += pair.p;
break;
} else if(timer && timer() >= TIMEOUT_THRESHOLD) {
// Note: it's always possible that the thread _executing_ our JS
// got paused by the OS, even if JS itself is single-threaded.
//
// The case where `alternates` is initialized (line 167) but empty
// (because of net-zero loop iterations) MUST be handled.
break;
}
const keepRuleBehavior = ruleBehavior != null;
// Should we swallow any further processing of keystroke events for this keydown-keypress sequence?
if(keepRuleBehavior) {
// alternates are our fat-finger alternate outputs. We don't build these for keys we detect as
// layer switch keys
let alternates = isOnlyLayerSwitchKey ? null : this.buildAlternates(ruleBehavior, keyEvent, preInputMock);
let mock = Mock.from(windowedMock, false);
// Now that we've done all the keystroke processing needed, ensure any extra effects triggered
// by the actual keystroke occur.
ruleBehavior.finalize(this.keyboardProcessor, outputTarget, false);
let altKey = activeLayout.getLayer(keyEvent.kbdLayer).getKey(pair.keyId);
if(!altKey) {
console.warn("Potential fat-finger key could not be found in layer!");
continue;
}
// -- All keystroke (and 'alternate') processing is now complete. Time to finalize everything! --
let altEvent = altKey.constructKeyEvent(this.keyboardProcessor, keyEvent.device);
let alternateBehavior = this.keyboardProcessor.processKeystroke(altEvent, mock);
// Notify the ModelManager of new input - it's predictive text time!
if(alternates && alternates.length > 0) {
ruleBehavior.transcription.alternates = alternates;
}
} else {
// We need a dummy RuleBehavior for keys which have no output (e.g. Shift)
ruleBehavior = new RuleBehavior();
ruleBehavior.transcription = outputTarget.buildTranscriptionFrom(outputTarget, null, false);
ruleBehavior.triggersDefaultCommand = true;
}
// If alternateBehavior.beep == true, ignore it. It's a disallowed key sequence,
// so we expect users to never intend their use.
// The keyboard may want to take an action after all other keystroke processing is
// finished, for example to switch layers. This action may not have any output
// but may change system store or variable store values. Given this, we don't need to
// save anything about the post behavior, after finalizing it
// We need to tell the keyboard if the layer has been changed, either by a keyboard rule itself,
// or by the touch layout 'nextlayer' control.
const hasLayerChanged = ruleBehavior.setStore[SystemStoreIDs.TSS_LAYER] || keyEvent.kNextLayer;
this.keyboardProcessor.newLayerStore.set(hasLayerChanged ? this.keyboardProcessor.layerId : '');
this.keyboardProcessor.oldLayerStore.set(hasLayerChanged ? startingLayerId : '');
let postRuleBehavior = this.keyboardProcessor.processPostKeystroke(this.contextDevice, outputTarget);
if(postRuleBehavior) {
postRuleBehavior.finalize(this.keyboardProcessor, outputTarget, true);
}
// Yes, even for ruleBehavior.triggersDefaultCommand. Those tend to change the context.
ruleBehavior.predictionPromise = this.languageProcessor.predict(ruleBehavior.transcription, this.keyboardProcessor.layerId);
// Text did not change (thus, no text "input") if we tabbed or merely moved the caret.
if(!ruleBehavior.triggersDefaultCommand) {
// For DOM-aware targets, this will trigger a DOM event page designers may listen for.
outputTarget.doInputEvent();
}
return keepRuleBehavior ? ruleBehavior : null;
}
private buildAlternates(ruleBehavior: RuleBehavior, keyEvent: KeyEvent, preInputMock: Mock): Alternate[] {
let alternates: Alternate[];
// If we're performing a 'default command', it's not a standard 'typing' event - don't do fat-finger stuff.
// Also, don't do fat-finger stuff if predictive text isn't enabled.
if(this.languageProcessor.isActive && !ruleBehavior.triggersDefaultCommand) {
let keyDistribution = keyEvent.keyDistribution;
// We don't need to track absolute indexing during alternate-generation;
// only position-relative, so it's better to use a sliding window for context
// when making alternates. (Slightly worse for short text, matters greatly
// for long text.)
let contextWindow = new ContextWindow(preInputMock, ContextWindow.ENGINE_RULE_WINDOW, this.keyboardProcessor.layerId);
let windowedMock = contextWindow.toMock();
// Note - we don't yet do fat-fingering with longpress keys.
if(this.languageProcessor.isActive && keyDistribution && keyEvent.kbdLayer) {
// Tracks a 'deadline' for fat-finger ops, just in case both context is long enough
// and device is slow enough that the calculation takes too long.
//
// Consider use of https://developer.mozilla.org/en-US/docs/Web/API/Performance/now instead?
// Would allow finer-tuned control.
let TIMEOUT_THRESHOLD: number = Number.MAX_VALUE;
let _globalThis = globalObject();
let timer: () => number;
// Available by default on `window` in browsers, but _not_ on `global` in Node,
// surprisingly. Since we can't use code dependent on `require` statements
// at present, we have to condition upon it actually existing.
if(_globalThis['performance'] && _globalThis['performance']['now']) {
timer = function() {
return _globalThis['performance']['now']();
};
TIMEOUT_THRESHOLD = timer() + 16; // + 16ms.
} // else {
// We _could_ just use Date.now() as a backup... but that (probably) only matters
// when unit testing. So... we actually don't _need_ time thresholding when in
// a Node environment.
// }
// Tracks a minimum probability for keystroke probability. Anything less will not be
// included in alternate calculations.
//
// Seek to match SearchSpace.EDIT_DISTANCE_COST_SCALE from the predictive-text engine.
// Reasoning for the selected value may be seen there. Short version - keystrokes
// that _appear_ very precise may otherwise not even consider directly-neighboring keys.
let KEYSTROKE_EPSILON = Math.exp(-5);
// Sort the distribution into probability-descending order.
keyDistribution.sort((a, b) => b.p - a.p);
let activeLayout = this.activeKeyboard.layout(keyEvent.device.formFactor);
alternates = [];
let totalMass = 0; // Tracks sum of non-error probabilities.
for(let pair of keyDistribution) {
if(pair.p < KEYSTROKE_EPSILON) {
totalMass += pair.p;
break;
} else if(timer && timer() >= TIMEOUT_THRESHOLD) {
// Note: it's always possible that the thread _executing_ our JS
// got paused by the OS, even if JS itself is single-threaded.
//
// Also possible that this set of conditions fail for all evaluated alternates.
if(alternateBehavior && !alternateBehavior.beep && pair.p > 0) {
let transform: Transform = alternateBehavior.transcription.transform;
// Ensure that the alternate's token id matches that of the current keystroke, as we only
// record the matched rule's context (since they match)
transform.id = ruleBehavior.transcription.token;
alternates.push({sample: transform, 'p': pair.p});
totalMass += pair.p;
}
// The case where `alternates` is initialized (line 167) but empty
// (because of net-zero loop iterations) MUST be handled.
break;
}
// Renormalizes the distribution, as any error (beep) results
// will result in a distribution that doesn't sum to 1 otherwise.
// All `.p` values are strictly positive, so totalMass is
// guaranteed to be > 0 if the array has entries.
alternates.forEach(function(alt) {
alt.p /= totalMass;
});
let mock = Mock.from(windowedMock, false);
let altKey = activeLayout.getLayer(keyEvent.kbdLayer).getKey(pair.keyId);
if(!altKey) {
console.warn("Potential fat-finger key could not be found in layer!");
continue;
}
let altEvent = this.keyboardProcessor.activeKeyboard.constructKeyEvent(altKey, keyEvent.device, this.keyboardProcessor.stateKeys);
let alternateBehavior = this.keyboardProcessor.processKeystroke(altEvent, mock);
// If alternateBehavior.beep == true, ignore it. It's a disallowed key sequence,
// so we expect users to never intend their use.
//
// Also possible that this set of conditions fail for all evaluated alternates.
if(alternateBehavior && !alternateBehavior.beep && pair.p > 0) {
let transform: Transform = alternateBehavior.transcription.transform;
// Ensure that the alternate's token id matches that of the current keystroke, as we only
// record the matched rule's context (since they match)
transform.id = ruleBehavior.transcription.token;
alternates.push({sample: transform, 'p': pair.p});
totalMass += pair.p;
}
}
}
return alternates;
}
public resetContext(outputTarget?: OutputTarget) {
this.keyboardProcessor.resetContext();
this.languageProcessor.invalidateContext(outputTarget, this.keyboardProcessor.layerId);
// Let the keyboard do its initial group processing
//console.log('processNewContextEvent called from resetContext');
if(outputTarget) {
this.processNewContextEvent(outputTarget);
// Renormalizes the distribution, as any error (beep) results
// will result in a distribution that doesn't sum to 1 otherwise.
// All `.p` values are strictly positive, so totalMass is
// guaranteed to be > 0 if the array has entries.
alternates.forEach(function(alt) {
alt.p /= totalMass;
});
}
}
return alternates;
}
}
(function () {
let ns = com.keyman.text;
// Let the InputProcessor be available both in the browser and in Node.
if (typeof module !== 'undefined' && typeof module.exports !== 'undefined') {
module.exports = ns.InputProcessor;
//@ts-ignore
ns.InputProcessor.com = com; // Export the root namespace so that all InputProcessor classes are accessible by unit tests.
public resetContext(outputTarget?: OutputTarget) {
this.keyboardProcessor.resetContext(outputTarget);
this.languageProcessor.invalidateContext(outputTarget, this.keyboardProcessor.layerId);
}
}());
}

816
common/web/input-processor/src/text/prediction/languageProcessor.ts
View file

@ -1,288 +1,226 @@
///<reference path="../../includes/events.ts" />
/// <reference path="../contextWindow.ts" />
import EventEmitter from "eventemitter3";
import { LMLayer } from "@keymanapp/lexical-model-layer/web";
import { OutputTarget, Transcription, Mock } from "@keymanapp/keyboard-processor";
import ContextWindow from "../contextWindow.js";
import ModelSpec from "./modelSpec.js"
namespace com.keyman.text.prediction {
export interface ModelSpec {
/**
* The model's unique identifier.
*/
id: string;
/**
* Corresponds to the 'suggestionsready' LanguageProcessor event.
*/
export type ReadySuggestionsHandler = (prediction: ReadySuggestions) => boolean;
/**
* The list of supported BCP-47 language codes. Only one language should be supported,
* although multiple variants based on region code (at min) may be specified.
*/
languages: string[];
export type StateChangeEnum = 'active'|'configured'|'inactive';
/**
* Corresponds to the 'statechange' LanguageProcessor event.
*/
export type StateChangeHandler = (state: StateChangeEnum) => boolean;
/**
* The path/URL to the file that defines the model. If both `path` and `raw` are specified,
* `path` takes precedence.
*/
path: string;
/**
* Covers 'tryaccept' events.
*/
export type TryUIHandler = (source: string) => boolean;
/**
* The raw JS script defining the model. Only used if `path` is not specified.
*/
code: string;
export type InvalidateSourceEnum = 'new'|'context';
/**
* Corresponds to the 'invalidatesuggestions' LanguageProcessor event.
*/
export type InvalidateSuggestionsHandler = (source: InvalidateSourceEnum) => boolean;
export class ReadySuggestions {
suggestions: Suggestion[];
transcriptionID: number;
constructor(suggestions: Suggestion[], id: number) {
this.suggestions = suggestions;
this.transcriptionID = id;
}
}
interface LanguageProcessorEventMap {
'suggestionsready': ReadySuggestionsHandler,
'invalidatesuggestions': InvalidateSuggestionsHandler,
'statechange': StateChangeHandler,
'tryaccept': TryUIHandler,
'tryrevert': () => void,
/**
* Is called synchronously once suggestion application is successful and the context has been updated.
*
* @param outputTarget The `OutputTarget` representation of the context the suggestion was applied to.
* @returns
*/
'suggestionapplied': (outputTarget: OutputTarget) => boolean
}
/* Is more like the model configuration engine */
export default class LanguageProcessor extends EventEmitter<LanguageProcessorEventMap> {
private lmEngine: LMLayer;
private currentModel?: ModelSpec;
private configuration?: Configuration;
private currentPromise?: Promise<Suggestion[]>;
private recentTranscriptions: Transcription[] = [];
private _mayPredict: boolean = true;
private _mayCorrect: boolean = true;
private _state: StateChangeEnum = 'inactive';
private static readonly TRANSCRIPTION_BUFFER: 10 = 10;
public constructor(predictiveTextWorker: Worker, supportsRightDeletions: boolean = false) {
super();
// Establishes KMW's platform 'capabilities', which limit the range of context a LMLayer
// model may expect.
let capabilities: Capabilities = {
maxLeftContextCodePoints: 64,
// Since the apps don't yet support right-deletions.
maxRightContextCodePoints: supportsRightDeletions ? 0 : 64
}
if(!this.canEnable() || !predictiveTextWorker) {
return;
}
// TODO: this needs to be supplied separately - runs on Node need to refer to a different type!
this.lmEngine = new LMLayer(capabilities, predictiveTextWorker);
}
/**
* Corresponds to the 'suggestionsready' LanguageProcessor event.
*/
export type ReadySuggestionsHandler = (prediction: ReadySuggestions) => boolean;
export type StateChangeEnum = 'active'|'configured'|'inactive';
/**
* Corresponds to the 'statechange' LanguageProcessor event.
*/
export type StateChangeHandler = (state: StateChangeEnum) => boolean;
/**
* Covers both 'tryaccept' and 'tryrevert' events.
*/
export type TryUIHandler = (source: string) => boolean;
export type InvalidateSourceEnum = 'new'|'context';
/**
* Corresponds to the 'invalidatesuggestions' LanguageProcessor event.
*/
export type InvalidateSuggestionsHandler = (source: InvalidateSourceEnum) => boolean;
export class ReadySuggestions {
suggestions: Suggestion[];
transcriptionID: number;
constructor(suggestions: Suggestion[], id: number) {
this.suggestions = suggestions;
this.transcriptionID = id;
}
public get activeModel(): ModelSpec {
return this.currentModel;
}
type SupportedEventNames = "suggestionsready" | "invalidatesuggestions" | "statechange" | "tryaccept" | "tryrevert";
type SupportedEventHandler = InvalidateSuggestionsHandler | ReadySuggestionsHandler | StateChangeHandler | TryUIHandler;
public get isConfigured(): boolean {
return !!this.configuration;
}
export class LanguageProcessor extends EventEmitter {
private lmEngine: LMLayer;
private currentModel?: ModelSpec;
private configuration?: Configuration;
private currentPromise?: Promise<Suggestion[]>;
public get state(): StateChangeEnum {
return this._state;
}
private recentTranscriptions: Transcription[] = [];
public unloadModel() {
this.lmEngine.unloadModel();
delete this.currentModel;
delete this.configuration;
private _mayPredict: boolean = true;
private _mayCorrect: boolean = true;
this._state = 'inactive';
this.emit('statechange', 'inactive');
}
private static readonly TRANSCRIPTION_BUFFER: 10 = 10;
public init(supportsRightDeletions: boolean = false) {
// Establishes KMW's platform 'capabilities', which limit the range of context a LMLayer
// model may expect.
let capabilities: Capabilities = {
maxLeftContextCodePoints: 64,
// Since the apps don't yet support right-deletions.
maxRightContextCodePoints: supportsRightDeletions ? 0 : 64
}
if(!this.canEnable()) {
return;
}
this.lmEngine = new LMLayer(capabilities);
loadModel(model: ModelSpec): Promise<void> {
if(!model) {
throw new Error("Null reference not allowed.");
}
public get activeModel(): ModelSpec {
return this.currentModel;
let specType: 'file'|'raw' = model.path ? 'file' : 'raw';
let source = specType == 'file' ? model.path : model.code;
// We pre-emptively emit so that the banner's DOM elements may update synchronously.
// Prevents an ugly "flash of unstyled content" layout issue during keyboard load
// on our mobile platforms when embedded.
this.currentModel = model;
if(this.mayPredict) {
this._state = 'active';
this.emit('statechange', 'active');
}
public unloadModel() {
this.lmEngine.unloadModel();
delete this.currentModel;
delete this.configuration;
this.emit('statechange', 'inactive');
}
loadModel(model: ModelSpec): Promise<void> {
if(!model) {
throw new Error("Null reference not allowed.");
}
let specType: 'file'|'raw' = model.path ? 'file' : 'raw';
let source = specType == 'file' ? model.path : model.code;
let lp = this;
// We pre-emptively emit so that the banner's DOM elements may update synchronously.
// Prevents an ugly "flash of unstyled content" layout issue during keyboard load
// on our mobile platforms when embedded.
lp.currentModel = model;
if(this.mayPredict) {
lp.emit('statechange', 'active');
}
return this.lmEngine.loadModel(source, specType).then(function(config: Configuration) {
lp.configuration = config;
lp.emit('statechange', 'configured');
}).catch(function(error) {
// Does this provide enough logging information?
let message: string;
if(error instanceof Error) {
message = error.message;
} else {
message = String(error);
}
console.error("Could not load model '" + model.id + "': " + message);
// Since the model couldn't load, immediately deactivate. Visually, it'll look
// like the banner crashed shortly after load.
lp.currentModel = null;
lp.emit('statechange', 'inactive');
});
}
public invalidateContext(outputTarget: OutputTarget, layerId: string) {
// Signal to any predictive text UI that the context has changed, invalidating recent predictions.
this.emit('invalidatesuggestions', 'context');
// If there's no active model, there can be no predictions.
// We'll also be missing important data needed to even properly REQUEST the predictions.
if(!this.currentModel || !this.configuration) {
return;
}
// Don't attempt predictions when disabled!
// invalidateContext otherwise bypasses .predict()'s check against this.
if(!this.isActive) {
return;
} else if(outputTarget) {
let transcription = outputTarget.buildTranscriptionFrom(outputTarget, null, false);
this.predict_internal(transcription, true, layerId);
}
}
public wordbreak(target: OutputTarget, layerId: string): Promise<string> {
if(!this.isActive) {
return null;
}
let context = new ContextWindow(Mock.from(target, false), this.configuration, layerId);
return this.lmEngine.wordbreak(context);
}
public predict(transcription: Transcription, layerId: string): Promise<Suggestion[]> {
if(!this.isActive) {
return null;
}
// If there's no active model, there can be no predictions.
// We'll also be missing important data needed to even properly REQUEST the predictions.
if(!this.currentModel || !this.configuration) {
return null;
}
// We've already invalidated any suggestions resulting from any previously-existing Promise -
// may as well officially invalidate them via event.
this.emit("invalidatesuggestions", 'new');
return this.predict_internal(transcription, false, layerId);
}
/**
*
* @param suggestion
* @param outputTarget
* @param getLayerId a function that returns the current layerId,
* required because layerid can be changed by PostKeystroke
* @returns
*/
public applySuggestion(suggestion: Suggestion, outputTarget: OutputTarget, getLayerId: ()=>string): Promise<Reversion> {
if(!outputTarget) {
throw "Accepting suggestions requires a destination OutputTarget instance."
}
// Find the state of the context at the time the suggestion was generated.
// This may refer to the context before an input keystroke or before application
// of a predictive suggestion.
let original = this.getPredictionState(suggestion.transformId);
if(!original) {
console.warn("Could not apply the Suggestion!");
return null;
return this.lmEngine.loadModel(source, specType).then((config: Configuration) => {
this.configuration = config;
this._state = 'configured';
this.emit('statechange', 'configured');
}).catch((error) => {
// Does this provide enough logging information?
let message: string;
if(error instanceof Error) {
message = error.message;
} else {
// Apply the Suggestion!
// Step 1: determine the final output text
let final = text.Mock.from(original.preInput, false);
final.apply(suggestion.transform);
// Step 2: build a final, master Transform that will produce the desired results from the CURRENT state.
// In embedded mode, both Android and iOS are best served by calculating this transform and applying its
// values as needed for use with their IME interfaces.
let transform = final.buildTransformFrom(outputTarget);
outputTarget.apply(transform);
// Tell the banner that a suggestion was applied, so it can call the
// keyboard's PostKeystroke entry point as needed
this.emit('suggestionapplied', outputTarget);
// Build a 'reversion' Transcription that can be used to undo this apply() if needed,
// replacing the suggestion transform with the original input text.
let preApply = text.Mock.from(original.preInput, false);
preApply.apply(original.transform);
// Builds the reversion option according to the loaded lexical model's known
// syntactic properties.
let suggestionContext = new ContextWindow(original.preInput, this.configuration, getLayerId());
// We must accept the Suggestion from its original context, which was before
// `original.transform` was applied.
let reversionPromise: Promise<Reversion> = this.lmEngine.acceptSuggestion(suggestion, suggestionContext, original.transform);
// Also, request new prediction set based on the resulting context.
let lp = this;
reversionPromise = reversionPromise.then(function(reversion) {
let mappedReversion: Reversion = {
// By mapping back to the original Transcription that generated the Suggestion,
// the input will be automatically rewound to the preInput state.
transform: original.transform,
// The ID part is critical; the reversion can't be applied without it.
transformId: -original.token, // reversions use the additive inverse.
displayAs: reversion.displayAs, // The real reason we needed to call the LMLayer.
id: reversion.id,
tag: reversion.tag
}
// // If using the version from lm-layer:
// let mappedReversion = reversion;
// mappedReversion.transformId = reversionTranscription.token;
lp.predictFromTarget(outputTarget, getLayerId());
return mappedReversion;
});
return reversionPromise;
message = String(error);
}
console.error("Could not load model '" + model.id + "': " + message);
// Since the model couldn't load, immediately deactivate. Visually, it'll look
// like the banner crashed shortly after load.
this.currentModel = null;
this._state = 'inactive';
this.emit('statechange', 'inactive');
});
}
public invalidateContext(outputTarget: OutputTarget, layerId: string): Promise<Suggestion[]> {
// Signal to any predictive text UI that the context has changed, invalidating recent predictions.
this.emit('invalidatesuggestions', 'context');
// If there's no active model, there can be no predictions.
// We'll also be missing important data needed to even properly REQUEST the predictions.
if(!this.currentModel || !this.configuration) {
return Promise.resolve([]);
}
public applyReversion(reversion: Reversion, outputTarget: OutputTarget) {
if(!outputTarget) {
throw "Accepting suggestions requires a destination OutputTarget instance."
}
// Don't attempt predictions when disabled!
// invalidateContext otherwise bypasses .predict()'s check against this.
if(!this.isActive) {
return Promise.resolve([]);
} else if(outputTarget) {
let transcription = outputTarget.buildTranscriptionFrom(outputTarget, null, false);
return this.predict_internal(transcription, true, layerId);
}
}
// Find the state of the context at the time the suggestion was generated.
// This may refer to the context before an input keystroke or before application
// of a predictive suggestion.
//
// Reversions use the additive inverse of the id token of the Transcription being
// reverted to.
let original = this.getPredictionState(-reversion.transformId);
if(!original) {
console.warn("Could not apply the Suggestion!");
return;
}
public wordbreak(target: OutputTarget, layerId: string): Promise<string> {
if(!this.isActive) {
return null;
}
// Apply the Reversion!
let context = new ContextWindow(Mock.from(target, false), this.configuration, layerId);
return this.lmEngine.wordbreak(context);
}
public predict(transcription: Transcription, layerId: string): Promise<Suggestion[]> {
if(!this.isActive) {
return null;
}
// If there's no active model, there can be no predictions.
// We'll also be missing important data needed to even properly REQUEST the predictions.
if(!this.currentModel || !this.configuration) {
return null;
}
// We've already invalidated any suggestions resulting from any previously-existing Promise -
// may as well officially invalidate them via event.
this.emit("invalidatesuggestions", 'new');
return this.predict_internal(transcription, false, layerId);
}
/**
*
* @param suggestion
* @param outputTarget
* @param getLayerId a function that returns the current layerId,
* required because layerid can be changed by PostKeystroke
* @returns
*/
public applySuggestion(suggestion: Suggestion, outputTarget: OutputTarget, getLayerId: ()=>string): Promise<Reversion> {
if(!outputTarget) {
throw "Accepting suggestions requires a destination OutputTarget instance."
}
// Find the state of the context at the time the suggestion was generated.
// This may refer to the context before an input keystroke or before application
// of a predictive suggestion.
let original = this.getPredictionState(suggestion.transformId);
if(!original) {
console.warn("Could not apply the Suggestion!");
return null;
} else {
// Apply the Suggestion!
// Step 1: determine the final output text
let final = text.Mock.from(original.preInput, false);
final.apply(reversion.transform); // Should match original.transform, actually. (See applySuggestion)
let final = Mock.from(original.preInput, false);
final.apply(suggestion.transform);
// Step 2: build a final, master Transform that will produce the desired results from the CURRENT state.
// In embedded mode, both Android and iOS are best served by calculating this transform and applying its
@ -290,155 +228,227 @@ namespace com.keyman.text.prediction {
let transform = final.buildTransformFrom(outputTarget);
outputTarget.apply(transform);
// The reason we need to preserve the additive-inverse 'transformId' property on Reversions.
let promise = this.lmEngine.revertSuggestion(reversion, new ContextWindow(original.preInput, this.configuration, null))
// Tell the banner that a suggestion was applied, so it can call the
// keyboard's PostKeystroke entry point as needed
this.emit('suggestionapplied', outputTarget);
let lp = this;
return promise.then(function(suggestions: Suggestion[]) {
let result = new ReadySuggestions(suggestions, transform.id);
lp.emit("suggestionsready", result);
lp.currentPromise = null;
// Build a 'reversion' Transcription that can be used to undo this apply() if needed,
// replacing the suggestion transform with the original input text.
let preApply = Mock.from(original.preInput, false);
preApply.apply(original.transform);
return suggestions;
});
}
// Builds the reversion option according to the loaded lexical model's known
// syntactic properties.
let suggestionContext = new ContextWindow(original.preInput, this.configuration, getLayerId());
public predictFromTarget(outputTarget: OutputTarget, layerId: string): Promise<Suggestion[]> {
if(!outputTarget) {
return null;
}
// We must accept the Suggestion from its original context, which was before
// `original.transform` was applied.
let reversionPromise: Promise<Reversion> = this.lmEngine.acceptSuggestion(suggestion, suggestionContext, original.transform);
let transcription = outputTarget.buildTranscriptionFrom(outputTarget, null, false);
return this.predict(transcription, layerId);
}
/**
* Called internally to do actual predictions after any relevant "invalidatesuggestions" events
* have been raised.
* @param transcription The triggering transcription (if it exists)
*/
private predict_internal(transcription: Transcription, resetContext: boolean, layerId: string): Promise<Suggestion[]> {
if(!transcription) {
return null;
}
let context = new ContextWindow(transcription.preInput, this.configuration, layerId);
this.recordTranscription(transcription);
if(resetContext) {
this.lmEngine.resetContext(context);
}
let alternates = transcription.alternates;
if(!alternates || alternates.length == 0) {
alternates = [{
sample: transcription.transform,
p: 1.0
}];
}
let transform = transcription.transform;
var promise = this.currentPromise = this.lmEngine.predict(alternates, context);
let lp = this;
return promise.then(function(suggestions: Suggestion[]) {
if(promise == lp.currentPromise) {
let result = new ReadySuggestions(suggestions, transform.id);
lp.emit("suggestionsready", result);
lp.currentPromise = null;
// Also, request new prediction set based on the resulting context.
reversionPromise = reversionPromise.then((reversion) => {
let mappedReversion: Reversion = {
// By mapping back to the original Transcription that generated the Suggestion,
// the input will be automatically rewound to the preInput state.
transform: original.transform,
// The ID part is critical; the reversion can't be applied without it.
transformId: -original.token, // reversions use the additive inverse.
displayAs: reversion.displayAs, // The real reason we needed to call the LMLayer.
id: reversion.id,
tag: reversion.tag
}
return suggestions;
// // If using the version from lm-layer:
// let mappedReversion = reversion;
// mappedReversion.transformId = reversionTranscription.token;
this.predictFromTarget(outputTarget, getLayerId());
return mappedReversion;
});
}
private recordTranscription(transcription: Transcription) {
this.recentTranscriptions.push(transcription);
if(this.recentTranscriptions.length > LanguageProcessor.TRANSCRIPTION_BUFFER) {
this.recentTranscriptions.splice(0, 1);
}
}
/**
* Retrieves the context and output state of KMW immediately before the prediction with
* token `id` was generated. Must correspond to a 'recent' one, as only so many are stored
* in `ModelManager`'s history buffer.
* @param id A unique identifier corresponding to a recent `Transcription`.
* @returns The matching `Transcription`, or `null` none is found.
*/
public getPredictionState(id: number): Transcription {
let match = this.recentTranscriptions.filter(function(t: Transcription) {
return t.token == id;
})
return match.length == 0 ? null : match[0];
}
public shutdown() {
this.lmEngine.shutdown();
}
public get isActive(): boolean {
if(!this.canEnable()) {
this._mayPredict = false;
return false;
}
return (this.activeModel || false) && this._mayPredict;
}
public canEnable(): boolean {
// Is overridden for dom-aware KMW in case of old IE versions.
return true;
}
public get mayPredict() {
return this._mayPredict;
}
public set mayPredict(flag: boolean) {
if(!this.canEnable()) {
return;
}
let oldVal = this._mayPredict;
this._mayPredict = flag;
if(oldVal != flag) {
// If there's no model to be activated and we've reached this point,
// the banner should remain inactive, as it already was.
// If it there was one and we've reached this point, we're globally
// deactivating, so we're fine.
if(this.activeModel) {
this.emit('statechange', flag ? 'active' : 'inactive');
}
}
}
public get mayCorrect() {
return this._mayCorrect;
}
public set mayCorrect(flag: boolean) {
this._mayCorrect = flag;
}
public get wordbreaksAfterSuggestions() {
return this.configuration.wordbreaksAfterSuggestions;
}
public tryAcceptSuggestion(source: string): boolean {
let returnObj = {shouldSwallow: false};
this.emit('tryaccept', source, returnObj);
return returnObj.shouldSwallow;
}
public tryRevertSuggestion(): boolean {
let returnObj = {shouldSwallow: false};
this.emit('tryrevert', returnObj);
return returnObj.shouldSwallow;
return reversionPromise;
}
}
public applyReversion(reversion: Reversion, outputTarget: OutputTarget) {
if(!outputTarget) {
throw "Accepting suggestions requires a destination OutputTarget instance."
}
// Find the state of the context at the time the suggestion was generated.
// This may refer to the context before an input keystroke or before application
// of a predictive suggestion.
//
// Reversions use the additive inverse of the id token of the Transcription being
// reverted to.
let original = this.getPredictionState(-reversion.transformId);
if(!original) {
console.warn("Could not apply the Suggestion!");
return;
}
// Apply the Reversion!
// Step 1: determine the final output text
let final = Mock.from(original.preInput, false);
final.apply(reversion.transform); // Should match original.transform, actually. (See applySuggestion)
// Step 2: build a final, master Transform that will produce the desired results from the CURRENT state.
// In embedded mode, both Android and iOS are best served by calculating this transform and applying its
// values as needed for use with their IME interfaces.
let transform = final.buildTransformFrom(outputTarget);
outputTarget.apply(transform);
// The reason we need to preserve the additive-inverse 'transformId' property on Reversions.
let promise = this.lmEngine.revertSuggestion(reversion, new ContextWindow(original.preInput, this.configuration, null))
return promise.then((suggestions: Suggestion[]) => {
let result = new ReadySuggestions(suggestions, transform.id);
this.emit("suggestionsready", result);
this.currentPromise = null;
return suggestions;
});
}
public predictFromTarget(outputTarget: OutputTarget, layerId: string): Promise<Suggestion[]> {
if(!outputTarget) {
return null;
}
let transcription = outputTarget.buildTranscriptionFrom(outputTarget, null, false);
return this.predict(transcription, layerId);
}
/**
* Called internally to do actual predictions after any relevant "invalidatesuggestions" events
* have been raised.
* @param transcription The triggering transcription (if it exists)
*/
private predict_internal(transcription: Transcription, resetContext: boolean, layerId: string): Promise<Suggestion[]> {
if(!transcription) {
return null;
}
let context = new ContextWindow(transcription.preInput, this.configuration, layerId);
this.recordTranscription(transcription);
if(resetContext) {
this.lmEngine.resetContext(context);
}
let alternates = transcription.alternates;
if(!alternates || alternates.length == 0) {
alternates = [{
sample: transcription.transform,
p: 1.0
}];
}
let transform = transcription.transform;
var promise = this.currentPromise = this.lmEngine.predict(alternates, context);
return promise.then((suggestions: Suggestion[]) => {
if(promise == this.currentPromise) {
let result = new ReadySuggestions(suggestions, transform.id);
this.emit("suggestionsready", result);
this.currentPromise = null;
}
return suggestions;
});
}
private recordTranscription(transcription: Transcription) {
this.recentTranscriptions.push(transcription);
if(this.recentTranscriptions.length > LanguageProcessor.TRANSCRIPTION_BUFFER) {
this.recentTranscriptions.splice(0, 1);
}
}
/**
* Retrieves the context and output state of KMW immediately before the prediction with
* token `id` was generated. Must correspond to a 'recent' one, as only so many are stored
* in `ModelManager`'s history buffer.
* @param id A unique identifier corresponding to a recent `Transcription`.
* @returns The matching `Transcription`, or `null` none is found.
*/
public getPredictionState(id: number): Transcription {
let match = this.recentTranscriptions.filter((t: Transcription) => {
return t.token == id;
})
return match.length == 0 ? null : match[0];
}
public shutdown() {
this.lmEngine.shutdown();
}
public get isActive(): boolean {
if(!this.canEnable()) {
this._mayPredict = false;
return false;
}
return (this.activeModel || false) && this._mayPredict;
}
public canEnable(): boolean {
// Is overridden for dom-aware KMW in case of old IE versions.
return true;
}
public get mayPredict() {
return this._mayPredict;
}
public set mayPredict(flag: boolean) {
if(!this.canEnable()) {
return;
}
let oldVal = this._mayPredict;
this._mayPredict = flag;
if(oldVal != flag) {
// If there's no model to be activated and we've reached this point,
// the banner should remain inactive, as it already was.
// If it there was one and we've reached this point, we're globally
// deactivating, so we're fine.
if(this.activeModel) {
let state: StateChangeEnum = flag ? 'active' : 'inactive';
this._state = state;
this.emit('statechange', state);
}
}
}
public get mayCorrect() {
return this._mayCorrect;
}
public set mayCorrect(flag: boolean) {
this._mayCorrect = flag;
}
public get wordbreaksAfterSuggestions() {
return this.configuration.wordbreaksAfterSuggestions;
}
public tryAcceptSuggestion(source: string): boolean {
// If and when we do auto-correct, the suggestion is to pass this object to the event and
// denote any mutations to the contained value.
//let returnObj = {shouldSwallow: false};
this.emit('tryaccept', source);
return false;
}
public tryRevertSuggestion(): boolean {
// If and when we do auto-revert, the suggestion is to pass this object to the event and
// denote any mutations to the contained value.
//let returnObj = {shouldSwallow: false};
this.emit('tryrevert');
return false;
}
}

27
common/web/input-processor/src/text/prediction/modelSpec.ts Normal file
View file

@ -0,0 +1,27 @@
/**
* Defines the metadata needed to support and load lexical models compatible with
* Keyman Engine for Web.
*/
export default interface ModelSpec {
/**
* The model's unique identifier.
*/
id: string;
/**
* The list of supported BCP-47 language codes. Only one language should be supported,
* although multiple variants based on region code (at min) may be specified.
*/
languages: string[];
/**
* The path/URL to the file that defines the model. If both `path` and `code` are specified,
* `path` takes precedence.
*/
path: string;
/**
* The raw JS script defining the model. Only used if `path` is not specified.
*/
code: string;
}

Some files were not shown because too many files have changed in this diff Show more