erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-10-13 17:37:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Merge branch 'epic/user-dict' into auto/A19S25-merge-master-into-user-dict

Browse source
This commit is contained in:
Keyman Server 2026-03-27 06:21:30 +01:00 • committed by GitHub
commit 0c6f7edcf2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 1568 additions and 771 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
common/web/types/src/lexical-model-types.ts
View file

@ -32,30 +32,37 @@ export type TextWithProbability = {
*/
export interface LexiconTraversal {
/**
* Provides an iterable pattern used to search for words with a 'keyed' prefix matching
* the current traversal state's prefix when a new character is appended. Iterating
* across `children` provides 'breadth' to a lexical search.
* Provides an iterable pattern used to search for words with a 'keyed' prefix
* matching the current traversal state's prefix when a new character is
* appended. Iterating across `children` provides 'breadth' to a lexical
* search.
*
* For an example with English, if the current traversal state corresponds to 'th',
* children() may return an iterator with states corresponding 'e'
* (for 'the', 'then', 'there'), 'a' (for 'than', 'that'), etc.
* For an example with English, if the current traversal state corresponds to
* 'th', children() may return an iterator with states corresponding 'e' (for
* 'the', 'then', 'there'), 'a' (for 'than', 'that'), etc.
*
* @param char A full character (a UTF-16 code point, which may be comprised of two
* code units) corresponding to the child node, appended to the current
* node's prefix to produce the child node's prefix.
* @param char A full character (a UTF-16 code point, which may be
* comprised of two code units) corresponding to the child
* node, appended to the current node's prefix to produce the
* child node's prefix.
* <p>
* Example: if the current traversal's represented prefix is 'th',
* char = 'e' would indicate a child node with prefix = 'the'.
* @param traversal a LexiconTraversal starting from (or "rooted at") the child node. Use
* of the returned object provides 'depth' to a lexical search.
* @param traversal a LexiconTraversal starting from (or "rooted at") the
* child node. Use of the returned object provides 'depth'
* to a lexical search.
* <p>
* Example:
*
* - Suppose our current LexiconTraversal represents a prefix of 'th'.
* - If `char` = 'e', the child represents a prefix of 'the'.
* - Then `traversal` allows traversing the part of the lexicon prefixed by 'the'.
* - Then `traversal` allows traversing the part of the lexicon prefixed by
* 'the'.
*
* Note that there is no requirement for the ordering of the returned entries;
* no sorting of any kind is required.
*/
children(): Generator<{char: string, traversal: () => LexiconTraversal}>;
children(): {char: string, p: number, traversal: () => LexiconTraversal}[];
/**
* Allows direct access to the traversal state that results when appending one

4
developer/src/kmc-model/build.sh
View file

@ -11,8 +11,10 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")"
builder_describe "Keyman kmc Lexical Model Compiler module" \
"@/common/web/keyman-version" \
"@/common/web/types" \
"@/developer/src/common/web/test-helpers" \
"@/web/src/engine/predictive-text/templates/ test" \
"@/developer/src/common/web/utils" \
"@/web/src/engine/predictive-text/templates/" \
"clean" \
"configure" \
"build" \

2
developer/src/kmc-model/package.json
View file

@ -32,11 +32,13 @@
"dependencies": {
"@keymanapp/common-types": "*",
"@keymanapp/keyman-version": "*",
"@keymanapp/models-templates": "*",
"typescript": "^5.4.5"
},
"devDependencies": {
"@keymanapp/developer-test-helpers": "*",
"@keymanapp/models-templates": "*",
"@types/node": "^20.4.1",
"c8": "^7.12.0",
"chalk": "^2.4.2",
"esbuild": "^0.25.0"

337
developer/src/kmc-model/src/build-trie.ts
View file

@ -1,6 +1,8 @@
import { ModelCompilerError, ModelCompilerMessageContext, ModelCompilerMessages } from "./model-compiler-messages.js";
import { callbacks } from "./compiler-callbacks.js";
import { TrieBuilder } from '@keymanapp/models-templates';
// Supports LF or CRLF line terminators.
const NEWLINE_SEPARATOR = /\u000d?\u000a/;
@ -29,7 +31,7 @@ export function createTrieDataStructure(filenames: string[], searchTermToKey?: (
const wordlist: WordList = {};
filenames.forEach(filename => parseWordListFromFilename(wordlist, filename));
const trie = Trie.buildTrie(wordlist, searchTermToKey as Trie.SearchTermToKey);
const trie = buildTrie(wordlist, searchTermToKey as SearchTermToKey);
return JSON.stringify(trie);
}
@ -188,302 +190,51 @@ function* enumerateLines(lines: string[]): Generator<LineNoAndText> {
}
}
namespace Trie {
/**
* An **opaque** type for a string that is exclusively used as a search key in
* the trie. There should be a function that converts arbitrary strings
* (queries) and converts them into a standard search key for a given language
* model.
*
* Fun fact: This opaque type has ALREADY saved my bacon and found a bug!
*/
type SearchKey = string & { _: 'SearchKey'};
/**
* An **opaque** type for a string that is exclusively used as a search key in
* the trie. There should be a function that converts arbitrary strings
* (queries) and converts them into a standard search key for a given language
* model.
*
* Fun fact: This opaque type has ALREADY saved my bacon and found a bug!
*/
type SearchKey = string & { _: 'SearchKey'};
/**
* A function that converts a string (word form or query) into a search key
* (secretly, this is also a string).
*/
export interface SearchTermToKey {
(wordform: string): SearchKey;
/**
* A function that converts a string (word form or query) into a search key
* (secretly, this is also a string).
*/
export interface SearchTermToKey {
(wordform: string): SearchKey;
}
/**
* Builds a trie from a word list.
*
* @param wordlist The wordlist with non-negative weights.
* @param keyFunction Function that converts word forms into indexed search keys
* @returns A JSON-serialiable object that can be given to the TrieModel constructor.
*/
export function buildTrie(wordlist: WordList, keyFunction: SearchTermToKey): object {
const collater = new TrieBuilder(keyFunction);
buildFromWordList(collater, wordlist);
return {
totalWeight: collater.getTotalWeight(),
root: collater.getRoot()
}
}
// The following trie implementation has been (heavily) derived from trie-ing
// by Conrad Irwin.
//
// trie-ing is distributed under the terms of the MIT license, reproduced here:
//
// The MIT License
// Copyright (c) 2015-2017 Conrad Irwin <conrad.irwin@gmail.com>
// Copyright (c) 2011 Marc Campbell <marc.e.campbell@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
//
// See: https://github.com/ConradIrwin/trie-ing/blob/df55d7af7068d357829db9e0a7faa8a38add1d1d/LICENSE
/**
* An entry in the prefix trie. The matched word is "content".
*/
interface Entry {
content: string;
key: SearchKey;
weight: number;
}
/**
* The trie is made up of nodes. A node can be EITHER an internal node (whose
* only children are other nodes) OR a leaf, which actually contains the word
* form entries.
*/
type Node = InternalNode | Leaf;
/**
* An internal node.
*/
interface InternalNode {
type: 'internal';
weight: number;
// TODO: As an optimization, "values" can be a single string!
values: string[];
children: { [codeunit: string]: Node };
unsorted?: true;
}
/**
* A leaf node.
*/
interface Leaf {
type: 'leaf';
weight: number;
entries: Entry[];
unsorted?: true;
}
/**
* A sentinel value for when an internal node has contents and requires an
* "internal" leaf. That is, this internal node has content. Instead of placing
* entries as children in an internal node, a "fake" leaf is created, and its
* key is this special internal value.
*
* The value is a valid Unicode BMP code point, but it is a "non-character".
* Unicode will never assign semantics to these characters, as they are
* intended to be used internally as sentinel values.
*/
const INTERNAL_VALUE = '\uFDD0';
/**
* Builds a trie from a word list.
*
* @param wordlist The wordlist with non-negative weights.
* @param keyFunction Function that converts word forms into indexed search keys
* @returns A JSON-serialiable object that can be given to the TrieModel constructor.
*/
export function buildTrie(wordlist: WordList, keyFunction: SearchTermToKey): object {
const root = new Trie(keyFunction).buildFromWordList(wordlist).root;
return {
totalWeight: sumWeights(root),
root: root
}
}
/**
* Wrapper class for the trie and its nodes and wordform to search
*/
class Trie {
readonly root = createRootNode();
toKey: SearchTermToKey;
constructor(wordform2key: SearchTermToKey) {
this.toKey = wordform2key;
}
/**
* Populates the trie with the contents of an entire wordlist.
* @param words a list of word and count pairs.
*/
buildFromWordList(words: WordList): Trie {
for (const [wordform, weight] of Object.entries(words)) {
const key = this.toKey(wordform);
addUnsorted(this.root, { key, weight, content: wordform }, 0);
}
sortTrie(this.root);
return this;
}
}
// "Constructors"
function createRootNode(): Node {
return {
type: 'leaf',
weight: 0,
entries: []
};
}
// Implement Trie creation.
/**
* Adds an entry to the trie.
*
* Note that the trie will likely be unsorted after the add occurs. Before
* performing a lookup on the trie, use call sortTrie() on the root note!
*
* @param node Which node should the entry be added to?
* @param entry the wordform/weight/key to add to the trie
* @param index the index in the key and also the trie depth. Should be set to
* zero when adding onto the root node of the trie.
*/
function addUnsorted(node: Node, entry: Entry, index: number = 0) {
// Each node stores the MAXIMUM weight out of all of its decesdents, to
// enable a greedy search through the trie.
node.weight = Math.max(node.weight, entry.weight);
// When should a leaf become an interior node?
// When it already has a value, but the key of the current value is longer
// than the prefix.
if (node.type === 'leaf' && index < entry.key.length && node.entries.length >= 1) {
convertLeafToInternalNode(node, index);
}
if (node.type === 'leaf') {
// The key matches this leaf node, so add yet another entry.
addItemToLeaf(node, entry);
} else {
// Push the node down to a lower node.
addItemToInternalNode(node, entry, index);
}
node.unsorted = true;
}
/**
* Adds an item to the internal node at a given depth.
* @param item
* @param index
*/
function addItemToInternalNode(node: InternalNode, item: Entry, index: number) {
let char = item.key[index];
// If an internal node is the proper site for item, it belongs under the
// corresponding (sentinel, internal-use) child node signifying this.
if(char == undefined) {
char = INTERNAL_VALUE;
}
if (!node.children[char]) {
node.children[char] = createRootNode();
node.values.push(char);
}
addUnsorted(node.children[char], item, index + 1);
}
function addItemToLeaf(leaf: Leaf, item: Entry) {
leaf.entries.push(item);
}
/**
* Mutates the given Leaf to turn it into an InternalNode.
*
* NOTE: the node passed in will be DESTRUCTIVELY CHANGED into a different
* type when passed into this function!
*
* @param depth depth of the trie at this level.
*/
function convertLeafToInternalNode(leaf: Leaf, depth: number): void {
const entries = leaf.entries;
// Alias the current node, as the desired type.
const internal = (<unknown> leaf) as InternalNode;
internal.type = 'internal';
delete leaf.entries;
internal.values = [];
internal.children = {};
// Convert the old values array into the format for interior nodes.
for (const item of entries) {
let char: string;
if (depth < item.key.length) {
char = item.key[depth];
} else {
char = INTERNAL_VALUE;
}
if (!internal.children[char]) {
internal.children[char] = createRootNode();
internal.values.push(char);
}
addUnsorted(internal.children[char], item, depth + 1);
}
internal.unsorted = true;
}
/**
* Recursively sort the trie, in descending order of weight.
* @param node any node in the trie
*/
function sortTrie(node: Node) {
if (node.type === 'leaf') {
if (!node.unsorted) {
return;
}
node.entries.sort(function (a, b) { return b.weight - a.weight; });
} else {
// We MUST recurse and sort children before returning.
for (const char of node.values) {
sortTrie(node.children[char]);
}
if (!node.unsorted) {
return;
}
node.values.sort((a, b) => {
return node.children[b].weight - node.children[a].weight;
});
}
delete node.unsorted;
}
/**
* O(n) recursive traversal to sum the total weight of all leaves in the
* trie, starting at the provided node.
*
* @param node The node to start summing weights.
*/
function sumWeights(node: Node): number {
let val: number;
if (node.type === 'leaf') {
val = node.entries
.map(entry => entry.weight)
//.map(entry => isNaN(entry.weight) ? 1 : entry.weight)
.reduce((acc, count) => acc + count, 0);
} else {
val = Object.keys(node.children)
.map((key) => sumWeights(node.children[key]))
.reduce((acc, count) => acc + count, 0);
}
if(isNaN(val)) {
throw new Error("Unexpected NaN has appeared!");
}
return val;
/**
* Populates the trie with the contents of an entire wordlist.
* @param words a list of word and count pairs.
*/
function buildFromWordList(trieCollator: TrieBuilder, words: WordList): TrieBuilder {
for (const [wordform, weight] of Object.entries(words)) {
trieCollator.addEntry(wordform, weight);
}
trieCollator.sort();
return trieCollator;
}
/**

2
package-lock.json generated
View file

@ -804,11 +804,13 @@
"dependencies": {
"@keymanapp/common-types": "*",
"@keymanapp/keyman-version": "*",
"@keymanapp/models-templates": "*",
"typescript": "^5.4.5"
},
"devDependencies": {
"@keymanapp/developer-test-helpers": "*",
"@keymanapp/models-templates": "*",
"@types/node": "^20.4.1",
"c8": "^7.12.0",
"chalk": "^2.4.2",
"esbuild": "^0.25.0"

16
web/src/engine/predictive-text/templates/src/common.ts
View file

@ -157,3 +157,19 @@ export function defaultApplyCasing(casing: CasingForm, text: string): string {
return text.substring(0, headUnitLength).toUpperCase() .concat(text.substring(headUnitLength));
}
}
/**
* An **opaque** type for a string that is exclusively used as a search key in
* the trie. There should be a function that converts arbitrary strings
* (queries) and converts them into a standard search key for a given language
* model.
*/
export type SearchKey = string & { _: 'SearchKey'};
/**
* A function that converts a string (word form or query) into a search key
* (which, internally, is also a string type).
*/
export interface Wordform2Key {
(wordform: string): SearchKey;
}

9
web/src/engine/predictive-text/templates/src/index.ts
View file

@ -1,7 +1,12 @@
export {
SENTINEL_CODE_UNIT, applyTransform, buildMergedTransform, isHighSurrogate, isLowSurrogate, isSentinel,
transformToSuggestion, defaultApplyCasing
applyTransform, buildMergedTransform, isHighSurrogate, isLowSurrogate, isSentinel,
SearchKey, SENTINEL_CODE_UNIT, transformToSuggestion, defaultApplyCasing
} from "./common.js";
export { default as QuoteBehavior } from "./quote-behavior.js";
export { getLastPreCaretToken, Token, Tokenization, tokenize, wordbreak } from "./tokenization.js";
export {
Entry, InternalNode, Leaf, Node
} from './trie.js';
export { TrieBuilder } from './trie-builder.js';
export * as trieConstruction from './trie-builder.js';
export { default as TrieModel, TrieModelOptions } from "./trie-model.js";

297
web/src/engine/predictive-text/templates/src/traversal-model.ts Normal file
View file

@ -0,0 +1,297 @@
/*
* Copyright (c) 2019 National Research Council Canada (author: Eddie A. Santos)
* Copyright (c) 2019 SIL International
* Copyright (c) 2015–2017 Conrad Irwin
* Copyright (c) 2011–2015 Marc Campbell
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to
* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
* the Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
// Allows the kmwstring bindings to resolve.
import { KMWString, PriorityQueue } from "@keymanapp/web-utils";
import { default as defaultWordBreaker } from "@keymanapp/models-wordbreakers";
import { applyTransform, SearchKey, transformToSuggestion, Wordform2Key } from "./common.js";
import { getLastPreCaretToken } from "./tokenization.js";
import { LexicalModelTypes } from "@keymanapp/common-types";
import Capabilities = LexicalModelTypes.Capabilities;
import CasingFunction = LexicalModelTypes.CasingFunction;
import Configuration = LexicalModelTypes.Configuration;
import Context = LexicalModelTypes.Context;
import Distribution = LexicalModelTypes.Distribution;
import LexicalModel = LexicalModelTypes.LexicalModel;
import LexicalModelPunctuation = LexicalModelTypes.LexicalModelPunctuation;
import LexiconTraversal = LexicalModelTypes.LexiconTraversal;
import Suggestion = LexicalModelTypes.Suggestion;
import TextWithProbability = LexicalModelTypes.TextWithProbability;
import Transform = LexicalModelTypes.Transform;
import WithOutcome = LexicalModelTypes.WithOutcome;
import WordBreakingFunction = LexicalModelTypes.WordBreakingFunction;
/**
* @file traversal-model.ts
*
* Defines a lexical model based upon the implementation of a `LexiconTraversal`.
*/
/** Upper bound on the amount of suggestions to generate. */
const MAX_SUGGESTIONS = 12;
/**
* Additional arguments to pass into the model, in addition to the model
* parameters themselves.
*/
export interface ModelOptions {
/**
* How to break words in a phrase.
*/
wordBreaker?: WordBreakingFunction;
/**
* This should simplify a search term into a key.
*/
searchTermToKey?: (searchTerm: string) => string;
/**
* Indicates that the model's written form has 'casing' behavior.
*/
languageUsesCasing?: boolean;
/**
* Specifies a function used to apply the language's casing rules to a word.
*/
applyCasing?: CasingFunction;
/**
* Any punctuation to expose to the user.
*/
punctuation?: LexicalModelPunctuation;
}
/**
* @class TraversalModel
*
* Defines a lexical model based upon the implementation of a `LexiconTraversal`.
*/
export class TraversalModel implements LexicalModel {
configuration?: Configuration;
private rootTraversal: LexiconTraversal;
readonly breakWords: WordBreakingFunction;
readonly punctuation?: LexicalModelPunctuation;
readonly languageUsesCasing?: boolean;
readonly _toKey: Wordform2Key;
readonly applyCasing?: CasingFunction;
constructor(rootTraversal: LexiconTraversal, options: ModelOptions = {}) {
this.languageUsesCasing = options.languageUsesCasing;
this.applyCasing = options.applyCasing;
this.rootTraversal = rootTraversal;
this._toKey = options.searchTermToKey as Wordform2Key || defaultSearchTermToKey
this.breakWords = options.wordBreaker || defaultWordBreaker;
this.punctuation = options.punctuation;
}
configure(capabilities: Capabilities): Configuration {
return this.configuration = {
leftContextCodePoints: capabilities.maxLeftContextCodePoints,
rightContextCodePoints: capabilities.maxRightContextCodePoints ?? 0
};
}
toKey(text: string): string {
return this._toKey(text);
}
predict(transform: Transform, context: Context): Distribution<Suggestion> {
const makeDistribution = function(suggestions: WithOutcome<Suggestion>[]): Distribution<Suggestion> {
let distribution: Distribution<Suggestion> = [];
for(const s of suggestions) {
distribution.push({sample: s, p: s.p});
}
return distribution;
}
// Special-case the empty buffer/transform: return the top suggestions.
if (!transform.insert && !context.left && !context.right && context.startOfBuffer && context.endOfBuffer) {
return makeDistribution(this.firstN(MAX_SUGGESTIONS).map(({text, p}) => ({
transform: {
insert: text,
deleteLeft: 0
},
displayAs: text,
p: p
})));
}
// Compute the results of the keystroke:
const newContext = applyTransform(transform, context);
// Computes the different in word length after applying the transform above.
const leftDelOffset = transform.deleteLeft - KMWString.length(transform.insert);
// All text to the left of the cursor INCLUDING anything that has
// just been typed.
const prefix = getLastPreCaretToken(this.breakWords, newContext);
// Return suggestions from the trie.
return makeDistribution(this.lookup(prefix).map(({text, p}) =>
transformToSuggestion({
insert: text,
// Delete whatever the prefix that the user wrote.
deleteLeft: leftDelOffset + KMWString.length(prefix)
// Note: a separate capitalization/orthography engine can take this
// result and transform it as needed.
},
p
)));
}
get wordbreaker(): WordBreakingFunction {
return this.breakWords;
}
public traverseFromRoot(): LexiconTraversal {
return this.rootTraversal;
}
/**
* Returns the top N suggestions from the trie.
* @param n How many suggestions, maximum, to return.
*/
firstN(n: number): TextWithProbability[] {
return getSortedResults(this.rootTraversal, n);
}
/**
* Lookups an arbitrary prefix (a query) in the trie. Returns the top 3
* results in sorted order.
*
* @param prefix
*/
lookup(prefix: string): TextWithProbability[] {
const searchKey = this.toKey(prefix);
const rootTraversal = this.traverseFromRoot().child(searchKey);
if(!rootTraversal) {
return [];
}
const directEntries = rootTraversal.entries;
// `Set` requires Chrome 38+, which is more recent than Chrome 35.
const directSet: Record<string, string> = {};
for(const entry of directEntries) {
directSet[entry.text] = entry.text;
}
const bestEntries = getSortedResults(rootTraversal);
const deduplicated = bestEntries.filter((entry) => !directSet[entry.text]);
// Any entries directly hosted on the current node should get full display
// priority over anything from its descendants.
return directEntries.concat(deduplicated);
}
};
/**
* The priority queue will always pop the most probable item - be it a Traversal
* state or a lexical entry reached via Traversal.
*/
type TraversableWithProb = TextWithProbability | LexiconTraversal;
/**
* Returns all entries matching the given prefix, in descending order of
* weight.
*
* @param prefix the prefix to match.
* @param results the current results
* @param queue
*/
function getSortedResults(traversal: LexiconTraversal, limit = MAX_SUGGESTIONS): TextWithProbability[] {
let queue = new PriorityQueue(function(a: TraversableWithProb, b: TraversableWithProb) {
// In case of Trie compilation issues that emit `null` or `undefined`
return (b ? b.p : 0) - (a ? a.p : 0);
});
let results: TextWithProbability[] = [];
queue.enqueue(traversal);
while(queue.count > 0) {
const entry = queue.dequeue();
if((entry as TextWithProbability)!.text !== undefined) {
const lexicalEntry = entry as TextWithProbability;
results.push(lexicalEntry);
if(results.length >= limit) {
return results;
}
} else {
const traversal = entry as LexiconTraversal;
queue.enqueueAll(traversal.entries);
let children: LexiconTraversal[] = []
for(let child of traversal.children()) {
children.push(child.traversal());
}
queue.enqueueAll(children);
}
}
return results;
}
/**
* Converts wordforms into an indexable form. It does this by
* normalizing into NFD, removing diacritics, and then converting
* the result to lowercase.
*
* This is a very naïve implementation, that I only think will work on
* some languages that use the Latin script. As of 2020-04-08, only
* 4 out of 11 (36%) of published language models use the Latin script,
* so this might not actually be a great default.
*
* This uses String.prototype.normalize() to convert normalize into NFD.
* NFD is an easy way to separate a Latin character from its diacritics;
* Even then, some Latin-based orthographies use code points that,
* under NFD normalization, do NOT decompose into an ASCII letter and a
* combining diacritical mark (e.g., SENĆOŦEN).
*
* Use this only in early iterations of the model. For a production lexical
* model, you SHOULD write/generate your own key function, tailored to your
* language.
*/
function defaultSearchTermToKey(wordform: string): SearchKey {
/**
* N.B.: this is (slightly) DIFFERENT than the version in
* keymanapp/lexical-model-compiler/build-trie
* as this is for compatibility for models built
* BEFORE the searchTermToKey function was bundled with
* all models.
*
* This compatibility version lowercases AFTER removing diacritics;
* the new version (bundled in future models) lowercases,
* NFD normalizes, THEN removes diacritics.
*/
return wordform
.normalize('NFD')
// Remove all combining diacritics (if input is in NFD)
// common to Latin-orthographies.
.replace(/[\u0300-\u036f]/g, '')
.toLowerCase() as SearchKey;
}

146
web/src/engine/predictive-text/templates/src/trie-builder.ts Normal file
View file

@ -0,0 +1,146 @@
import { SENTINEL_CODE_UNIT, Wordform2Key } from "./common.js";
import { Entry, InternalNode, Leaf, Node, Trie, sortNode } from "./trie.js";
export function createRootNode(): Node {
return {
type: 'leaf',
weight: 0,
entries: []
};
}
/**
* Adds an entry to the trie. Currently assumes there is no pre-existing match
* for the entry.
*
* Note that the trie will likely be unsorted after the add occurs. Before
* performing a lookup on the trie, use call sortTrie() on the root note!
*
* @param node Which node should the entry be added to?
* @param entry the wordform/weight/key to add to the trie
* @param index the index in the key and also the trie depth. Should be set to
* zero when adding onto the root node of the trie.
*/
export function addUnsorted(node: Node, entry: Entry, index: number = 0) {
// Each node stores the MAXIMUM weight out of all of its decesdents, to
// enable a greedy search through the trie.
node.weight = Math.max(node.weight, entry.weight);
// When should a leaf become an interior node?
// When it already has a value, but the key of the current value is longer
// than the prefix.
if (node.type === 'leaf' && index < entry.key.length && node.entries.length >= 1) {
convertLeafToInternalNode(node, index);
}
if (node.type === 'leaf') {
// The key matches this leaf node, so add yet another entry.
addItemToLeaf(node, entry);
} else {
// Push the node down to a lower node.
addItemToInternalNode(node, entry, index);
}
node.unsorted = true;
}
/**
* Adds an item to the internal node at a given depth.
* @param item
* @param index
*/
export function addItemToInternalNode(node: InternalNode, item: Entry, index: number) {
let char = item.key[index];
// If an internal node is the proper site for item, it belongs under the
// corresponding (sentinel, internal-use) child node signifying this.
if(char == undefined) {
char = SENTINEL_CODE_UNIT;
}
if (!node.children[char]) {
node.children[char] = createRootNode();
node.values.push(char);
}
addUnsorted(node.children[char], item, index + 1);
}
export function addItemToLeaf(leaf: Leaf, item: Entry) {
leaf.entries.push(item);
}
/**
* Mutates the given Leaf to turn it into an InternalNode.
*
* NOTE: the node passed in will be DESTRUCTIVELY CHANGED into a different
* type when passed into this function!
*
* @param depth depth of the trie at this level.
*/
export function convertLeafToInternalNode(leaf: Leaf, depth: number): void {
let entries = leaf.entries;
// Alias the current node, as the desired type.
let internal = (<unknown> leaf) as InternalNode;
internal.type = 'internal';
delete (leaf as Partial<Leaf>).entries;
internal.values = [];
internal.children = {};
// Convert the old values array into the format for interior nodes.
for (let item of entries) {
let char: string;
if (depth < item.key.length) {
char = item.key[depth];
} else {
char = SENTINEL_CODE_UNIT;
}
if (!internal.children[char]) {
internal.children[char] = createRootNode();
internal.values.push(char);
}
addUnsorted(internal.children[char], item, depth + 1);
}
internal.unsorted = true;
}
/**
* Wrapper class for the trie and its nodes.
*/
export class TrieBuilder extends Trie {
/** The total weight of the entire trie. */
totalWeight: number;
constructor(toKey: Wordform2Key) {
super(createRootNode(), 0, toKey);
this.totalWeight = 0;
}
addEntry(word: string, weight?: number) {
weight = (isNaN(weight ?? NaN) || !weight) ? 1 : weight;
this.totalWeight += weight;
addUnsorted(
this.root, {
key: this.toKey(word),
content: word,
weight: weight
},
0
);
}
sort() {
// Sorts the full Trie, not just a part of it.
sortNode(this.root);
}
getRoot(): Node {
return this.root;
}
getTotalWeight(): number {
return this.totalWeight;
}
}

448
web/src/engine/predictive-text/templates/src/trie-model.ts
View file

@ -25,25 +25,15 @@
// Worth noting: we're starting to get quite a 'library' of common model/LMLayer functionality.
// Should probably make a 'lm-utils' submodule.
import { KMWString, PriorityQueue } from "@keymanapp/web-utils";
import { default as defaultWordBreaker } from "@keymanapp/models-wordbreakers";
import { applyTransform, isHighSurrogate, isSentinel, SENTINEL_CODE_UNIT, transformToSuggestion } from "./common.js";
import { getLastPreCaretToken } from "./tokenization.js";
import { LexicalModelTypes } from "@keymanapp/common-types";
import Capabilities = LexicalModelTypes.Capabilities;
import CasingFunction = LexicalModelTypes.CasingFunction;
import Configuration = LexicalModelTypes.Configuration;
import Context = LexicalModelTypes.Context;
import Distribution = LexicalModelTypes.Distribution;
import LexicalModel = LexicalModelTypes.LexicalModel;
import LexicalModelPunctuation = LexicalModelTypes.LexicalModelPunctuation;
import LexiconTraversal = LexicalModelTypes.LexiconTraversal;
import Suggestion = LexicalModelTypes.Suggestion;
import TextWithProbability = LexicalModelTypes.TextWithProbability;
import Transform = LexicalModelTypes.Transform;
import WithOutcome = LexicalModelTypes.WithOutcome;
import WordBreakingFunction = LexicalModelTypes.WordBreakingFunction;
import { SearchKey, Wordform2Key } from "./common.js";
import { Node, Trie, TrieTraversal } from './trie.js';
import { TraversalModel } from './traversal-model.js';
/**
* @file trie-model.ts
@ -51,9 +41,6 @@ import WordBreakingFunction = LexicalModelTypes.WordBreakingFunction;
* Defines a simple word list (unigram) model.
*/
/** Upper bound on the amount of suggestions to generate. */
const MAX_SUGGESTIONS = 12;
/**
* Additional arguments to pass into the model, in addition to the model
* parameters themselves.
@ -84,182 +71,6 @@ export interface TrieModelOptions {
punctuation?: LexicalModelPunctuation;
}
export class Traversal implements LexiconTraversal {
/**
* The lexical prefix corresponding to the current traversal state.
*/
readonly prefix: string;
/**
* The current traversal node. Serves as the 'root' of its own sub-Trie,
* and we cannot navigate back to its parent.
*/
readonly root: Node;
/**
* The max weight for the Trie being 'traversed'. Needed for probability
* calculations.
*/
readonly totalWeight: number;
constructor(root: Node, prefix: string, totalWeight: number) {
this.root = root;
this.prefix = prefix;
this.totalWeight = totalWeight;
}
child(char: string): Traversal | undefined {
// May result for blank tokens resulting immediately after whitespace.
if(char == '') {
return this;
}
// Split into individual code units.
let steps = char.split('');
let traversal: Traversal | undefined = this;
while(steps.length > 0 && traversal) {
const step: string = steps.shift()!;
traversal = traversal._child(step);
}
return traversal;
}
// Handles one code unit at a time.
private _child(char: string): Traversal | undefined {
const root = this.root;
const totalWeight = this.totalWeight;
const nextPrefix = this.prefix + char;
if(root.type == 'internal') {
let childNode = root.children[char];
if(!childNode) {
return undefined;
}
return new Traversal(childNode, nextPrefix, totalWeight);
} else {
// root.type == 'leaf';
const legalChildren = root.entries.filter(function(entry) {
return entry.key.indexOf(nextPrefix) == 0;
});
if(!legalChildren.length) {
return undefined;
}
return new Traversal(root, nextPrefix, totalWeight);
}
}
*children(): Generator<{char: string, traversal: () => Traversal}> {
let root = this.root;
// We refer to the field multiple times in this method, and it doesn't change.
// This also assists minification a bit, since we can't minify when re-accessing
// through `this.`.
const totalWeight = this.totalWeight;
if(root.type == 'internal') {
for(let entry of root.values) {
let entryNode = root.children[entry];
// UTF-16 astral plane check.
if(isHighSurrogate(entry)) {
// First code unit of a UTF-16 code point.
// For now, we'll just assume the second always completes such a char.
//
// Note: Things get nasty here if this is only sometimes true; in the future,
// we should compile-time enforce that this assumption is always true if possible.
if(entryNode.type == 'internal') {
let internalNode = entryNode;
for(let lowSurrogate of internalNode.values) {
let prefix = this.prefix + entry + lowSurrogate;
yield {
char: entry + lowSurrogate,
traversal: function() { return new Traversal(internalNode.children[lowSurrogate], prefix, totalWeight) }
}
}
} else {
// Determine how much of the 'leaf' entry has no Trie nodes, emulate them.
let fullText = entryNode.entries[0].key;
entry = entry + fullText[this.prefix.length + 1]; // The other half of the non-BMP char.
let prefix = this.prefix + entry;
yield {
char: entry,
traversal: function () {return new Traversal(entryNode, prefix, totalWeight)}
}
}
} else if(isSentinel(entry)) {
continue;
} else if(!entry) {
// Prevent any accidental 'null' or 'undefined' entries from having an effect.
continue;
} else {
let prefix = this.prefix + entry;
yield {
char: entry,
traversal: function() { return new Traversal(entryNode, prefix, totalWeight)}
}
}
}
return;
} else { // type == 'leaf'
let prefix = this.prefix;
let children = root.entries.filter(function(entry) {
return entry.key != prefix && prefix.length < entry.key.length;
})
for(let {key} of children) {
let nodeKey = key[prefix.length];
if(isHighSurrogate(nodeKey)) {
// Merge the other half of an SMP char in!
nodeKey = nodeKey + key[prefix.length+1];
}
yield {
char: nodeKey,
traversal: function() { return new Traversal(root, prefix + nodeKey, totalWeight)}
}
};
return;
}
}
get entries() {
const entryMapper = (value: Entry) => {
return {
text: value.content,
p: value.weight / this.totalWeight
}
}
if(this.root.type == 'leaf') {
let prefix = this.prefix;
let matches = this.root.entries.filter(function(entry) {
return entry.key == prefix;
});
return matches.map(entryMapper);
} else {
let matchingLeaf = this.root.children[SENTINEL_CODE_UNIT];
if(matchingLeaf && matchingLeaf.type == 'leaf') {
return matchingLeaf.entries.map(entryMapper);
} else {
return [];
}
}
}
get p(): number {
return this.root.weight / this.totalWeight;
}
}
/**
* @class TrieModel
*
@ -269,267 +80,30 @@ export class Traversal implements LexiconTraversal {
* prefix searches within words, however they are not very good
* at predicting the next word.
*/
export default class TrieModel implements LexicalModel {
export default class TrieModel extends TraversalModel implements LexicalModel {
configuration?: Configuration;
private _trie: Trie;
readonly breakWords: WordBreakingFunction;
public readonly trie: Trie;
readonly punctuation?: LexicalModelPunctuation;
readonly languageUsesCasing?: boolean;
readonly applyCasing?: CasingFunction;
constructor(trieData: {root: Node, totalWeight: number}, options: TrieModelOptions = {}) {
this.languageUsesCasing = options.languageUsesCasing;
this.applyCasing = options.applyCasing;
this._trie = new Trie(
const trie = new Trie(
trieData['root'],
trieData['totalWeight'],
options.searchTermToKey as Wordform2Key || defaultSearchTermToKey
);
this.breakWords = options.wordBreaker || defaultWordBreaker;
this.punctuation = options.punctuation;
super(trie.traverseFromRoot(), options);
this.trie = trie;
}
configure(capabilities: Capabilities): Configuration {
return this.configuration = {
leftContextCodePoints: capabilities.maxLeftContextCodePoints,
rightContextCodePoints: capabilities.maxRightContextCodePoints ?? 0
};
}
toKey(text: string): string {
return this._trie.toKey(text);
}
predict(transform: Transform, context: Context): Distribution<Suggestion> {
const makeDistribution = function(suggestions: WithOutcome<Suggestion>[]): Distribution<Suggestion> {
const distribution: Distribution<Suggestion> = [];
for (const s of suggestions) {
distribution.push({ sample: s, p: s.p });
}
return distribution;
}
// Special-case the empty buffer/transform: return the top suggestions.
if (!transform.insert && !context.left && !context.right && context.startOfBuffer && context.endOfBuffer) {
return makeDistribution(this._trie.firstN(MAX_SUGGESTIONS).map(({text, p}) => ({
transform: {
insert: text,
deleteLeft: 0
},
displayAs: text,
p: p
})));
}
// Compute the results of the keystroke:
const newContext = applyTransform(transform, context);
// Computes the different in word length after applying the transform above.
const leftDelOffset = transform.deleteLeft - KMWString.length(transform.insert);
// All text to the left of the cursor INCLUDING anything that has
// just been typed.
const prefix = getLastPreCaretToken(this.breakWords, newContext);
// Return suggestions from the trie.
return makeDistribution(this._trie.lookup(prefix).map(({text, p}) =>
transformToSuggestion({
insert: text,
// Delete whatever the prefix that the user wrote.
deleteLeft: leftDelOffset + KMWString.length(prefix)
// Note: a separate capitalization/orthography engine can take this
// result and transform it as needed.
},
p
)));
}
get wordbreaker(): WordBreakingFunction {
return this.breakWords;
}
public traverseFromRoot(): Traversal {
return this._trie.traverseFromRoot();
public traverseFromRoot(): TrieTraversal {
return super.traverseFromRoot() as TrieTraversal;
}
};
/////////////////////////////////////////////////////////////////////////////////
// What remains in this file is the trie implementation proper. Note: to //
// reduce bundle size, any functions/methods related to creating the trie have //
// been removed. //
/////////////////////////////////////////////////////////////////////////////////
/**
* An **opaque** type for a string that is exclusively used as a search key in
* the trie. There should be a function that converts arbitrary strings
* (queries) and converts them into a standard search key for a given language
* model.
*
* Fun fact: This opaque type has ALREADY saved my bacon and found a bug!
*/
type SearchKey = string & { _: 'SearchKey'};
/**
* The priority queue will always pop the most probable item - be it a Traversal
* state or a lexical entry reached via Traversal.
*/
type TraversableWithProb = TextWithProbability | LexiconTraversal;
/**
* A function that converts a string (word form or query) into a search key
* (secretly, this is also a string).
*/
interface Wordform2Key {
(wordform: string): SearchKey;
}
// The following trie implementation has been (heavily) derived from trie-ing
// by Conrad Irwin.
// trie-ing is copyright (C) 2015–2017 Conrad Irwin.
// Distributed under the terms of the MIT license:
// https://github.com/ConradIrwin/trie-ing/blob/df55d7af7068d357829db9e0a7faa8a38add1d1d/LICENSE
type Node = InternalNode | Leaf;
/**
* An internal node in the trie. Internal nodes NEVER contain entries; if an
* internal node should contain an entry, then it has a dummy leaf node (see
* below), that can be accessed by node.children["\uFDD0"].
*/
interface InternalNode {
type: 'internal';
weight: number;
/** Maintains the keys of children in descending order of weight. */
values: string[]; // TODO: As an optimization, "values" can be a single string!
/**
* Maps a single UTF-16 code unit to a child node in the trie. This child
* node may be a leaf or an internal node. The keys of this object are kept
* in sorted order in the .values array.
*/
children: { [codeunit: string]: Node };
}
/** Only leaf nodes actually contain entries (i.e., the words proper). */
interface Leaf {
type: 'leaf';
weight: number;
entries: Entry[];
}
/**
* An entry in the prefix trie (stored in leaf nodes exclusively!)
*/
interface Entry {
/** The actual word form, stored in the trie. */
content: string;
/** A search key that usually simplifies the word form, for ease of search. */
key: SearchKey;
weight: number;
}
/**
* Wrapper class for the trie and its nodes.
*/
class Trie {
public readonly root: Node;
/** The total weight of the entire trie. */
readonly totalWeight: number;
/**
* Converts arbitrary strings to a search key. The trie is built up of
* search keys; not each entry's word form!
*/
toKey: Wordform2Key;
constructor(root: Node, totalWeight: number, wordform2key: Wordform2Key) {
this.root = root;
this.toKey = wordform2key;
this.totalWeight = totalWeight;
}
public traverseFromRoot(): Traversal {
return new Traversal(this.root, '', this.totalWeight);
}
/**
* Lookups an arbitrary prefix (a query) in the trie. Returns the top 3
* results in sorted order.
*
* @param prefix
*/
lookup(prefix: string): TextWithProbability[] {
const searchKey = this.toKey(prefix);
const rootTraversal = this.traverseFromRoot().child(searchKey);
if(!rootTraversal) {
return [];
}
const directEntries = rootTraversal.entries;
// `Set` requires Chrome 38+, which is more recent than Chrome 35.
const directSet: Record<string, string> = {};
for(const entry of directEntries) {
directSet[entry.text] = entry.text;
}
const bestEntries = getSortedResults(rootTraversal);
const deduplicated = bestEntries.filter((entry) => !directSet[entry.text]);
// Any entries directly hosted on the current node should get full display
// priority over anything from its descendants.
return directEntries.concat(deduplicated);
}
/**
* Returns the top N suggestions from the trie.
* @param n How many suggestions, maximum, to return.
*/
firstN(n: number): TextWithProbability[] {
return getSortedResults(this.traverseFromRoot(), n);
}
}
/**
* Returns all entries matching the given prefix, in descending order of
* weight.
*
* @param prefix the prefix to match.
* @param results the current results
* @param queue
*/
function getSortedResults(traversal: LexiconTraversal, limit = MAX_SUGGESTIONS): TextWithProbability[] {
let queue = new PriorityQueue(function(a: TraversableWithProb, b: TraversableWithProb) {
// In case of Trie compilation issues that emit `null` or `undefined`
return (b ? b.p : 0) - (a ? a.p : 0);
});
let results: TextWithProbability[] = [];
queue.enqueue(traversal);
while(queue.count > 0) {
const entry = queue.dequeue();
if((entry as TextWithProbability)!.text !== undefined) {
const lexicalEntry = entry as TextWithProbability;
results.push(lexicalEntry);
if(results.length >= limit) {
return results;
}
} else {
const traversal = entry as LexiconTraversal;
queue.enqueueAll(traversal.entries);
let children: LexiconTraversal[] = []
for(let child of traversal.children()) {
children.push(child.traversal());
}
queue.enqueueAll(children);
}
}
return results;
}
/**
* Converts wordforms into an indexable form. It does this by
* normalizing into NFD, removing diacritics, and then converting

308
web/src/engine/predictive-text/templates/src/trie.ts Normal file
View file

@ -0,0 +1,308 @@
import { isHighSurrogate, isSentinel, SearchKey, SENTINEL_CODE_UNIT, Wordform2Key } from "./common.js";
import { LexicalModelTypes } from "@keymanapp/common-types";
import LexiconTraversal = LexicalModelTypes.LexiconTraversal;
// The following trie implementation has been (heavily) derived from trie-ing
// by Conrad Irwin.
// trie-ing is copyright (C) 2015–2017 Conrad Irwin.
// Distributed under the terms of the MIT license:
// https://github.com/ConradIrwin/trie-ing/blob/df55d7af7068d357829db9e0a7faa8a38add1d1d/LICENSE
export type Node = InternalNode | Leaf;
/**
* An internal node in the trie. Internal nodes NEVER contain entries; if an
* internal node should contain an entry, then it has a dummy leaf node (see
* below), that can be accessed by node.children["\uFDD0"].
*/
export interface InternalNode {
type: 'internal';
weight: number;
/** Maintains the keys of children in descending order of weight. */
values: string[]; // TODO: As an optimization, "values" can be a single string!
/**
* Maps a single UTF-16 code unit to a child node in the trie. This child
* node may be a leaf or an internal node. The keys of this object are kept
* in sorted order in the .values array.
*/
children: { [codeunit: string]: Node };
/**
* Used during compilation.
*/
unsorted?: boolean;
}
/** Only leaf nodes actually contain entries (i.e., the words proper). */
export interface Leaf {
type: 'leaf';
weight: number;
entries: Entry[];
/**
* Used during compilation.
*/
unsorted?: boolean;
}
/**
* An entry in the prefix trie (stored in leaf nodes exclusively!)
*/
export interface Entry {
/** The actual word form, stored in the trie. */
content: string;
/** A search key that usually simplifies the word form, for ease of search. */
key: SearchKey;
weight: number;
}
/**
* Recursively sort the trie, in descending order of weight.
* @param node any node in the trie
*/
export function sortNode(node: Node, onlyLocal?: boolean) {
if (node.type === 'leaf') {
if (!node.unsorted) {
return;
}
node.entries.sort(function (a, b) { return b.weight - a.weight; });
} else {
if(!onlyLocal) {
// We recurse and sort children before returning if sorting the full Trie.
for (let char of node.values) {
sortNode(node.children[char], onlyLocal);
}
}
if (!node.unsorted) {
return;
}
node.values.sort((a, b) => {
return node.children[b].weight - node.children[a].weight;
});
}
delete node.unsorted;
}
export class TrieTraversal implements LexiconTraversal {
/**
* The lexical prefix corresponding to the current traversal state.
*/
readonly prefix: string;
/**
* The current traversal node. Serves as the 'root' of its own sub-Trie,
* and we cannot navigate back to its parent.
*/
readonly root: Node;
/**
* The max weight for the Trie being 'traversed'. Needed for probability
* calculations.
*/
readonly totalWeight: number;
constructor(root: Node, prefix: string, totalWeight: number) {
this.root = root;
this.prefix = prefix;
this.totalWeight = totalWeight;
}
child(char: string): LexiconTraversal | undefined {
// May result for blank tokens resulting immediately after whitespace.
if(char == '') {
return this;
}
// Split into individual code units.
let steps = char.split('');
let traversal: ReturnType<TrieTraversal["_child"]> = this;
while(steps.length > 0 && traversal) {
const step: string = steps.shift()!;
traversal = traversal._child(step);
}
return traversal;
}
// Handles one code unit at a time.
private _child(char: string): TrieTraversal | undefined {
const root = this.root;
const totalWeight = this.totalWeight;
const nextPrefix = this.prefix + char;
// Sorts _just_ the current level, and only if needed.
// We only care about sorting parts that we're actually accessing.
sortNode(root, true);
if(root.type == 'internal') {
let childNode = root.children[char];
if(!childNode) {
return undefined;
}
return new TrieTraversal(childNode, nextPrefix, totalWeight);
} else {
// root.type == 'leaf';
const legalChildren = root.entries.filter(function(entry) {
return entry.key.indexOf(nextPrefix) == 0;
});
if(!legalChildren.length) {
return undefined;
}
return new TrieTraversal(root, nextPrefix, totalWeight);
}
}
children(): {char: string, p: number, traversal: () => LexiconTraversal}[] {
let root = this.root;
// We refer to the field multiple times in this method, and it doesn't change.
// This also assists minification a bit, since we can't minify when re-accessing
// through `this.`.
const totalWeight = this.totalWeight;
const results: {char: string, p: number, traversal: () => LexiconTraversal}[] = [];
// Sorts _just_ the current level, and only if needed.
// We only care about sorting parts that we're actually accessing.
sortNode(root, true);
if(root.type == 'internal') {
for(let entry of root.values) {
let entryNode = root.children[entry];
// UTF-16 astral plane check.
if(isHighSurrogate(entry)) {
// First code unit of a UTF-16 code point.
// For now, we'll just assume the second always completes such a char.
//
// Note: Things get nasty here if this is only sometimes true; in the future,
// we should compile-time enforce that this assumption is always true if possible.
if(entryNode.type == 'internal') {
let internalNode = entryNode;
for(let lowSurrogate of internalNode.values) {
let prefix = this.prefix + entry + lowSurrogate;
results.push({
char: entry + lowSurrogate,
p: internalNode.children[lowSurrogate].weight / totalWeight,
traversal: function() { return new TrieTraversal(internalNode.children[lowSurrogate], prefix, totalWeight) }
});
}
} else {
// No deduplication needed. If there were multiple keys available,
// we wouldn't be in a leaf node.
let fullText = entryNode.entries[0].key;
entry = entry + fullText[this.prefix.length + 1]; // The other half of the non-BMP char.
let prefix = this.prefix + entry;
const weight = entryNode.entries.reduce((best, current) => Math.max(best, current.weight), 0);
results.push({
char: entry,
p: weight / totalWeight,
traversal: function () {return new TrieTraversal(entryNode, prefix, totalWeight)}
});
}
} else if(isSentinel(entry)) {
continue;
} else if(!entry) {
// Prevent any accidental 'null' or 'undefined' entries from having an effect.
continue;
} else {
let prefix = this.prefix + entry;
results.push({
char: entry,
p: entryNode.weight / totalWeight,
traversal: function() { return new TrieTraversal(entryNode, prefix, totalWeight)}
});
}
}
return results;
} else { // type == 'leaf'
let prefix = this.prefix;
// No actual deduplication needed. If there were multiple keys available,
// we wouldn't be in a leaf node.
let children = root.entries.filter(function(entry) {
return entry.key != prefix && prefix.length < entry.key.length;
});
let weight = children.reduce((best, current) => Math.max(current.weight, best), 0);
if(children.length > 0) {
const key = children[0].content;
let nodeKey = key[prefix.length];
if(isHighSurrogate(nodeKey)) {
// Merge the other half of an SMP char in!
nodeKey = nodeKey + key[prefix.length+1];
}
results.push({
char: nodeKey,
p: weight / totalWeight,
traversal: function() { return new TrieTraversal(root, prefix + nodeKey, totalWeight)}
});
};
return results;
}
}
get entries() {
const entryMapper = (value: Entry) => {
return {
text: value.content,
p: value.weight / this.totalWeight
}
}
if(this.root.type == 'leaf') {
let prefix = this.prefix;
let matches = this.root.entries.filter(function(entry) {
return entry.key == prefix;
});
return matches.map(entryMapper);
} else {
let matchingLeaf = this.root.children[SENTINEL_CODE_UNIT];
if(matchingLeaf && matchingLeaf.type == 'leaf') {
return matchingLeaf.entries.map(entryMapper);
} else {
return [];
}
}
}
get p(): number {
return this.root.weight / this.totalWeight;
}
}
/**
* Wrapper class for the trie and its nodes.
*/
export class Trie {
protected root: Node;
/** The total weight of the entire trie. */
readonly totalWeight: number;
/**
* Converts arbitrary strings to a search key. The trie is built up of
* search keys; not each entry's word form!
*/
toKey: Wordform2Key;
constructor(root: Node, totalWeight: number, wordform2key: Wordform2Key) {
this.root = root;
this.toKey = wordform2key;
this.totalWeight = totalWeight;
}
public traverseFromRoot(): LexiconTraversal {
return new TrieTraversal(this.root, '', this.totalWeight);
}
}

20
web/src/engine/predictive-text/worker-main/README.md
View file

@ -17,11 +17,13 @@ Build
Run `build.sh`. This will also automatically install dependencies with `npm`.
```sh
./build.sh
./build.sh configure build
```
### Two-stage compilation process
TODO: this is well and truly out of date...
Since the primary LMLayer code runs within a [Web Worker][], `build.sh` compiles the
LMLayer in two stages:
@ -43,19 +45,7 @@ This will run both headless unit tests, and in-browser unit tests and integratio
tests:
```sh
./build.sh -test
./build.sh test
```
### Test-Driven Development
I like to use [entr]() to automatically build and re-run the unit tests anytime I
change a source code file. Here's the command I run in separate window:
```sh
git ls-files | entr -c ./build.sh -tdd
```
Importantly, `./build.sh -tdd` skips running the in-browser tests, and skips
downloading/updating `npm` dependencies.
[entr]: http://eradman.com/entrproject/
### TODO: epic/user-dict

262
web/src/engine/predictive-text/worker-thread/src/main/models/composited-traversal.ts Normal file
View file

@ -0,0 +1,262 @@
import { LexicalModelTypes } from "@keymanapp/common-types";
import Distribution = LexicalModelTypes.Distribution;
import LexiconTraversal = LexicalModelTypes.LexiconTraversal;
import TextWithProbability = LexicalModelTypes.TextWithProbability;
import { PriorityQueue } from "@keymanapp/web-utils";
type ChildEdge = { char: string; p: number, traversal: () => LexiconTraversal };
export class CompositedTraversal implements LexiconTraversal {
private readonly componentTraversals: Distribution<LexiconTraversal>;
// Used both internally and externally - and we don't want to re-weight internal calls!
constructor(traversalsWithWeighting: Distribution<LexiconTraversal>) {
// let totalP = traversalsWithWeighting.reduce((accum, current) => accum + current.p, 0);
this.componentTraversals = traversalsWithWeighting.map((entry) => {
// Shallow-copy the traversals while normalizing the associated weightings.
// The sum of all `p` values should sum to 1.
return {
sample: entry.sample,
p: entry.p // totalP
};
});
}
children(): ChildEdge[] {
// Easy case: when there's no source on the current Traversal path, there
// are no children.
//
// Though... we shouldn't even reach this point; if a child path lacks
// support; `undefined` should have been returned instead of a class
// instance.
/* c8 ignore start */
if(this.componentTraversals.length == 0) {
return [];
}
/* c8 ignore stop */
// If there were no duplicated children, merging the arrays returned from the
// following statement, as is, would be the correct answer.
const unblendedComponents = this.componentTraversals.map((component) => {
const weight = component.p;
return {
sample: component.sample.children().map((entry) => {
return {
char: entry.char,
p: entry.p,
traversal: () => entry.traversal()
};
}),
// Make sure to track the model-source's weighting!
weight: weight
};
});
// If there's only a single active component, well, there actually are
// no duplicates - thus it is the right answer.
if(unblendedComponents.length == 1) {
const singleSource = unblendedComponents[0];
return singleSource.sample.map((child) => {
return {
char: child.char,
p: singleSource.weight * child.p,
traversal: () => new CompositedTraversal([{ sample: child.traversal(), p: singleSource.weight }])
};
});
}
// Now, the tricky part: actual blending.
// Find sets of traversals that traverse the same char path after the represented prefix.
const componentBuckets: Record<string, { component: ChildEdge, weight: number }[]> = {};
unblendedComponents.forEach((componentChildren) => {
componentChildren.sample.forEach((entry) => {
componentBuckets[entry.char] ??= [];
componentBuckets[entry.char].push({
component: entry,
weight: componentChildren.weight
});
});
});
// For each char path...
const keys = Object.keys(componentBuckets);
return keys.map((key) => {
const bucket = componentBuckets[key];
// Build the blended traversal for that path
const traversalFunc = () => {
const traversalsWithWeighting = bucket.map((entry) => {
return {
sample: entry.component.traversal(),
// Apply the model's weighting!
// (This is not the `p` for that model's traversal path.)
p: entry.weight
};
});
return new CompositedTraversal(traversalsWithWeighting);
};
// If there's only one contributing model, there's no need to worry about duplicates
// affecting the "highest probability" value for the path; no blending needed.
//
// It's thus safe to just return the model's probability for the path.
if(bucket.length == 1) {
return {
char: key,
p: bucket[0].weight * bucket[0].component.p,
traversal: traversalFunc
};
} else {
// Here, there's a non-zero chance that duplicate entries may exist.
// We thus need to worry about the potential for blending...
// which means we're best off delegating "highest probability" calculation
// to the child, which is the level at which the issue may be resolved.
const traversal = traversalFunc();
return {
char: key,
// Use the traversal early to ensure duplicate entries aren't improperly
// neglected due to their probability sources being split.
p: traversal.p,
// We've actually built it already, so just re-use the instance.
traversal: () => traversal
};
}
});
}
child(char: string): LexiconTraversal {
let children = this.componentTraversals.map((traversalTuple) => {
return {
sample: traversalTuple.sample.child(char),
p: traversalTuple.p
};
});
children = children.filter((entry) => !!entry.sample);
if(children.length == 0) {
return undefined;
} else {
// Will sort out blending on its own.
return new CompositedTraversal(children);
}
}
get entries(): TextWithProbability[] {
// If there were guaranteed to be no duplicates, .flatMap instead of .map would be fine here.
const entrySets = this.componentTraversals.map((traversalTuple) => {
return traversalTuple.sample.entries.map((entry) => {
const reweightedEntry: TextWithProbability = {
text: entry.text,
p: entry.p * traversalTuple.p
};
return reweightedEntry;
});
});
// Combine any duplicated entries.
const resultMap: Record<string, TextWithProbability> = {};
entrySets.forEach((set) => {
set.forEach((entry) => {
if(resultMap[entry.text]) {
resultMap[entry.text].p += entry.p;
} else {
resultMap[entry.text] = entry;
}
});
});
// Finalize as array + sort by descending probability
const results = Object.values(resultMap);
results.sort((a, b) => b.p - a.p);
return results;
}
get p(): number {
// If it is impossible for a blended traversal path to yield a higher-
// probability entry than the best unblended result, there's no point
// in considering it here.
const bestUnblendedProb = this.componentTraversals
// to 'traversal weighting' * 'traversal node's max probability'
.map((entry) => entry.p * entry.sample.p)
// find the best entry post-weighting
.reduce((best, current) => (current > best) ? current : best, 0);
// Easy case: 1 component at this level? No blending is possible,
// so just return the best entry found this way.
if(this.componentTraversals.length == 1) {
return bestUnblendedProb;
}
// Check for traversal children that travel the same path. O(sum of child
// nodes across all components)
//
// If we wanted to try optimizing this...
// - priority queue for each componentTraversal, descending prob order
// - track which sources have contributed while accumulating path probs
// - if best entry from sources not contributed + current accumulation can't
// win, we can stop blending early.
// - further iterations can no longer exceed bestUnblendedProb.
//
// This likely avoids iterating over all paths... but invoking priority
// queues means worst-case O(sum N_i log N_i), which is worse. (With N_i =
// path count per component traversal)
// - constructable "worst case": fully balanced pMaxes?
// - for "good cases"... would the complexity even be worth it?
// - suppose 3 sources + English: worst-case iteration count with current
// setup is 4 * 26 = 104. That's not "terrible"; fairly low to be
// considering big-O optimizations.
// - +1 multiplier due to the following `filter` block.
const childBuckets: Record<string, { p?: number, pMax: number, char: string }> = {};
const entryBuckets: Record<string, number> = {};
this.componentTraversals.forEach((entry) => {
const weight = entry.p;
const component = entry.sample;
const entries = component.entries;
component.children().forEach((child) => {
const record = childBuckets[child.char] ??= { pMax: 0, char: child.char };
record.pMax += weight * child.p;
});
entries.forEach((entry) => entryBuckets[entry.text] = (entryBuckets[entry.text] ?? 0) + entry.p * weight);
});
// If a path's maximum potential probability does not exceed the best entry,
// it can't affect our answer; skip it.
// Is O(alphabet count)
const potentialBlendedChildren = Object.keys(childBuckets).map((char) => {
const record = childBuckets[char];
if(record.pMax <= bestUnblendedProb) {
return null;
} else {
return record;
}
}).filter((record) => !!record);
let bestBlendedEntry = Object.values(entryBuckets).reduce((best, current) => Math.max(best, current), 0);
// Prep the path-search for a potential winner.
let bestProb = Math.max(bestUnblendedProb, bestBlendedEntry);
// May have zero entries if no blended path can exceed the best non-blended path's probability.
// (Assumption: this is the common case.)
const queue = new PriorityQueue((a, b) => b.pMax - a.pMax, potentialBlendedChildren);
while(queue.peek()?.pMax > bestProb) {
const entry = queue.dequeue();
const childPath = this.child(entry.char);
// Note: this pattern, as-is, does not allow cross-node optimizations
// (such as seeing a better potential here once we've started evaluating
// potentials within the child)
//
// It's a reasonable "first pass" though.
bestProb = Math.max(childPath.p, bestProb);
}
return bestProb;
}
}

1
web/src/engine/predictive-text/worker-thread/src/main/models/index.ts
View file

@ -1,2 +1,3 @@
export * from '@keymanapp/models-templates';
export { CompositedTraversal } from './composited-traversal.js';
export { default as DummyModel, DummyOptions } from './dummy-model.js';

264
web/src/test/auto/headless/engine/predictive-text/templates/trie-builder.tests.ts Normal file
View file

@ -0,0 +1,264 @@
/*
* Unit tests for the Trie prediction model.
*/
import { assert } from 'chai';
import { InternalNode, Leaf, SearchKey, SENTINEL_CODE_UNIT, trieConstruction, TrieBuilder } from '@keymanapp/models-templates';
describe('trie construction', () => {
it('default root node', () => {
const defaultRoot = trieConstruction.createRootNode() as Leaf;
assert.equal(defaultRoot.type, 'leaf');
assert.equal(defaultRoot.weight, 0);
assert.sameMembers(defaultRoot.entries, []);
});
it('addItemToLeaf', () => {
// `weight` is managed by addUnsorted, not by addItemToLeaf.
// As a result, we don't test for it here.
const defaultRoot = trieConstruction.createRootNode() as Leaf;
trieConstruction.addItemToLeaf(defaultRoot, {
content: 'cafe',
weight: 2,
key: 'cafe' as SearchKey
});
assert.equal(defaultRoot.type, 'leaf');
assert.equal(defaultRoot.entries.length, 1);
trieConstruction.addItemToLeaf(defaultRoot, {
content: 'café',
weight: 1,
key: 'cafe' as SearchKey
});
assert.equal(defaultRoot.type, 'leaf');
assert.equal(defaultRoot.entries.length, 2);
});
it('convertLeafToInternalNode', () => {
const defaultRoot = trieConstruction.createRootNode() as InternalNode;
const cafeEntry = {
content: 'cafe',
weight: 5,
key: 'cafe' as SearchKey
};
trieConstruction.addItemToLeaf(defaultRoot as unknown as Leaf, cafeEntry);
defaultRoot.weight = 5;
trieConstruction.convertLeafToInternalNode(defaultRoot as unknown as Leaf, 4);
assert.equal(defaultRoot.type, 'internal');
assert.sameMembers(defaultRoot.values, [SENTINEL_CODE_UNIT]);
assert.sameMembers(Object.keys(defaultRoot.children), defaultRoot.values);
assert.equal(defaultRoot.weight, 5);
const sentinelChild = defaultRoot.children[SENTINEL_CODE_UNIT] as Leaf;
assert.equal(sentinelChild.type, 'leaf');
assert.equal(sentinelChild.weight, 5);
assert.equal(sentinelChild.entries.length, 1);
// Should be strict-equal too, but we only really care if it's deep-equal.
assert.deepEqual(sentinelChild.entries[0], cafeEntry);
});
it('addUnsorted - simple initial entry', () => {
// `weight` is managed by addUnsorted, not by addItemToLeaf.
// As a result, we don't test for it here.
const defaultRoot = trieConstruction.createRootNode() as Leaf;
trieConstruction.addUnsorted(defaultRoot, {
content: 'cafe',
weight: 2,
key: 'cafe' as SearchKey
}, 4);
assert.equal(defaultRoot.type, 'leaf');
assert.equal(defaultRoot.weight, 2);
assert.equal(defaultRoot.entries.length, 1);
// smaller weight, will not override. Is not additive.
trieConstruction.addUnsorted(defaultRoot, {
content: 'café',
weight: 1,
key: 'cafe' as SearchKey
}, 4);
assert.equal(defaultRoot.type, 'leaf');
assert.equal(defaultRoot.weight, 2);
assert.equal(defaultRoot.entries.length, 2);
});
it('addUnsorted - short word, then longer word with same prefix', () => {
const defaultRoot = trieConstruction.createRootNode();
trieConstruction.addUnsorted(defaultRoot, {
content: 'cafe',
weight: 5,
key: 'cafe' as SearchKey
}, 4);
const rootAsLeaf = defaultRoot as Leaf;
assert.equal(rootAsLeaf.type, 'leaf');
assert.equal(rootAsLeaf.weight, 5);
assert.equal(rootAsLeaf.entries.length, 1);
// smaller weight, will not override. Is not additive.
trieConstruction.addUnsorted(defaultRoot, {
content: 'cafeteria',
weight: 3,
key: 'cafeteria' as SearchKey
}, 4);
const rootAsInternal = defaultRoot as InternalNode;
assert.equal(rootAsInternal.type, 'internal');
assert.equal(rootAsInternal.weight, 5);
assert.deepEqual(rootAsInternal.values, [SENTINEL_CODE_UNIT, 't']);
assert.equal((rootAsInternal.children[SENTINEL_CODE_UNIT] as Leaf).weight, 5);
assert.equal((rootAsInternal.children['t'] as Leaf).weight, 3);
});
it('addUnsorted', () => {
// Perspective: current root is actually representing `ca`.
const defaultRoot = trieConstruction.createRootNode();
trieConstruction.addUnsorted(defaultRoot, {
content: 'cafe',
weight: 5,
key: 'cafe' as SearchKey
}, 2);
// There's only one child, so there's no reason to start making
// internal structure just yet.
assert.equal(defaultRoot.type, 'leaf');
assert.equal(defaultRoot.weight, 5);
// Adds a second child; that child has more than one letter not-in-common with the first.
trieConstruction.addUnsorted(defaultRoot, {
content: 'call',
weight: 8,
key: 'call' as SearchKey
}, 2);
const rootAsInternal = defaultRoot as InternalNode;
assert.equal(rootAsInternal.type, 'internal');
assert.equal(rootAsInternal.weight, 8);
assert.sameMembers(rootAsInternal.values, ['f', 'l']);
assert.equal((rootAsInternal.children['f'] as Leaf).weight, 5);
assert.equal((rootAsInternal.children['l'] as Leaf).weight, 8);
});
it('addUnsorted - lower frequency prefix', () => {
// Will correspond to 'thin'.
const root = trieConstruction.createRootNode();
trieConstruction.addUnsorted(root, {
key: 'think' as SearchKey,
content: 'think',
weight: 100
}, 4);
trieConstruction.addUnsorted(root, {
key: 'thing' as SearchKey,
content: 'thing',
weight: 40
}, 4);
const rootAsInternal = root as InternalNode;
assert.equal(rootAsInternal.type, 'internal');
assert.sameMembers(rootAsInternal.values, ['k', 'g']);
// Interesting noted behavior: if just 'think' into 'thin', and nothing else...
// it remains a leaf! That doesn't seem... entirely proper. With 'thing', it
// will at least be an 'internal' node already.
trieConstruction.addUnsorted(rootAsInternal, {
key: 'thin' as SearchKey,
content: 'thin',
weight: 20
}, 4);
assert.sameMembers(rootAsInternal.values, ['k', 'g', SENTINEL_CODE_UNIT]);
assert.isOk(rootAsInternal.children[SENTINEL_CODE_UNIT]);
assert.equal((rootAsInternal.children[SENTINEL_CODE_UNIT] as Leaf).weight, 20);
});
describe('TrieBuilder', () => {
it('standard Trie construction', () => {
const builder = new TrieBuilder((text) => text as SearchKey);
builder.addEntry('caffeine', 2);
builder.addEntry('cafe', 5);
builder.addEntry('calm', 3);
builder.addEntry('calf', 4);
builder.addEntry('call', 6); // total: 20
builder.addEntry('can', 10); // total: 30
builder.addEntry('and', 20); // total: 50
assert.equal(builder.getTotalWeight(), 50);
const root = builder.getRoot() as InternalNode;
const aNode = (root.children['c'] as InternalNode).children['a'] as InternalNode;
// As the nodes were not added in sorted order, they should not currently be ordered.
assert.sameMembers(aNode.values, ['n', 'l', 'f']);
assert.notSameOrderedMembers(aNode.values, ['n', 'l', 'f']);
const lNode = aNode.children['l'] as InternalNode;
assert.sameMembers(lNode.values, ['l', 'f', 'm']);
assert.notSameOrderedMembers(lNode.values, ['l', 'f', 'm']);
builder.sort();
assert.sameOrderedMembers(aNode.values, ['n', 'l', 'f']);
assert.sameOrderedMembers(lNode.values, ['l', 'f', 'm']);
});
// In case of high startup-time for user-dictionary processing, this could allow use
// of a 'partially' processed user wordlist. (We'd prioritize predicting when the
// user is interacting with text, then resume processing once 'idle'.)
it('interspersed construction + lookup', () => {
const builder = new TrieBuilder((text) => text as SearchKey);
builder.addEntry('caffeine', 2);
builder.addEntry('cafe', 5);
builder.addEntry('calm', 3);
builder.addEntry('calf', 4);
builder.addEntry('call', 6); // total: 20
// ------------------------------------------
// Pause construction; actually use the Trie.
assert.equal(builder.getTotalWeight(), 20);
const root = builder.getRoot() as InternalNode;
const caNode = (root.children['c'] as InternalNode).children['a'] as InternalNode;
// As the nodes were not added in sorted order, they should not currently be ordered.
assert.sameMembers(caNode.values, ['l', 'f']);
assert.notSameOrderedMembers(caNode.values, ['l', 'f']);
const cal = builder.traverseFromRoot().child('cal');
// Actually traversing through the node should auto-sort the entries.
assert.sameOrderedMembers(caNode.values, ['l', 'f']);
// Including the reached node's children.
assert.sameOrderedMembers([...cal.children()].map((entry) => entry.char), ['l', 'f', 'm']);
const cafNode = caNode.children['f'] as InternalNode;
// 'caffeine' was added before 'cafe'.
// Parts not 'traversed' should not be unnecessarily sorted.
assert.notSameOrderedMembers(cafNode.values, ['e', 'f']);
// -------------------
// Resume construction
builder.addEntry('can', 10); // total: 30
builder.addEntry('and', 20); // total: 50
assert.equal(builder.getTotalWeight(), 50);
// As the nodes were not added in sorted order, they should not currently be ordered.
assert.sameMembers(caNode.values, ['n', 'l', 'f']);
// 'n' was added later, thus will be out of sorted order.
assert.notSameOrderedMembers(caNode.values, ['n', 'l', 'f']);
builder.sort();
assert.sameOrderedMembers(caNode.values, ['n', 'l', 'f']);
});
});
});

16
web/src/test/auto/headless/engine/predictive-text/templates/trie-traversal.tests.ts
View file

@ -175,15 +175,13 @@ describe('Trie traversal abstractions', function() {
do {
assert.isNotEmpty(leafChildSequence);
let iter = curChild.traversal().children();
let curr = iter.next();
curChild = curr.value;
let children = curChild.traversal().children();
assert.equal(children.length, 1);
curChild = children[0];
// Test generator behavior - there should be one child, then the 'done' state.
assert.isDefined(curChild);
assert.equal(curChild.char, leafChildSequence[0]);
curr = iter.next();
assert.isTrue(curr.done);
// Prepare for iteration.
leafChildSequence.shift();
@ -276,15 +274,13 @@ describe('Trie traversal abstractions', function() {
do {
assert.isNotEmpty(leafChildSequence);
let iter = curChild.traversal().children();
let curr = iter.next();
curChild = curr.value;
let children = curChild.traversal().children();
assert.equal(children.length, 1);
curChild = children[0];
// Test generator behavior - there should be one child, then the 'done' state.
assert.isDefined(curChild);
assert.equal(curChild.char, leafChildSequence[0]);
curr = iter.next();
assert.isTrue(curr.done);
// Prepare for iteration.
leafChildSequence.shift();

174
web/src/test/auto/headless/engine/predictive-text/worker-thread/composited-traversal.ts Normal file
View file

@ -0,0 +1,174 @@
import { assert } from 'chai';
import { SearchKey, TrieBuilder } from '@keymanapp/models-templates';
import { models } from '@keymanapp/lm-worker/test-index';
import CompositedTraversal = models.CompositedTraversal;
const simpleLexicalModel = (function() {
const builder = new TrieBuilder((str) => str as SearchKey);
// Sum total: 10.
builder.addEntry('are', 1);
builder.addEntry('and', 2);
builder.addEntry('not', 1);
builder.addEntry('the', 3);
builder.addEntry('try', 1);
builder.addEntry('to', 1);
builder.addEntry('tom', 1);
return builder;
}());
const namesModel = (function() {
const builder = new TrieBuilder((str) => str as SearchKey);
builder.addEntry('amy', 1);
builder.addEntry('bob', 1);
builder.addEntry('mark', 1);
builder.addEntry('tom', 1);
return builder;
}());
const blendedModel = new CompositedTraversal([
{ sample: simpleLexicalModel.traverseFromRoot(), p: 0.5 },
{ sample: namesModel.traverseFromRoot(), p: 0.5 }
]);
describe('CompositedTraversal', () => {
describe('direct prefix lookup', () => {
it('returns undefined for non-existent paths', () => {
assert.isUndefined(blendedModel.child('tomcat'));
assert.isUndefined(blendedModel.child('tone'));
assert.isUndefined(blendedModel.child('ton'));
assert.isUndefined(blendedModel.child('apple'));
assert.isUndefined(blendedModel.child('aim'));
assert.isUndefined(blendedModel.child('xylophone'));
});
it('reweighs entries found in only one source', () => {
const andNode = blendedModel.child('and');
assert.equal(andNode.p, 0.1);
assert.equal(andNode.entries.length, 1);
const and = andNode.entries[0];
assert.equal(and.text, 'and');
assert.equal(and.p, 0.1);
});
it('merges entries found in multiple sources, combining their probability and entries', () => {
const tomNode = blendedModel.child('tom');
assert.equal(tomNode.p, 0.175);
assert.equal(tomNode.entries.length, 1);
const tom = tomNode.entries[0];
assert.equal(tom.text, 'tom');
assert.equal(tom.p, 0.175);
});
it('correctly returns the probability of intermediate child paths', () => {
// simpleLexical: best is 'and', 0.2
// names: best is 'amy', 0.25
// Weight of each is equal at 0.5; 0.25 * 0.5 = 0.125.
// Should not combine.
const aNode = blendedModel.child('a');
assert.equal(aNode.p, 0.125, "probability of paths were merged despite lack of duplicate");
assert.equal(aNode.entries.length, 0);
// 'tom' is on both paths, and is highest-frequency entry when merged.
// We want the combined, net probability to show up on its predecessor paths.
const toNode = blendedModel.child('to');
assert.equal(toNode.p, 0.175, "probability of path with winning duplicate not merged");
// simpleLexical - p('to') = 0.1; is then multiplied by model weight of 0.5.
assert.equal(toNode.entries.length, 1);
const to = toNode.entries[0];
assert.equal(to.text, 'to');
assert.equal(to.p, 0.05);
});
});
describe('iteration for correction-search', () => {
it('correctly enumerates child paths from root', () => {
const topChildren = blendedModel.children();
// 'n' from "simpleLexical", 'b' and 'm' from "names"
assert.sameMembers(topChildren.map((child) => child.char), ['a', 'b', 'm', 'n', 't']);
assert.sameDeepMembers(topChildren.map((child) => {
return {
char: child.char,
p: child.p
}
}), [
{ char: 'a', p: .125 }, // highest: 'amy'
{ char: 'b', p: .125 }, // highest: 'bob'
{ char: 'm', p: .125 }, // highest: 'mark'
{ char: 'n', p: .05 }, // highest: 'not'
{ char: 't', p: .175 } // highest: 'tom'
]);
topChildren.forEach((entry) => assert.equal(entry.p, entry.traversal().p));
});
it('correctly enumerates child paths from section supported by a single member', () => {
const topChildren = blendedModel.children();
const bNode = topChildren.find((entry) => entry.char == 'b').traversal();
const bChildren = bNode.children();
assert.sameDeepMembers(bChildren.map((child) => {
return {
char: child.char,
p: child.p
};
}), [
{ char: 'o', p: 0.125 }
]);
bChildren.forEach((entry) => assert.equal(entry.p, entry.traversal().p));
const boNode = bChildren.find((entry) => entry.char == 'o').traversal();
const boChildren = boNode.children();
assert.sameDeepMembers(boChildren.map((child) => {
return {
char: child.char,
p: child.p
};
}), [
{ char: 'b', p: 0.125 }
]);
boChildren.forEach((entry) => assert.equal(entry.p, entry.traversal().p));
});
it('correctly enumerates child paths from section supported by multiple members', () => {
const topChildren = blendedModel.children();
const tNode = topChildren.find((entry) => entry.char == 't').traversal();
const tChildren = tNode.children();
assert.sameMembers(tChildren.map((child) => child.char), ['h', 'o', 'r']);
assert.sameDeepMembers(tChildren.map((child) => {
return {
char: child.char,
p: child.p
}
}), [
{ char: 'h', p: 0.15 },
{ char: 'o', p: 0.175 },
{ char: 'r', p: 0.05 }
]);
tChildren.forEach((entry) => assert.equal(entry.p, entry.traversal().p));
const toNode = tChildren.find((entry) => entry.char == 'o').traversal();
const toChildren = toNode.children();
assert.sameMembers(toChildren.map((child) => child.char), ['m']);
assert.sameDeepMembers(toChildren.map((child) => {
return {
char: child.char,
p: child.p
};
}), [
{ char: 'm', p: 0.175 }
]);
toChildren.forEach((entry) => assert.equal(entry.p, entry.traversal().p));
});
});
});