erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-08-08 09:55:32 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

Merge branch 'epic/model-encoding' into auto/A19S29-merge-master-into-model-encoding

Browse source
This commit is contained in:
Keyman Server 2026-05-22 07:23:01 +02:00 • committed by GitHub
commit d3fc9aebf3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 1448 additions and 607 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
developer/src/kmc-model/build.sh
View file

@ -12,7 +12,7 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")"
builder_describe "Keyman kmc Lexical Model Compiler module" \
"@/common/web/keyman-version" \
"@/developer/src/common/web/test-helpers" \
"@/web/src/engine/predictive-text/templates/ test" \
"@/web/src/engine/predictive-text/templates/" \
"clean" \
"configure" \
"build" \

2
developer/src/kmc-model/package.json
View file

@ -32,11 +32,13 @@
"dependencies": {
"@keymanapp/common-types": "*",
"@keymanapp/keyman-version": "*",
"@keymanapp/models-templates": "*",
"typescript": "^5.4.5"
},
"devDependencies": {
"@keymanapp/developer-test-helpers": "*",
"@keymanapp/models-templates": "*",
"@types/node": "^20.4.1",
"c8": "^7.12.0",
"chalk": "^2.4.2",
"esbuild": "^0.25.0"

341
developer/src/kmc-model/src/build-trie.ts
View file

@ -1,6 +1,8 @@
import { ModelCompilerError, ModelCompilerMessageContext, ModelCompilerMessages } from "./model-compiler-messages.js";
import { callbacks } from "./compiler-callbacks.js";
import { TrieBuilder } from '@keymanapp/models-templates';
// Supports LF or CRLF line terminators.
const NEWLINE_SEPARATOR = /\u000d?\u000a/;
@ -29,8 +31,11 @@ export function createTrieDataStructure(filenames: string[], searchTermToKey?: (
const wordlist: WordList = {};
filenames.forEach(filename => parseWordListFromFilename(wordlist, filename));
const trie = Trie.buildTrie(wordlist, searchTermToKey as Trie.SearchTermToKey);
return JSON.stringify(trie);
const trie = buildTrie(wordlist, searchTermToKey as SearchTermToKey);
return JSON.stringify({
data: trie.compress(),
totalWeight: trie.getTotalWeight()
});
}
/**
@ -188,302 +193,46 @@ function* enumerateLines(lines: string[]): Generator<LineNoAndText> {
}
}
namespace Trie {
/**
* An **opaque** type for a string that is exclusively used as a search key in
* the trie. There should be a function that converts arbitrary strings
* (queries) and converts them into a standard search key for a given language
* model.
*
* Fun fact: This opaque type has ALREADY saved my bacon and found a bug!
*/
type SearchKey = string & { _: 'SearchKey'};
/**
* An **opaque** type for a string that is exclusively used as a search key in
* the trie. There should be a function that converts arbitrary strings
* (queries) and converts them into a standard search key for a given language
* model.
*/
type SearchKey = string & { _: 'SearchKey'};
/**
* A function that converts a string (word form or query) into a search key
* (secretly, this is also a string).
*/
export interface SearchTermToKey {
(wordform: string): SearchKey;
}
// The following trie implementation has been (heavily) derived from trie-ing
// by Conrad Irwin.
//
// trie-ing is distributed under the terms of the MIT license, reproduced here:
//
// The MIT License
// Copyright (c) 2015-2017 Conrad Irwin <conrad.irwin@gmail.com>
// Copyright (c) 2011 Marc Campbell <marc.e.campbell@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
//
// See: https://github.com/ConradIrwin/trie-ing/blob/df55d7af7068d357829db9e0a7faa8a38add1d1d/LICENSE
/**
* An entry in the prefix trie. The matched word is "content".
*/
interface Entry {
content: string;
key: SearchKey;
weight: number;
}
/**
* The trie is made up of nodes. A node can be EITHER an internal node (whose
* only children are other nodes) OR a leaf, which actually contains the word
* form entries.
*/
type Node = InternalNode | Leaf;
/**
* An internal node.
*/
interface InternalNode {
type: 'internal';
weight: number;
// TODO: As an optimization, "values" can be a single string!
values: string[];
children: { [codeunit: string]: Node };
unsorted?: true;
}
/**
* A leaf node.
*/
interface Leaf {
type: 'leaf';
weight: number;
entries: Entry[];
unsorted?: true;
}
/**
* A sentinel value for when an internal node has contents and requires an
* "internal" leaf. That is, this internal node has content. Instead of placing
* entries as children in an internal node, a "fake" leaf is created, and its
* key is this special internal value.
*
* The value is a valid Unicode BMP code point, but it is a "non-character".
* Unicode will never assign semantics to these characters, as they are
* intended to be used internally as sentinel values.
*/
const INTERNAL_VALUE = '\uFDD0';
/**
* Builds a trie from a word list.
*
* @param wordlist The wordlist with non-negative weights.
* @param keyFunction Function that converts word forms into indexed search keys
* @returns A JSON-serialiable object that can be given to the TrieModel constructor.
*/
export function buildTrie(wordlist: WordList, keyFunction: SearchTermToKey): object {
const root = new Trie(keyFunction).buildFromWordList(wordlist).root;
return {
totalWeight: sumWeights(root),
root: root
}
}
/**
* Wrapper class for the trie and its nodes and wordform to search
*/
class Trie {
readonly root = createRootNode();
toKey: SearchTermToKey;
constructor(wordform2key: SearchTermToKey) {
this.toKey = wordform2key;
}
/**
* Populates the trie with the contents of an entire wordlist.
* @param words a list of word and count pairs.
*/
buildFromWordList(words: WordList): Trie {
for (const [wordform, weight] of Object.entries(words)) {
const key = this.toKey(wordform);
addUnsorted(this.root, { key, weight, content: wordform }, 0);
}
sortTrie(this.root);
return this;
}
}
// "Constructors"
function createRootNode(): Node {
return {
type: 'leaf',
weight: 0,
entries: []
};
}
// Implement Trie creation.
/**
* Adds an entry to the trie.
*
* Note that the trie will likely be unsorted after the add occurs. Before
* performing a lookup on the trie, use call sortTrie() on the root note!
*
* @param node Which node should the entry be added to?
* @param entry the wordform/weight/key to add to the trie
* @param index the index in the key and also the trie depth. Should be set to
* zero when adding onto the root node of the trie.
*/
function addUnsorted(node: Node, entry: Entry, index: number = 0) {
// Each node stores the MAXIMUM weight out of all of its decesdents, to
// enable a greedy search through the trie.
node.weight = Math.max(node.weight, entry.weight);
// When should a leaf become an interior node?
// When it already has a value, but the key of the current value is longer
// than the prefix.
if (node.type === 'leaf' && index < entry.key.length && node.entries.length >= 1) {
convertLeafToInternalNode(node, index);
}
if (node.type === 'leaf') {
// The key matches this leaf node, so add yet another entry.
addItemToLeaf(node, entry);
} else {
// Push the node down to a lower node.
addItemToInternalNode(node, entry, index);
}
node.unsorted = true;
}
/**
* Adds an item to the internal node at a given depth.
* @param item
* @param index
*/
function addItemToInternalNode(node: InternalNode, item: Entry, index: number) {
let char = item.key[index];
// If an internal node is the proper site for item, it belongs under the
// corresponding (sentinel, internal-use) child node signifying this.
if(char == undefined) {
char = INTERNAL_VALUE;
}
if (!node.children[char]) {
node.children[char] = createRootNode();
node.values.push(char);
}
addUnsorted(node.children[char], item, index + 1);
}
function addItemToLeaf(leaf: Leaf, item: Entry) {
leaf.entries.push(item);
}
/**
* Mutates the given Leaf to turn it into an InternalNode.
*
* NOTE: the node passed in will be DESTRUCTIVELY CHANGED into a different
* type when passed into this function!
*
* @param depth depth of the trie at this level.
*/
function convertLeafToInternalNode(leaf: Leaf, depth: number): void {
const entries = leaf.entries;
// Alias the current node, as the desired type.
const internal = (<unknown> leaf) as InternalNode;
internal.type = 'internal';
delete leaf.entries;
internal.values = [];
internal.children = {};
// Convert the old values array into the format for interior nodes.
for (const item of entries) {
let char: string;
if (depth < item.key.length) {
char = item.key[depth];
} else {
char = INTERNAL_VALUE;
}
if (!internal.children[char]) {
internal.children[char] = createRootNode();
internal.values.push(char);
}
addUnsorted(internal.children[char], item, depth + 1);
}
internal.unsorted = true;
}
/**
* Recursively sort the trie, in descending order of weight.
* @param node any node in the trie
*/
function sortTrie(node: Node) {
if (node.type === 'leaf') {
if (!node.unsorted) {
return;
}
node.entries.sort(function (a, b) { return b.weight - a.weight; });
} else {
// We MUST recurse and sort children before returning.
for (const char of node.values) {
sortTrie(node.children[char]);
}
if (!node.unsorted) {
return;
}
node.values.sort((a, b) => {
return node.children[b].weight - node.children[a].weight;
});
}
delete node.unsorted;
}
/**
* O(n) recursive traversal to sum the total weight of all leaves in the
* trie, starting at the provided node.
*
* @param node The node to start summing weights.
*/
function sumWeights(node: Node): number {
let val: number;
if (node.type === 'leaf') {
val = node.entries
.map(entry => entry.weight)
//.map(entry => isNaN(entry.weight) ? 1 : entry.weight)
.reduce((acc, count) => acc + count, 0);
} else {
val = Object.keys(node.children)
.map((key) => sumWeights(node.children[key]))
.reduce((acc, count) => acc + count, 0);
}
if(isNaN(val)) {
throw new Error("Unexpected NaN has appeared!");
}
return val;
/**
* A function that converts a string (word form or query) into a search key
* (secretly, this is also a string).
*/
export interface SearchTermToKey {
(wordform: string): SearchKey;
}
/**
* Builds a trie from a word list.
*
* @param wordlist The wordlist with non-negative weights.
* @param keyFunction Function that converts word forms into indexed search keys
* @returns A JSON-serialiable object that can be given to the TrieModel constructor.
*/
export function buildTrie(wordlist: WordList, keyFunction: SearchTermToKey): TrieBuilder {
const collater = new TrieBuilder(keyFunction);
buildFromWordList(collater, wordlist);
return collater;
}
/**
* Populates the trie with the contents of an entire wordlist.
* @param words a list of word and count pairs.
*/
function buildFromWordList(trieCollator: TrieBuilder, words: WordList): TrieBuilder {
for (const [wordform, weight] of Object.entries(words)) {
trieCollator.addEntry(wordform, weight);
}
trieCollator.sort();
return trieCollator;
}
/**

9
developer/src/kmc-model/test/compile-model-with-pseudoclosure.tests.ts
View file

@ -60,7 +60,8 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () {
assert.match(code, /'-'/);
assert.match(code, /'\+'/);
assert.match(code, /'\^'/);
assert.match(code, /§/);
// From searchTermToKey:
assert.match(code, /'§'/);
const modelInitIndex = code.indexOf('LMLayerWorker.loadModel');
let modelInitCode = code.substring(modelInitIndex);
@ -74,7 +75,7 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () {
// Instead, our custom keyer should ensure that the following symbol DOES appear.
// Verifies that the compiler uses the custom searchTermToKey definition.
assert.match(modelInitCode, /['"]§['"]/);
assert.match(modelInitCode, /[^ ]§/);
// Make sure it compiles!
const compilation = compileModelSourceCode(code);
@ -119,7 +120,7 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () {
// Check that the prepended lowercase "-" DOES appear within the Trie, as keying
// does not remove it in this variant. Verifies that the compiler actually
// used the custom applyCasing definition!
assert.match(modelInitCode, /['"]-['"]/);
assert.match(modelInitCode, /[^ ]-/); // ' -' is indicative of a compressed number
// Make sure it compiles!
const compilation = compileModelSourceCode(code);
@ -158,7 +159,7 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () {
// Check that the prepended lowercase "-" DOES appear within the Trie, as keying
// does not remove it in this variant. Verifies that the compiler actually
// used the custom applyCasing definition!
assert.match(modelInitCode, /['"]-['"]/);
assert.match(modelInitCode, /[^ ]-/); // ' -' is indicative of a compressed number
// Make sure it compiles!
const compilation = compileModelSourceCode(code);

30
developer/src/kmc-model/test/compile-trie.tests.ts
View file

@ -152,14 +152,34 @@ describe('createTrieDataStructure()', function () {
const lowercaseSourceCode = createTrieDataStructure([WORDLIST_FILENAME], (wf) => {
return wf.toLowerCase()
})
assert.match(lowercaseSourceCode, /"key":\s*"turtles"/);
assert.notMatch(lowercaseSourceCode, /"key":\s*"TURTLES"/);
// 'I' should be keyed to 'i', which should appear here.
assert.match(lowercaseSourceCode, /[^ ]i/);
// It's a sparse data set, so only the first letter of each word should appear
// in keyed form when compressed.
const lowerKeyCharMatches = ['L', 'T']
.map((char) => lowercaseSourceCode.indexOf(char))
.filter((entry) => entry > -1);
// At least one letter of 'L' and 'T' should be missing; ideally none,
// but it's possible for one to appear as the encoding for a number.
assert.isAtMost(lowerKeyCharMatches.length, 1);
// 0 assumes that none of the chars appears in the encoded form.
// Fortunately... it's actually true for this fixture as-is.
assert.equal(lowerKeyCharMatches.length, 0);
const uppercaseSourceCode = createTrieDataStructure([WORDLIST_FILENAME], (wf) => {
return wf.toUpperCase()
})
assert.match(uppercaseSourceCode, /"key":\s*"TURTLES"/);
assert.notMatch(uppercaseSourceCode, /"key":\s*"turtles"/);
});
// We don't do a a check for 'i' here because it appears both within the
// wordlist-word 'like' and within the property name 'totalWeight'.
const upperKeyCharMatches = ['I', 'L', 'T']
.map((char) => uppercaseSourceCode.indexOf(char))
.filter((entry) => entry > -1);
// All first letters should appear in keyed form.
assert.equal(upperKeyCharMatches.length, 3);
});
it('does not create `null`/"undefined"-keyed children', function () {

1
developer/src/kmc-model/tsconfig.json
View file

@ -13,5 +13,6 @@
"references": [
{ "path": "../../../common/web/keyman-version" },
{ "path": "../../../common/web/types" },
{ "path": "../../../web/src/engine/predictive-text/templates/" },
]
}

2
package-lock.json generated
View file

@ -854,11 +854,13 @@
"dependencies": {
"@keymanapp/common-types": "*",
"@keymanapp/keyman-version": "*",
"@keymanapp/models-templates": "*",
"typescript": "^5.4.5"
},
"devDependencies": {
"@keymanapp/developer-test-helpers": "*",
"@keymanapp/models-templates": "*",
"@types/node": "^20.4.1",
"c8": "^7.12.0",
"chalk": "^2.4.2",
"esbuild": "^0.25.0"

16
web/src/engine/predictive-text/templates/src/common.ts
View file

@ -157,3 +157,19 @@ export function defaultApplyCasing(casing: CasingForm, text: string): string {
return text.substring(0, headUnitLength).toUpperCase() .concat(text.substring(headUnitLength));
}
}
/**
* An **opaque** type for a string that is exclusively used as a search key in
* the trie. There should be a function that converts arbitrary strings
* (queries) and converts them into a standard search key for a given language
* model.
*/
export type SearchKey = string & { _: 'SearchKey'};
/**
* A function that converts a string (word form or query) into a search key
* (which, internally, is also a string type).
*/
export interface Wordform2Key {
(wordform: string): SearchKey;
}

9
web/src/engine/predictive-text/templates/src/index.ts
View file

@ -1,7 +1,12 @@
export {
SENTINEL_CODE_UNIT, applyTransform, buildMergedTransform, isHighSurrogate, isLowSurrogate, isSentinel,
transformToSuggestion, defaultApplyCasing
applyTransform, buildMergedTransform, isHighSurrogate, isLowSurrogate, isSentinel,
SearchKey, SENTINEL_CODE_UNIT, transformToSuggestion, defaultApplyCasing
} from "./common.js";
export { default as QuoteBehavior } from "./quote-behavior.js";
export { getLastPreCaretToken, Token, Tokenization, tokenize, wordbreak } from "./tokenization.js";
export {
Entry, InternalNode, Leaf, Node, Trie
} from './trie.js';
export { TrieBuilder } from './trie-builder.js';
export * as trieConstruction from './trie-builder.js';
export { default as TrieModel, TrieModelOptions } from "./trie-model.js";

163
web/src/engine/predictive-text/templates/src/trie-builder.ts Normal file
View file

@ -0,0 +1,163 @@
import { SENTINEL_CODE_UNIT, Wordform2Key } from "./common.js";
import { compressNode } from "./trie-compression.js";
import { Entry, InternalNode, Leaf, Node, Trie, sortNode } from "./trie.js";
export function createRootNode(): Node {
return {
type: 'leaf',
weight: 0,
entries: []
};
}
/**
* Adds an entry to the trie. Currently assumes there is no pre-existing match
* for the entry.
*
* Note that the trie will likely be unsorted after the add occurs. Before
* performing a lookup on the trie, use call sortTrie() on the root note!
*
* @param node Which node should the entry be added to?
* @param entry the wordform/weight/key to add to the trie
* @param index the index in the key and also the trie depth. Should be set to
* zero when adding onto the root node of the trie.
*/
export function addUnsorted(node: Node, entry: Entry, index: number = 0) {
// Each node stores the MAXIMUM weight out of all of its decesdents, to
// enable a greedy search through the trie.
node.weight = Math.max(node.weight, entry.weight);
// When should a leaf become an interior node?
// When it already has a value, but the key of the current value is longer
// than the prefix.
if (node.type === 'leaf' && index < entry.key.length && node.entries.length >= 1) {
convertLeafToInternalNode(node, index);
}
if (node.type === 'leaf') {
// The key matches this leaf node, so add yet another entry.
addItemToLeaf(node, entry);
} else {
// Push the node down to a lower node.
addItemToInternalNode(node, entry, index);
}
node.unsorted = true;
}
/**
* Adds an item to the internal node at a given depth.
* @param item
* @param index
*/
export function addItemToInternalNode(node: InternalNode, item: Entry, index: number) {
let char = item.key[index];
// If an internal node is the proper site for item, it belongs under the
// corresponding (sentinel, internal-use) child node signifying this.
if(char == undefined) {
char = SENTINEL_CODE_UNIT;
}
if (!node.children[char]) {
node.children[char] = createRootNode();
node.values.push(char);
}
// Assertion - the path being taken is not compressed.
addUnsorted(node.children[char] as Node, item, index + 1);
}
export function addItemToLeaf(leaf: Leaf, item: Entry) {
leaf.entries.push(item);
}
/**
* Mutates the given Leaf to turn it into an InternalNode.
*
* NOTE: the node passed in will be DESTRUCTIVELY CHANGED into a different
* type when passed into this function!
*
* @param depth depth of the trie at this level.
*/
export function convertLeafToInternalNode(leaf: Leaf, depth: number): void {
let entries = leaf.entries;
// Alias the current node, as the desired type.
let internal = (<unknown> leaf) as InternalNode;
internal.type = 'internal';
delete (leaf as Partial<Leaf>).entries;
internal.values = [];
internal.children = {};
// Convert the old values array into the format for interior nodes.
for (let item of entries) {
let char: string;
if (depth < item.key.length) {
char = item.key[depth];
} else {
char = SENTINEL_CODE_UNIT;
}
if (!internal.children[char]) {
internal.children[char] = createRootNode();
internal.values.push(char);
}
// Assertion - the path being taken is not compressed.
addUnsorted(internal.children[char] as Node, item, depth + 1);
}
internal.unsorted = true;
}
/**
* Wrapper class for the trie and its nodes.
*/
export class TrieBuilder extends Trie {
/** The total weight of the entire trie. */
totalWeight: number;
constructor(toKey: Wordform2Key);
constructor(toKey: Wordform2Key, root: Node, totalWeight: number);
constructor(toKey: Wordform2Key, root?: Node, totalWeight?: number) {
super(root ?? createRootNode(), 0, toKey);
this.totalWeight = totalWeight ?? 0;
}
addEntry(word: string, weight?: number) {
weight = (isNaN(weight ?? NaN) || !weight) ? 1 : weight;
this.totalWeight += weight;
// // Should the Trie have previously been compressed, this will decompress
// // the needed parts of the path for `addUnsorted` and its helpers.
// //
// // kmc-model doesn't do this, of course, but it may matter down the road
// // for 'learning' features. Such features would benefit from this
// // partial decompression strategy.
// this.traverseFromRoot().child(this.toKey(word));
addUnsorted(
this.root, {
key: this.toKey(word),
content: word,
weight: weight
},
0
);
}
sort() {
// Sorts the full Trie, not just a part of it.
sortNode(this.root, this.toKey);
}
getRoot(): Node {
return this.root;
}
getTotalWeight(): number {
return this.totalWeight;
}
compress(): string {
return compressNode(this.root);
}
}

280
web/src/engine/predictive-text/templates/src/trie-compression.ts Normal file
View file

@ -0,0 +1,280 @@
import { Wordform2Key } from "./common.js";
import { Entry, InternalNode, Leaf, Node } from "./trie.js";
// const SINGLE_CHAR_RANGE = Math.pow(2, 16) - 64;
// const ENCODED_NUM_BASE = 64;
// Offsetting by even just 0x0020 avoids control-code chars + avoids VS Code not liking the encoding.
export const ENCODED_NUM_BASE = 0x0020;
export const SINGLE_CHAR_RANGE = Math.pow(2, 16) - ENCODED_NUM_BASE;
/**
* The number of characters allocated to representing the total length of a node or entry.
*/
const NODE_SIZE_WIDTH = 2;
/**
* The number of characters allocated to representing the weight (frequency) of entries.
*/
const WEIGHT_WIDTH = 2;
export function decompressNumber(str: string, start: number, end?: number) {
end ??= str.length;
let num = 0;
for(let i = start; i < end; i++) {
let val = str.charCodeAt(i);
num = num * SINGLE_CHAR_RANGE + val - ENCODED_NUM_BASE;
}
return num;
}
export function compressNumber(num: number, width?: number) {
let compressed = '';
width ||= 1;
// Note: JS bit-shift operators assume 32-bit signed ints.
// JS numbers can easily represent larger ints, though.
while(width > 0) {
const piece = num % SINGLE_CHAR_RANGE + ENCODED_NUM_BASE;
num = Math.floor(num / SINGLE_CHAR_RANGE);
compressed = String.fromCharCode(piece) + compressed;
width--;
}
if(num) {
throw new Error(`Could not properly compress ${num} within specified char width of ${width}`);
}
return compressed;
}
/**
* Length of the 'header' section for encoded entries, representing the
* total size for the entry + its weight.
*/
const ENTRY_HEADER_WIDTH = NODE_SIZE_WIDTH + WEIGHT_WIDTH;
// encoded ENTRY:
// - entryLen: 2 char
// - total encoded size of the entry, including `entryLen`
// - as raw, concatenated string data - no JSON.stringify action taken.
// - weight: 2 chars
// - frequency of the entry
// - contentLen: safe to infer from all other values
// - content: string: from [header+4] to [header+4 + contentLen - 1]
//
// - key: not encoded; we can regenerate it via the keying function
export function compressEntry(entry: Entry): string {
const { content, weight } = entry;
const weightEnc = compressNumber(weight, WEIGHT_WIDTH);
const entryLenEnc = compressNumber(content.length + ENTRY_HEADER_WIDTH, 2);
return `${entryLenEnc}${weightEnc}${content}`;
}
export function decompressEntry(str: string, keyFunction: Wordform2Key, baseIndex?: number): Entry {
baseIndex ||= 0;
const entryLen = decompressNumber(str, baseIndex + 0, baseIndex + NODE_SIZE_WIDTH);
/* c8 ignore start */
if(str.length < baseIndex + entryLen) {
throw new Error('Parts of the encoded entry are missing');
}
/* c8 ignore end */
const headerEnd = baseIndex + ENTRY_HEADER_WIDTH;
const weight = decompressNumber(str, baseIndex + NODE_SIZE_WIDTH, headerEnd);
const content = str.substring(headerEnd, baseIndex + entryLen);
return {
key: keyFunction(content) as any, // needed due to special `SearchKey` type shenanigans in its definition.
content: content,
weight: weight
}
}
// BOTH node types:
// - totalLen: 2 chars (fixed position, size) - size of the encoded node.
// - weight: weight of the highest frequency entry within the node's sub-trie.
// - 2^32 ~= 4*10^9, representable in 2 chars... if it weren't for
// `"`-escaping.
//
// Next char: indicates BOTH a count of something (entries or direct children)
// and a high-bit indicating 'leaf' or 'internal'.
// - function of other bits will be indicated by their sections.
/**
* Length of the 'header' section for encoded leaf and internal nodes, representing the
* total size for their children/entries + the weight of the highest-frequency entry
* represented by each node's represented sub-trie.
*/
export const NODE_TYPE_INDEX = NODE_SIZE_WIDTH + WEIGHT_WIDTH;
export function compressNode(node: Node) {
let encodedSpecifics = node.type == 'leaf' ? compressLeaf(node) : compressInternal(node);
let weightEnc = compressNumber(node.weight, WEIGHT_WIDTH);
let charLength = encodedSpecifics.length + NODE_SIZE_WIDTH + WEIGHT_WIDTH;
return `${compressNumber(charLength, 2)}${weightEnc}${encodedSpecifics}`;
}
export function decompressNode(str: string, keyFunction: Wordform2Key, baseIndex?: number) {
baseIndex ||= 0;
const entryLen = decompressNumber(str, baseIndex + 0, baseIndex + NODE_SIZE_WIDTH);
/* c8 ignore start */
if(str.length < baseIndex + entryLen) {
throw new Error('Parts of the encoded node are missing');
}
/* c8 ignore end */
const typeFlagSrc = decompressNumber(str, baseIndex + NODE_TYPE_INDEX, baseIndex + NODE_TYPE_INDEX + 1);
const isLeafType = typeFlagSrc & 0x8000;
return isLeafType ? decompressLeaf(str, keyFunction, baseIndex) : decompressInternal(str, baseIndex);
}
export function inflateChild(childMap: Record<string, string | Node>, char: string, toKey: Wordform2Key) {
let child = childMap[char];
const inflated = typeof child == 'string' ? decompressNode(child, toKey, 0) : child;
childMap[char] = inflated;
return inflated;
}
// encoded LEAF:
// - BOTH-section header
// - entriesCnt: 1 char (fixed position)
// - type flag overlaps here - high bit of the representing char should be one.
// - entries: full encoding of all contained entries.
function compressLeaf(leaf: Leaf): string {
const entries = leaf.entries;
// key, content, weight - per entry
const entryCntAndType = entries.length | 0x8000;
/* c8 ignore start */
if(entries.length >= 0x8000) {
throw new Error("Cannot encode leaf: too many direct entries");
}
/* c8 ignore end */
let compressedEntries = [compressNumber(entryCntAndType)].concat(entries.map((entry) => {
// if already compressed, no need to recompress it.
return typeof entry == 'string' ? entry : compressEntry(entry);
}));
return compressedEntries.join('');
}
function decompressLeaf(str: string, keyFunction: Wordform2Key, baseIndex: number): Leaf {
const weight = decompressNumber(str, baseIndex + NODE_SIZE_WIDTH, baseIndex + NODE_SIZE_WIDTH + WEIGHT_WIDTH);
// Assumes string-subsection size check has passed.
const entryCntSrc = decompressNumber(str, baseIndex + NODE_TYPE_INDEX, baseIndex + NODE_TYPE_INDEX + 1);
// Remove the type-flag bit indicating 'leaf node'.
const entryCnt = entryCntSrc & 0x7FFF;
let entries: Entry[] = [];
baseIndex = baseIndex + NODE_TYPE_INDEX + 1;
for(let i = 0; i < entryCnt; i++) {
const entryWidth = decompressNumber(str, baseIndex, baseIndex+NODE_SIZE_WIDTH);
const nextIndex = baseIndex + entryWidth;
entries.push(decompressEntry(str, keyFunction, baseIndex));
baseIndex = nextIndex;
}
return {
type: 'leaf',
weight: weight,
// To consider: is it better to just make a 'lazy span' against the original string?
// - would use more memory, especially once a Trie is "mostly" decompressed
// - current approach 'discards' decoded parts of the original string.
// - would likely decompress a bit faster.
entries: entries
}
}
// encoded INTERNAL:
// - BOTH-section header
// - valLen: 1 char (fixed position, size) - we shouldn't ever have an array of > 65000, right?
// - is also the count for children.
// - type flag overlaps here - high bit of the representing char should be zero.
// - values: string
// - ezpz - they're already single-char strings.
// - children: full encoding of next-layer nodes. Same order as the entries are found within `values`.
// - that is, no need to insert keys.
// - first two bits: length of following bits... so can use that to calculate offset for next entry's
// encoding start.
function compressInternal(node: InternalNode): string {
let values = node.values;
const valueCntAndType = values.length;
/* c8 ignore start */
if(valueCntAndType >= 0x8000) {
throw new Error("Cannot encode node: too many direct children");
}
/* c8 ignore end */
const compressedChildren = values.map((value) => {
// In case of regression for #11073
if(value === null) {
value = "undefined"; // yes, really.
}
const child = node.children[value];
/* c8 ignore start */
if(!child) {
throw new Error("unexpected empty reference for child");
}
/* c8 ignore end */
// No need to recompress it if it's already compressed.
return typeof child == 'string' ? child : compressNode(child);
});
// Properly fix the sentinel-value issue.
// Also of note: this array may contain halves of surrogate pairs.
values = values.map((value) => value === null ? '\ufdd0' : value);
const totalArr = [compressNumber(valueCntAndType)].concat(values).concat(compressedChildren);
return totalArr.join('');
}
function decompressInternal(str: string, baseIndex: number): Omit<InternalNode, 'children'> & { children: {[char: string]: string} } {
const weight = decompressNumber(str, baseIndex + NODE_SIZE_WIDTH, baseIndex + NODE_SIZE_WIDTH + WEIGHT_WIDTH);
// Assumes string-subsection size check has passed.
const childCnt = decompressNumber(str, baseIndex + NODE_TYPE_INDEX, baseIndex + NODE_TYPE_INDEX + 1);
baseIndex = baseIndex + NODE_TYPE_INDEX + 1;
let nextIndex = baseIndex + childCnt;
const values = str.substring(baseIndex, nextIndex).split('');
baseIndex = nextIndex;
let compressedChildren: {[char: string]: string} = {};
for(let i = 0; i < childCnt; i++) {
const childWidth = decompressNumber(str, baseIndex, baseIndex+NODE_SIZE_WIDTH);
nextIndex = baseIndex + childWidth;
compressedChildren[values[i]] = str.substring(baseIndex, nextIndex);
baseIndex = nextIndex;
}
return {
type: 'internal',
weight: weight,
values: values,
// To consider: is it better to just make a 'lazy span' against the original string?
// - would use more memory, especially once a Trie is "mostly" decompressed
// - would likely decompress a bit faster.
// - would not be wise if we ever wish to have live editing of a loaded Trie; it's only
// a useful strategy if the backing data is static.
children: compressedChildren
}
}

312
web/src/engine/predictive-text/templates/src/trie-model.ts
View file

@ -28,7 +28,8 @@
import { KMWString, PriorityQueue } from "@keymanapp/web-utils";
import { default as defaultWordBreaker } from "@keymanapp/models-wordbreakers";
import { applyTransform, isHighSurrogate, isSentinel, SENTINEL_CODE_UNIT, transformToSuggestion } from "./common.js";
import { applyTransform, SearchKey, transformToSuggestion, Wordform2Key } from "./common.js";
import { Node, Trie, TrieTraversal } from './trie.js';
import { getLastPreCaretToken } from "./tokenization.js";
import { LexicalModelTypes } from "@keymanapp/common-types";
import Capabilities = LexicalModelTypes.Capabilities;
@ -84,182 +85,6 @@ export interface TrieModelOptions {
punctuation?: LexicalModelPunctuation;
}
export class Traversal implements LexiconTraversal {
/**
* The lexical prefix corresponding to the current traversal state.
*/
readonly prefix: string;
/**
* The current traversal node. Serves as the 'root' of its own sub-Trie,
* and we cannot navigate back to its parent.
*/
readonly root: Node;
/**
* The max weight for the Trie being 'traversed'. Needed for probability
* calculations.
*/
readonly totalWeight: number;
constructor(root: Node, prefix: string, totalWeight: number) {
this.root = root;
this.prefix = prefix;
this.totalWeight = totalWeight;
}
child(char: string): Traversal | undefined {
// May result for blank tokens resulting immediately after whitespace.
if(char == '') {
return this;
}
// Split into individual code units.
let steps = char.split('');
let traversal: Traversal | undefined = this;
while(steps.length > 0 && traversal) {
const step: string = steps.shift()!;
traversal = traversal._child(step);
}
return traversal;
}
// Handles one code unit at a time.
private _child(char: string): Traversal | undefined {
const root = this.root;
const totalWeight = this.totalWeight;
const nextPrefix = this.prefix + char;
if(root.type == 'internal') {
let childNode = root.children[char];
if(!childNode) {
return undefined;
}
return new Traversal(childNode, nextPrefix, totalWeight);
} else {
// root.type == 'leaf';
const legalChildren = root.entries.filter(function(entry) {
return entry.key.indexOf(nextPrefix) == 0;
});
if(!legalChildren.length) {
return undefined;
}
return new Traversal(root, nextPrefix, totalWeight);
}
}
*children(): Generator<{char: string, traversal: () => Traversal}> {
let root = this.root;
// We refer to the field multiple times in this method, and it doesn't change.
// This also assists minification a bit, since we can't minify when re-accessing
// through `this.`.
const totalWeight = this.totalWeight;
if(root.type == 'internal') {
for(let entry of root.values) {
let entryNode = root.children[entry];
// UTF-16 astral plane check.
if(isHighSurrogate(entry)) {
// First code unit of a UTF-16 code point.
// For now, we'll just assume the second always completes such a char.
//
// Note: Things get nasty here if this is only sometimes true; in the future,
// we should compile-time enforce that this assumption is always true if possible.
if(entryNode.type == 'internal') {
let internalNode = entryNode;
for(let lowSurrogate of internalNode.values) {
let prefix = this.prefix + entry + lowSurrogate;
yield {
char: entry + lowSurrogate,
traversal: function() { return new Traversal(internalNode.children[lowSurrogate], prefix, totalWeight) }
}
}
} else {
// Determine how much of the 'leaf' entry has no Trie nodes, emulate them.
let fullText = entryNode.entries[0].key;
entry = entry + fullText[this.prefix.length + 1]; // The other half of the non-BMP char.
let prefix = this.prefix + entry;
yield {
char: entry,
traversal: function () {return new Traversal(entryNode, prefix, totalWeight)}
}
}
} else if(isSentinel(entry)) {
continue;
} else if(!entry) {
// Prevent any accidental 'null' or 'undefined' entries from having an effect.
continue;
} else {
let prefix = this.prefix + entry;
yield {
char: entry,
traversal: function() { return new Traversal(entryNode, prefix, totalWeight)}
}
}
}
return;
} else { // type == 'leaf'
let prefix = this.prefix;
let children = root.entries.filter(function(entry) {
return entry.key != prefix && prefix.length < entry.key.length;
})
for(let {key} of children) {
let nodeKey = key[prefix.length];
if(isHighSurrogate(nodeKey)) {
// Merge the other half of an SMP char in!
nodeKey = nodeKey + key[prefix.length+1];
}
yield {
char: nodeKey,
traversal: function() { return new Traversal(root, prefix + nodeKey, totalWeight)}
}
};
return;
}
}
get entries() {
const entryMapper = (value: Entry) => {
return {
text: value.content,
p: value.weight / this.totalWeight
}
}
if(this.root.type == 'leaf') {
let prefix = this.prefix;
let matches = this.root.entries.filter(function(entry) {
return entry.key == prefix;
});
return matches.map(entryMapper);
} else {
let matchingLeaf = this.root.children[SENTINEL_CODE_UNIT];
if(matchingLeaf && matchingLeaf.type == 'leaf') {
return matchingLeaf.entries.map(entryMapper);
} else {
return [];
}
}
}
get p(): number {
return this.root.weight / this.totalWeight;
}
}
/**
* @class TrieModel
*
@ -278,13 +103,18 @@ export default class TrieModel implements LexicalModel {
readonly applyCasing?: CasingFunction;
constructor(trieData: {root: Node, totalWeight: number}, options: TrieModelOptions = {}) {
constructor(trieData: {root: Node, totalWeight: number}, options?: TrieModelOptions);
constructor(trieData: {data: string, totalWeight: number}, options?: TrieModelOptions);
constructor(trieData: {root: Node, totalWeight: number} | {data: string, totalWeight: number}, options: TrieModelOptions = {}) {
this.languageUsesCasing = options.languageUsesCasing;
this.applyCasing = options.applyCasing;
// @ts-ignore
const trieDataSrc: Node | string = trieData.data || trieData.root;
this._trie = new Trie(
trieData['root'],
trieData['totalWeight'],
trieDataSrc,
trieData.totalWeight,
options.searchTermToKey as Wordform2Key || defaultSearchTermToKey
);
this.breakWords = options.wordBreaker || defaultWordBreaker;
@ -315,7 +145,7 @@ export default class TrieModel implements LexicalModel {
// Special-case the empty buffer/transform: return the top suggestions.
if (!transform.insert && !context.left && !context.right && context.startOfBuffer && context.endOfBuffer) {
return makeDistribution(this._trie.firstN(MAX_SUGGESTIONS).map(({text, p}) => ({
return makeDistribution(this.firstN(MAX_SUGGESTIONS).map(({text, p}) => ({
transform: {
insert: text,
deleteLeft: 0
@ -336,7 +166,7 @@ export default class TrieModel implements LexicalModel {
const prefix = getLastPreCaretToken(this.breakWords, newContext);
// Return suggestions from the trie.
return makeDistribution(this._trie.lookup(prefix).map(({text, p}) =>
return makeDistribution(this.lookup(prefix).map(({text, p}) =>
transformToSuggestion({
insert: text,
// Delete whatever the prefix that the user wrote.
@ -352,104 +182,16 @@ export default class TrieModel implements LexicalModel {
return this.breakWords;
}
public traverseFromRoot(): Traversal {
public traverseFromRoot(): TrieTraversal {
return this._trie.traverseFromRoot();
}
};
/////////////////////////////////////////////////////////////////////////////////
// What remains in this file is the trie implementation proper. Note: to //
// reduce bundle size, any functions/methods related to creating the trie have //
// been removed. //
/////////////////////////////////////////////////////////////////////////////////
/**
* An **opaque** type for a string that is exclusively used as a search key in
* the trie. There should be a function that converts arbitrary strings
* (queries) and converts them into a standard search key for a given language
* model.
*
* Fun fact: This opaque type has ALREADY saved my bacon and found a bug!
*/
type SearchKey = string & { _: 'SearchKey'};
/**
* The priority queue will always pop the most probable item - be it a Traversal
* state or a lexical entry reached via Traversal.
*/
type TraversableWithProb = TextWithProbability | LexiconTraversal;
/**
* A function that converts a string (word form or query) into a search key
* (secretly, this is also a string).
*/
interface Wordform2Key {
(wordform: string): SearchKey;
}
// The following trie implementation has been (heavily) derived from trie-ing
// by Conrad Irwin.
// trie-ing is copyright (C) 2015–2017 Conrad Irwin.
// Distributed under the terms of the MIT license:
// https://github.com/ConradIrwin/trie-ing/blob/df55d7af7068d357829db9e0a7faa8a38add1d1d/LICENSE
type Node = InternalNode | Leaf;
/**
* An internal node in the trie. Internal nodes NEVER contain entries; if an
* internal node should contain an entry, then it has a dummy leaf node (see
* below), that can be accessed by node.children["\uFDD0"].
*/
interface InternalNode {
type: 'internal';
weight: number;
/** Maintains the keys of children in descending order of weight. */
values: string[]; // TODO: As an optimization, "values" can be a single string!
/**
* Maps a single UTF-16 code unit to a child node in the trie. This child
* node may be a leaf or an internal node. The keys of this object are kept
* in sorted order in the .values array.
* Returns the top N suggestions from the trie.
* @param n How many suggestions, maximum, to return.
*/
children: { [codeunit: string]: Node };
}
/** Only leaf nodes actually contain entries (i.e., the words proper). */
interface Leaf {
type: 'leaf';
weight: number;
entries: Entry[];
}
/**
* An entry in the prefix trie (stored in leaf nodes exclusively!)
*/
interface Entry {
/** The actual word form, stored in the trie. */
content: string;
/** A search key that usually simplifies the word form, for ease of search. */
key: SearchKey;
weight: number;
}
/**
* Wrapper class for the trie and its nodes.
*/
class Trie {
public readonly root: Node;
/** The total weight of the entire trie. */
readonly totalWeight: number;
/**
* Converts arbitrary strings to a search key. The trie is built up of
* search keys; not each entry's word form!
*/
toKey: Wordform2Key;
constructor(root: Node, totalWeight: number, wordform2key: Wordform2Key) {
this.root = root;
this.toKey = wordform2key;
this.totalWeight = totalWeight;
}
public traverseFromRoot(): Traversal {
return new Traversal(this.root, '', this.totalWeight);
firstN(n: number): TextWithProbability[] {
return getSortedResults(this._trie.traverseFromRoot(), n);
}
/**
@ -480,15 +222,19 @@ class Trie {
// priority over anything from its descendants.
return directEntries.concat(deduplicated);
}
};
/**
* Returns the top N suggestions from the trie.
* @param n How many suggestions, maximum, to return.
*/
firstN(n: number): TextWithProbability[] {
return getSortedResults(this.traverseFromRoot(), n);
}
}
/////////////////////////////////////////////////////////////////////////////////
// What remains in this file is the trie implementation proper. Note: to //
// reduce bundle size, any functions/methods related to creating the trie have //
// been removed. //
/////////////////////////////////////////////////////////////////////////////////
/**
* The priority queue will always pop the most probable item - be it a Traversal
* state or a lexical entry reached via Traversal.
*/
type TraversableWithProb = TextWithProbability | LexiconTraversal;
/**
* Returns all entries matching the given prefix, in descending order of

301
web/src/engine/predictive-text/templates/src/trie.ts Normal file
View file

@ -0,0 +1,301 @@
import { LexicalModelTypes } from "@keymanapp/common-types";
import LexiconTraversal = LexicalModelTypes.LexiconTraversal;
import { isHighSurrogate, isSentinel, SearchKey, SENTINEL_CODE_UNIT, Wordform2Key } from "./common.js";
import { decompressNode, inflateChild } from "./trie-compression.js";
// The following trie implementation has been (heavily) derived from trie-ing
// by Conrad Irwin.
// trie-ing is copyright (C) 2015–2017 Conrad Irwin.
// Distributed under the terms of the MIT license:
// https://github.com/ConradIrwin/trie-ing/blob/df55d7af7068d357829db9e0a7faa8a38add1d1d/LICENSE
export type Node = InternalNode | Leaf;
/**
* An internal node in the trie. Internal nodes NEVER contain entries; if an
* internal node should contain an entry, then it has a dummy leaf node (see
* below), that can be accessed by node.children["\uFDD0"].
*/
export interface InternalNode {
type: 'internal';
weight: number;
/** Maintains the keys of children in descending order of weight. */
values: string[]; // TODO: As an optimization, "values" can be a single string!
/**
* Maps a single UTF-16 code unit to a child node in the trie. This child
* node may be a leaf or an internal node. The keys of this object are kept
* in sorted order in the .values array.
*/
children: { [codeunit: string]: Node | string };
/**
* Used during compilation.
*/
unsorted?: boolean;
}
/** Only leaf nodes actually contain entries (i.e., the words proper). */
export interface Leaf {
type: 'leaf';
weight: number;
entries: Entry[];
/**
* Used during compilation.
*/
unsorted?: boolean;
}
/**
* An entry in the prefix trie (stored in leaf nodes exclusively!)
*/
export interface Entry {
/** The actual word form, stored in the trie. */
content: string;
/** A search key that usually simplifies the word form, for ease of search. */
key: SearchKey;
weight: number;
}
/**
* Recursively sort the trie, in descending order of weight.
* @param node any node in the trie
*/
export function sortNode(node: Node, toKey: Wordform2Key, onlyLocal?: boolean) {
if (node.type === 'leaf') {
if (!node.unsorted) {
return;
}
node.entries.sort(function (a, b) { return b.weight - a.weight; });
} else {
if(!onlyLocal) {
// We recurse and sort children before returning if sorting the full Trie.
for (let char of node.values) {
const childNode = inflateChild(node.children, char, toKey);
sortNode(childNode, toKey, onlyLocal);
}
}
if (!node.unsorted) {
return;
}
node.values.sort((a, b) => {
return (node.children[b] as Node).weight - (node.children[a] as Node).weight;
});
}
delete node.unsorted;
}
export class TrieTraversal implements LexiconTraversal {
/**
* The lexical prefix corresponding to the current traversal state.
*/
readonly prefix: String;
/**
* The current traversal node. Serves as the 'root' of its own sub-Trie,
* and we cannot navigate back to its parent.
*/
readonly root: Node;
readonly toKey: Wordform2Key;
/**
* The max weight for the Trie being 'traversed'. Needed for probability
* calculations.
*/
readonly totalWeight: number;
constructor(root: Node, toKey: Wordform2Key, prefix: string, totalWeight: number) {
this.root = root;
this.toKey = toKey;
this.prefix = prefix;
this.totalWeight = totalWeight;
}
child(char: string): LexiconTraversal | undefined {
// May result for blank tokens resulting immediately after whitespace.
if(char == '') {
return this;
}
// Split into individual code units.
let steps = char.split('');
let traversal: TrieTraversal | undefined = this;
while(steps.length > 0 && traversal) {
const step: string = steps.shift()!;
traversal = traversal._child(step);
}
return traversal;
}
// Handles one code unit at a time.
private _child(char: string): TrieTraversal | undefined {
const root = this.root;
const nextPrefix = this.prefix + char;
// Sorts _just_ the current level, and only if needed.
// We only care about sorting parts that we're actually accessing.
sortNode(root, this.toKey, true);
if(root.type == 'internal') {
let child = root.children[char];
if(!child) {
return undefined;
}
const childNode = inflateChild(root.children, char, this.toKey);
return new TrieTraversal(childNode, this.toKey, nextPrefix, this.totalWeight);
} else {
// root.type == 'leaf';
const legalChildren = root.entries.filter(function(entry) {
return entry.key.indexOf(nextPrefix) == 0;
});
if(!legalChildren.length) {
return undefined;
}
return new TrieTraversal(root, this.toKey, nextPrefix, this.totalWeight);
}
}
*children(): Generator<{char: string, traversal: () => LexiconTraversal}> {
const root = this.root;
// Sorts _just_ the current level, and only if needed.
// We only care about sorting parts that we're actually accessing.
sortNode(root, this.toKey, true);
if(root.type == 'internal') {
for(let entry of root.values) {
let entryNode = inflateChild(root.children, entry, this.toKey);
// UTF-16 astral plane check.
if(isHighSurrogate(entry)) {
// First code unit of a UTF-16 code point.
// For now, we'll just assume the second always completes such a char.
//
// Note: Things get nasty here if this is only sometimes true; in the future,
// we should compile-time enforce that this assumption is always true if possible.
if(entryNode.type == 'internal') {
let internalNode = entryNode;
for(let lowSurrogate of internalNode.values) {
let prefix = this.prefix + entry + lowSurrogate;
yield {
char: entry + lowSurrogate,
traversal: () => {
return new TrieTraversal(
inflateChild(internalNode.children, lowSurrogate, this.toKey),
this.toKey,
prefix,
this.totalWeight
);
}
}
}
} else {
// Determine how much of the 'leaf' entry has no Trie nodes, emulate them.
let fullText = entryNode.entries[0].key;
entry = entry + fullText[this.prefix.length + 1]; // The other half of the non-BMP char.
let prefix = this.prefix + entry;
yield {
char: entry,
traversal: () => {return new TrieTraversal(entryNode, this.toKey, prefix, this.totalWeight)}
}
}
} else if(isSentinel(entry)) {
continue;
} else if(!entry) {
// Prevent any accidental 'null' or 'undefined' entries from having an effect.
continue;
} else {
let prefix = this.prefix + entry;
yield {
char: entry,
traversal: () => { return new TrieTraversal(entryNode, this.toKey, prefix, this.totalWeight)}
}
}
}
return;
} else { // type == 'leaf'
let prefix = this.prefix;
let children = root.entries.filter(function(entry) {
return entry.key != prefix && prefix.length < entry.key.length;
})
for(let {key} of children) {
let nodeKey = key[prefix.length];
if(isHighSurrogate(nodeKey)) {
// Merge the other half of an SMP char in!
nodeKey = nodeKey + key[prefix.length+1];
}
yield {
char: nodeKey,
traversal: () => { return new TrieTraversal(root, this.toKey, prefix + nodeKey, this.totalWeight)}
}
};
return;
}
}
get entries() {
const root = this.root;
const entryMapper = (value: Entry) => {
return {
text: value.content,
p: value.weight / this.totalWeight
}
}
if(root.type == 'leaf') {
let matches = root.entries.filter((entry) => {
return entry.key == this.prefix;
});
return matches.map(entryMapper);
} else {
let matchingLeaf = inflateChild(root.children, SENTINEL_CODE_UNIT, this.toKey);
if(matchingLeaf && matchingLeaf.type == 'leaf') {
return matchingLeaf.entries.map(entryMapper);
} else {
return [];
}
}
}
get p(): number {
return this.root.weight / this.totalWeight;
}
}
/**
* Wrapper class for the trie and its nodes.
*/
export class Trie {
readonly root: Node;
/** The total weight of the entire trie. */
readonly totalWeight: number;
/**
* Converts arbitrary strings to a search key. The trie is built up of
* search keys; not each entry's word form!
*/
toKey: Wordform2Key;
constructor(trie: Node | string, totalWeight: number, wordform2key: Wordform2Key) {
this.root = (typeof trie == 'string') ? decompressNode(trie, wordform2key, 0) : trie;
this.toKey = wordform2key;
this.totalWeight = totalWeight;
}
public traverseFromRoot(): TrieTraversal {
return new TrieTraversal(this.root, this.toKey, '', this.totalWeight);
}
}

20
web/src/engine/predictive-text/worker-main/README.md
View file

@ -17,11 +17,13 @@ Build
Run `build.sh`. This will also automatically install dependencies with `npm`.
```sh
./build.sh
./build.sh configure build
```
### Two-stage compilation process
TODO: this is well and truly out of date...
Since the primary LMLayer code runs within a [Web Worker][], `build.sh` compiles the
LMLayer in two stages:
@ -43,19 +45,7 @@ This will run both headless unit tests, and in-browser unit tests and integratio
tests:
```sh
./build.sh -test
./build.sh test
```
### Test-Driven Development
I like to use [entr]() to automatically build and re-run the unit tests anytime I
change a source code file. Here's the command I run in separate window:
```sh
git ls-files | entr -c ./build.sh -tdd
```
Importantly, `./build.sh -tdd` skips running the in-browser tests, and skips
downloading/updating `npm` dependencies.
[entr]: http://eradman.com/entrproject/
### TODO: epic/user-dict

264
web/src/test/auto/headless/engine/predictive-text/templates/trie-builder.tests.ts Normal file
View file

@ -0,0 +1,264 @@
/*
* Unit tests for the Trie prediction model.
*/
import { assert } from 'chai';
import { InternalNode, Leaf, SearchKey, SENTINEL_CODE_UNIT, trieConstruction, TrieBuilder } from '@keymanapp/models-templates';
describe('trie construction', () => {
it('default root node', () => {
const defaultRoot = trieConstruction.createRootNode() as Leaf;
assert.equal(defaultRoot.type, 'leaf');
assert.equal(defaultRoot.weight, 0);
assert.sameMembers(defaultRoot.entries, []);
});
it('addItemToLeaf', () => {
// `weight` is managed by addUnsorted, not by addItemToLeaf.
// As a result, we don't test for it here.
const defaultRoot = trieConstruction.createRootNode() as Leaf;
trieConstruction.addItemToLeaf(defaultRoot, {
content: 'cafe',
weight: 2,
key: 'cafe' as SearchKey
});
assert.equal(defaultRoot.type, 'leaf');
assert.equal(defaultRoot.entries.length, 1);
trieConstruction.addItemToLeaf(defaultRoot, {
content: 'café',
weight: 1,
key: 'cafe' as SearchKey
});
assert.equal(defaultRoot.type, 'leaf');
assert.equal(defaultRoot.entries.length, 2);
});
it('convertLeafToInternalNode', () => {
const defaultRoot = trieConstruction.createRootNode() as InternalNode;
const cafeEntry = {
content: 'cafe',
weight: 5,
key: 'cafe' as SearchKey
};
trieConstruction.addItemToLeaf(defaultRoot as unknown as Leaf, cafeEntry);
defaultRoot.weight = 5;
trieConstruction.convertLeafToInternalNode(defaultRoot as unknown as Leaf, 4);
assert.equal(defaultRoot.type, 'internal');
assert.sameMembers(defaultRoot.values, [SENTINEL_CODE_UNIT]);
assert.sameMembers(Object.keys(defaultRoot.children), defaultRoot.values);
assert.equal(defaultRoot.weight, 5);
const sentinelChild = defaultRoot.children[SENTINEL_CODE_UNIT] as Leaf;
assert.equal(sentinelChild.type, 'leaf');
assert.equal(sentinelChild.weight, 5);
assert.equal(sentinelChild.entries.length, 1);
// Should be strict-equal too, but we only really care if it's deep-equal.
assert.deepEqual(sentinelChild.entries[0], cafeEntry);
});
it('addUnsorted - simple initial entry', () => {
// `weight` is managed by addUnsorted, not by addItemToLeaf.
// As a result, we don't test for it here.
const defaultRoot = trieConstruction.createRootNode() as Leaf;
trieConstruction.addUnsorted(defaultRoot, {
content: 'cafe',
weight: 2,
key: 'cafe' as SearchKey
}, 4);
assert.equal(defaultRoot.type, 'leaf');
assert.equal(defaultRoot.weight, 2);
assert.equal(defaultRoot.entries.length, 1);
// smaller weight, will not override. Is not additive.
trieConstruction.addUnsorted(defaultRoot, {
content: 'café',
weight: 1,
key: 'cafe' as SearchKey
}, 4);
assert.equal(defaultRoot.type, 'leaf');
assert.equal(defaultRoot.weight, 2);
assert.equal(defaultRoot.entries.length, 2);
});
it('addUnsorted - short word, then longer word with same prefix', () => {
const defaultRoot = trieConstruction.createRootNode();
trieConstruction.addUnsorted(defaultRoot, {
content: 'cafe',
weight: 5,
key: 'cafe' as SearchKey
}, 4);
const rootAsLeaf = defaultRoot as Leaf;
assert.equal(rootAsLeaf.type, 'leaf');
assert.equal(rootAsLeaf.weight, 5);
assert.equal(rootAsLeaf.entries.length, 1);
// smaller weight, will not override. Is not additive.
trieConstruction.addUnsorted(defaultRoot, {
content: 'cafeteria',
weight: 3,
key: 'cafeteria' as SearchKey
}, 4);
const rootAsInternal = defaultRoot as InternalNode;
assert.equal(rootAsInternal.type, 'internal');
assert.equal(rootAsInternal.weight, 5);
assert.deepEqual(rootAsInternal.values, [SENTINEL_CODE_UNIT, 't']);
assert.equal((rootAsInternal.children[SENTINEL_CODE_UNIT] as Leaf).weight, 5);
assert.equal((rootAsInternal.children['t'] as Leaf).weight, 3);
});
it('addUnsorted', () => {
// Perspective: current root is actually representing `ca`.
const defaultRoot = trieConstruction.createRootNode();
trieConstruction.addUnsorted(defaultRoot, {
content: 'cafe',
weight: 5,
key: 'cafe' as SearchKey
}, 2);
// There's only one child, so there's no reason to start making
// internal structure just yet.
assert.equal(defaultRoot.type, 'leaf');
assert.equal(defaultRoot.weight, 5);
// Adds a second child; that child has more than one letter not-in-common with the first.
trieConstruction.addUnsorted(defaultRoot, {
content: 'call',
weight: 8,
key: 'call' as SearchKey
}, 2);
const rootAsInternal = defaultRoot as InternalNode;
assert.equal(rootAsInternal.type, 'internal');
assert.equal(rootAsInternal.weight, 8);
assert.sameMembers(rootAsInternal.values, ['f', 'l']);
assert.equal((rootAsInternal.children['f'] as Leaf).weight, 5);
assert.equal((rootAsInternal.children['l'] as Leaf).weight, 8);
});
it('addUnsorted - lower frequency prefix', () => {
// Will correspond to 'thin'.
const root = trieConstruction.createRootNode();
trieConstruction.addUnsorted(root, {
key: 'think' as SearchKey,
content: 'think',
weight: 100
}, 4);
trieConstruction.addUnsorted(root, {
key: 'thing' as SearchKey,
content: 'thing',
weight: 40
}, 4);
const rootAsInternal = root as InternalNode;
assert.equal(rootAsInternal.type, 'internal');
assert.sameMembers(rootAsInternal.values, ['k', 'g']);
// Interesting noted behavior: if just 'think' into 'thin', and nothing else...
// it remains a leaf! That doesn't seem... entirely proper. With 'thing', it
// will at least be an 'internal' node already.
trieConstruction.addUnsorted(rootAsInternal, {
key: 'thin' as SearchKey,
content: 'thin',
weight: 20
}, 4);
assert.sameMembers(rootAsInternal.values, ['k', 'g', SENTINEL_CODE_UNIT]);
assert.isOk(rootAsInternal.children[SENTINEL_CODE_UNIT]);
assert.equal((rootAsInternal.children[SENTINEL_CODE_UNIT] as Leaf).weight, 20);
});
describe('TrieBuilder', () => {
it('standard Trie construction', () => {
const builder = new TrieBuilder((text) => text as SearchKey);
builder.addEntry('caffeine', 2);
builder.addEntry('cafe', 5);
builder.addEntry('calm', 3);
builder.addEntry('calf', 4);
builder.addEntry('call', 6); // total: 20
builder.addEntry('can', 10); // total: 30
builder.addEntry('and', 20); // total: 50
assert.equal(builder.getTotalWeight(), 50);
const root = builder.getRoot() as InternalNode;
const aNode = (root.children['c'] as InternalNode).children['a'] as InternalNode;
// As the nodes were not added in sorted order, they should not currently be ordered.
assert.sameMembers(aNode.values, ['n', 'l', 'f']);
assert.notSameOrderedMembers(aNode.values, ['n', 'l', 'f']);
const lNode = aNode.children['l'] as InternalNode;
assert.sameMembers(lNode.values, ['l', 'f', 'm']);
assert.notSameOrderedMembers(lNode.values, ['l', 'f', 'm']);
builder.sort();
assert.sameOrderedMembers(aNode.values, ['n', 'l', 'f']);
assert.sameOrderedMembers(lNode.values, ['l', 'f', 'm']);
});
// In case of high startup-time for user-dictionary processing, this could allow use
// of a 'partially' processed user wordlist. (We'd prioritize predicting when the
// user is interacting with text, then resume processing once 'idle'.)
it('interspersed construction + lookup', () => {
const builder = new TrieBuilder((text) => text as SearchKey);
builder.addEntry('caffeine', 2);
builder.addEntry('cafe', 5);
builder.addEntry('calm', 3);
builder.addEntry('calf', 4);
builder.addEntry('call', 6); // total: 20
// ------------------------------------------
// Pause construction; actually use the Trie.
assert.equal(builder.getTotalWeight(), 20);
const root = builder.getRoot() as InternalNode;
const caNode = (root.children['c'] as InternalNode).children['a'] as InternalNode;
// As the nodes were not added in sorted order, they should not currently be ordered.
assert.sameMembers(caNode.values, ['l', 'f']);
assert.notSameOrderedMembers(caNode.values, ['l', 'f']);
const cal = builder.traverseFromRoot().child('cal');
// Actually traversing through the node should auto-sort the entries.
assert.sameOrderedMembers(caNode.values, ['l', 'f']);
// Including the reached node's children.
assert.sameOrderedMembers([...cal.children()].map((entry) => entry.char), ['l', 'f', 'm']);
const cafNode = caNode.children['f'] as InternalNode;
// 'caffeine' was added before 'cafe'.
// Parts not 'traversed' should not be unnecessarily sorted.
assert.notSameOrderedMembers(cafNode.values, ['e', 'f']);
// -------------------
// Resume construction
builder.addEntry('can', 10); // total: 30
builder.addEntry('and', 20); // total: 50
assert.equal(builder.getTotalWeight(), 50);
// As the nodes were not added in sorted order, they should not currently be ordered.
assert.sameMembers(caNode.values, ['n', 'l', 'f']);
// 'n' was added later, thus will be out of sorted order.
assert.notSameOrderedMembers(caNode.values, ['n', 'l', 'f']);
builder.sort();
assert.sameOrderedMembers(caNode.values, ['n', 'l', 'f']);
});
});
});

301
web/src/test/auto/headless/engine/predictive-text/templates/trie-compression.tests.ts Normal file
View file

@ -0,0 +1,301 @@
/*
* Unit tests for the Trie prediction model.
*/
import { assert } from 'chai';
import {
compressEntry, decompressEntry,
compressNode, decompressNode,
compressNumber, decompressNumber,
ENCODED_NUM_BASE
} from '@keymanapp/models-templates/obj/trie-compression.js';
import { Entry, InternalNode, Leaf, SearchKey, Trie, TrieBuilder } from '@keymanapp/models-templates';
import { jsonFixture } from './helpers.js';
const smpWordlist: [string, number?][] = [
['CRAZ🤪', 13644],
['🙄', 9134],
['😇', 4816],
['🇸'],
['u']
];
/**
* @param {string} str
* @returns
*/
const identityKey = (str: string) => str as SearchKey;
/**
* @param {string} str
* @returns
*/
const lowercaseKey = (str: string) => str.toLowerCase() as SearchKey;
// Written with:
// const ENCODED_NUM_BASE = 0; // 0x0020;
// const SINGLE_CHAR_RANGE = Math.pow(2, 16) - ENCODED_NUM_BASE;
//
// Will need manual re-encoding for \u-sequences if ENCODED_NUM_BASE is changed.
const TEST_ENTRIES = {
four: {
// total length: header = 4, text = 4 -> 8. (Made with weight-width 2)
// -totalLen- -weight-
compressed: `${compressNumber(8, 2)}${compressNumber(8, 2)}four`,
decompressed: {
key: 'four',
content: 'four',
weight: 8
} as Entry,
original: {
key: 'four',
content: 'four',
weight: 8
} as Entry
}
};
const TEST_LEAVES = {
four: {
// expected width difference: 5 (2: total size, 2: weight, 1: entry count)
// -totalLen- -weight- -type=leaf + size-
compressed: `${compressNumber(13, 2)}${compressNumber(8, 2)}${compressNumber(0x8000 + 1, 1)}${TEST_ENTRIES.four.compressed}`,
decompressed: {
type: 'leaf',
weight: 8,
entries: [TEST_ENTRIES.four.decompressed]
} as Leaf,
original: {
type: 'leaf',
weight: 8,
entries: [TEST_ENTRIES.four.original]
} as Leaf
}
}
const TEST_NODES = {
four: {
// expected width difference: 6 (2: total size, 2: weight, 1: entry count, 1: value count)
// -totalLen- -weight- -type/size-
compressed: `${compressNumber(19, 2)}${compressNumber(8, 2)}${compressNumber(1)}r${TEST_LEAVES.four.compressed}`,
decompressed: {
type: 'internal',
weight: 8,
values: ['r'],
children: {r: TEST_LEAVES.four.compressed}
} as InternalNode,
original: {
type: 'internal',
weight: 8,
values: ['r'],
children: {r: TEST_LEAVES.four.original}
} as InternalNode
}
}
const TEST_DATA = {
ENTRIES: TEST_ENTRIES,
LEAVES: TEST_LEAVES,
NODES: TEST_NODES
};
describe('Trie compression', function() {
describe('`number`s', () => {
it('uses single-char compression by default', () => {
assert.equal(compressNumber(0x0020).length, 1);
});
it('compresses properly when targeting single-char width', () => {
assert.equal(compressNumber(0x0020, 1), String.fromCharCode(0x0020 + ENCODED_NUM_BASE));
assert.equal(compressNumber('"'.charCodeAt(0), 1), String.fromCharCode('"'.charCodeAt(0) + ENCODED_NUM_BASE));
});
it('has non-null leading char for numbers needing two-char representations', () => {
assert.notEqual(compressNumber(0x00200020, 2).charAt(0), String.fromCharCode(0));
assert.notEqual(
compressNumber(0x0321ad20, 2).charAt(0), String.fromCharCode(0)
);
});
it('width 2: compressing values one-char wide', () => {
assert.equal(compressNumber(0x0020, 2), `${String.fromCharCode(ENCODED_NUM_BASE)}${String.fromCharCode(0x0020 + ENCODED_NUM_BASE)}`);
});
it('throws when numbers are too large for the specified width', () => {
assert.throws(() => compressNumber(0x00200020, 1));
assert.throws(() => compressNumber(0x002000200020, 2));
})
});
describe('`Entry`s', () => {
it('compresses properly', () => {
assert.equal(compressEntry(TEST_DATA.ENTRIES.four.original), TEST_DATA.ENTRIES.four.compressed);
});
});
describe('Leaf nodes', () => {
it('compresses itself and represented entries', () => {
assert.equal(compressNode(TEST_DATA.LEAVES.four.original), TEST_DATA.LEAVES.four.compressed);
});
});
describe('Internal nodes', () => {
it('compresses (mocked Leaf)', () => {
// Should not attempt to recompress the mock-compressed leaf.
assert.equal(compressNode(TEST_DATA.NODES.four.decompressed), TEST_DATA.NODES.four.compressed);
});
it('compresses (unmocked Leaf)', () => {
assert.equal(compressNode(TEST_DATA.NODES.four.original), TEST_DATA.NODES.four.compressed);
});
});
});
describe('Trie decompression', function () {
describe('`number`s', () => {
describe('not inlined', () => {
it('decompresses single-char strings', () => {
assert.equal(decompressNumber(String.fromCharCode(0x0020 + ENCODED_NUM_BASE), 0), 0x0020);
assert.equal(decompressNumber(String.fromCharCode('"'.charCodeAt(0) + ENCODED_NUM_BASE), 0), '"'.charCodeAt(0));
assert.equal(decompressNumber('\ufffe', 0), 0xfffe - ENCODED_NUM_BASE);
});
it('decompresses two-char strings of one-char value width', () => {
assert.equal(decompressNumber(`${String.fromCharCode(ENCODED_NUM_BASE)}${String.fromCharCode(0x0020 + ENCODED_NUM_BASE)}`, 0), 0x0020);
});
});
describe('with mock-inlining', () => {
it('decompresses single-char strings', () => {
assert.equal(decompressNumber(`xxx${String.fromCharCode(0x0020 + ENCODED_NUM_BASE)}xx`, 3, 4), 0x0020);
assert.equal(decompressNumber(`xx${String.fromCharCode('"'.charCodeAt(0) + ENCODED_NUM_BASE)}x`, 2, 3), '"'.charCodeAt(0));
assert.equal(decompressNumber('\uffff\ufffe', 1), 0xfffe - ENCODED_NUM_BASE);
});
it('decompresses two-char strings', () => {
assert.equal(decompressNumber(`xxxx${compressNumber(0x00200020, 2)}xx`, 4, 6), 0x00200020);
});
});
});
describe('`Entry`s', () => {
it('not inlined', () => {
const mockedDecompression = TEST_DATA.ENTRIES.four.decompressed;
const compressionSrc = TEST_DATA.ENTRIES.four.compressed;
assert.deepEqual(decompressEntry(compressionSrc, identityKey), mockedDecompression);
});
it('inlined', () => {
const mockedDecompression = TEST_DATA.ENTRIES.four.decompressed;
// total length: header = 5, text = 8 -> 13.
const compressionSrc = `xxxxx${TEST_DATA.ENTRIES.four.compressed}xx`;
assert.deepEqual(decompressEntry(compressionSrc, identityKey, /* start index */ 5), mockedDecompression);
});
});
describe('Leaf nodes', () => {
describe('bootstrapping cases', () => {
it('not inlined', () => {
const encodedLeaf = TEST_DATA.LEAVES.four.compressed;
assert.deepEqual(decompressNode(encodedLeaf, identityKey, 0), TEST_DATA.LEAVES.four.decompressed);
});
it('inlined', () => {
const encodedLeaf = TEST_DATA.LEAVES.four.compressed;
assert.deepEqual(decompressNode(`xxxxxxxxx${encodedLeaf}xx`, identityKey, 9), TEST_DATA.LEAVES.four.decompressed);
});
});
});
describe('Internal nodes', () => {
describe('bootstrapping cases', () => {
it('not inlined', () => {
const encodedNode = TEST_DATA.NODES.four.compressed;
assert.deepEqual(decompressNode(encodedNode, identityKey, 0) as InternalNode, TEST_DATA.NODES.four.decompressed);
});
it('inlined', () => {
const encodedNode = TEST_DATA.NODES.four.compressed;
assert.deepEqual(decompressNode(`xxxxxxx${encodedNode}xx`, identityKey, 7) as InternalNode, TEST_DATA.NODES.four.decompressed);
});
});
});
it('compresses fixture successfully: english-1000', () => {
const trieFixture = jsonFixture('tries/english-1000');
const trie = new TrieBuilder(identityKey, trieFixture.root, trieFixture.totalWeight);
assert.doesNotThrow(() => { return {
// The encoding pattern used above achives FAR better compression than
// JSON.stringify, which \u-escapes most chars. As of the commit when
// this was written, before we stopped storing 'key' for entries...
// - length of encoding below: 26097
// - JSON-encoding length: 69122
// - Source fixture's filesize: 141309 bytes
root: `\`${trie.compress()}\``,
totalWeight: trie.getTotalWeight()
} });
// The test: did it throw? If no, we'll assume we're good.
});
describe('surrogate-pair handling', () => {
it('compresses a Trie with non-BMP characters without throwing an error', () => {
const builder = new TrieBuilder(identityKey);
smpWordlist.forEach((tuple) => builder.addEntry(tuple[0], tuple[1]));
assert.doesNotThrow(() => builder.compress());
});
it('properly round-trips a Trie through compression and decompression', () => {
const builder = new TrieBuilder(lowercaseKey);
smpWordlist.forEach((tuple) => builder.addEntry(tuple[0], tuple[1]));
const compressedTrie = builder.compress();
const root = decompressNode(compressedTrie, lowercaseKey) as InternalNode;
assert.equal(root.weight, smpWordlist[0][1]);
assert.sameDeepMembers(
root.values, [
'c', /* keyed to lowercase */
'🙄'.charAt(0), /* is non-BMP, shares same high surrogate as '😇' */
'🇸'.charAt(0), /* is non-BMP, uses a different high surrogate */
'u' /* was already lowercase */
]
);
root.values.forEach((key) => assert.isOk(root.children[key]));
const roundtrippedTrie = new Trie(root, builder.getTotalWeight(), lowercaseKey);
// Decompresses the path traversed; we wish to ensure it's usable even if
// initially compressed.
const uNode = roundtrippedTrie.traverseFromRoot().child('u');
assert.isOk(uNode);
const uEntries = uNode.entries;
assert.isOk(uEntries);
assert.sameDeepMembers(uEntries, [{text: 'u', p: 1.0 / builder.getTotalWeight()}]);
// Tests a simple surrogate-pair path; this path must also cross an internal node
// before reaching a leaf.
const emojiNode = roundtrippedTrie.traverseFromRoot().child('🙄');
assert.isOk(emojiNode);
const emojiEntries = emojiNode.entries;
assert.isOk(emojiEntries);
assert.sameDeepMembers(emojiEntries.map((entry) => entry.text), ['🙄']);
// This sequence tests that keying is fully in-place, even if keys aren't
// directly emitted.
const crazNode = roundtrippedTrie.traverseFromRoot().child('craz');
// There is no internal node at this position; it's an intermediate stage
// of traversal based upon the key in the leaf's entry.
assert.isOk(crazNode);
const crazyNode = crazNode.child('🤪');
assert.isOk(crazyNode);
const crazyEntries = crazyNode.entries;
assert.isOk(crazyEntries);
assert.sameDeepMembers(crazyEntries.map((entry) => entry.text), ['CRAZ🤪']);
});
});
});

2
web/src/test/auto/resources/fixtures/tries/english-1000.json
View file

File diff suppressed because one or more lines are too long