diff --git a/common/web/types/src/lexical-model-types.ts b/common/web/types/src/lexical-model-types.ts index 82619b4ce4..a3c04063fd 100644 --- a/common/web/types/src/lexical-model-types.ts +++ b/common/web/types/src/lexical-model-types.ts @@ -32,30 +32,37 @@ export type TextWithProbability = { */ export interface LexiconTraversal { /** - * Provides an iterable pattern used to search for words with a 'keyed' prefix matching - * the current traversal state's prefix when a new character is appended. Iterating - * across `children` provides 'breadth' to a lexical search. + * Provides an iterable pattern used to search for words with a 'keyed' prefix + * matching the current traversal state's prefix when a new character is + * appended. Iterating across `children` provides 'breadth' to a lexical + * search. * - * For an example with English, if the current traversal state corresponds to 'th', - * children() may return an iterator with states corresponding 'e' - * (for 'the', 'then', 'there'), 'a' (for 'than', 'that'), etc. + * For an example with English, if the current traversal state corresponds to + * 'th', children() may return an iterator with states corresponding 'e' (for + * 'the', 'then', 'there'), 'a' (for 'than', 'that'), etc. * - * @param char A full character (a UTF-16 code point, which may be comprised of two - * code units) corresponding to the child node, appended to the current - * node's prefix to produce the child node's prefix. + * @param char A full character (a UTF-16 code point, which may be + * comprised of two code units) corresponding to the child + * node, appended to the current node's prefix to produce the + * child node's prefix. *

* Example: if the current traversal's represented prefix is 'th', * char = 'e' would indicate a child node with prefix = 'the'. - * @param traversal a LexiconTraversal starting from (or "rooted at") the child node. Use - * of the returned object provides 'depth' to a lexical search. + * @param traversal a LexiconTraversal starting from (or "rooted at") the + * child node. Use of the returned object provides 'depth' + * to a lexical search. *

* Example: * * - Suppose our current LexiconTraversal represents a prefix of 'th'. * - If `char` = 'e', the child represents a prefix of 'the'. - * - Then `traversal` allows traversing the part of the lexicon prefixed by 'the'. + * - Then `traversal` allows traversing the part of the lexicon prefixed by + * 'the'. + * + * Note that there is no requirement for the ordering of the returned entries; + * no sorting of any kind is required. */ - children(): Generator<{char: string, traversal: () => LexiconTraversal}>; + children(): {char: string, p: number, traversal: () => LexiconTraversal}[]; /** * Allows direct access to the traversal state that results when appending one diff --git a/developer/src/kmc-model/build.sh b/developer/src/kmc-model/build.sh index 4b5f15cc30..9cefefa6e7 100755 --- a/developer/src/kmc-model/build.sh +++ b/developer/src/kmc-model/build.sh @@ -11,8 +11,10 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" builder_describe "Keyman kmc Lexical Model Compiler module" \ "@/common/web/keyman-version" \ + "@/common/web/types" \ "@/developer/src/common/web/test-helpers" \ - "@/web/src/engine/predictive-text/templates/ test" \ + "@/developer/src/common/web/utils" \ + "@/web/src/engine/predictive-text/templates/" \ "clean" \ "configure" \ "build" \ diff --git a/developer/src/kmc-model/package.json b/developer/src/kmc-model/package.json index 52ef40c425..f306a85bb7 100644 --- a/developer/src/kmc-model/package.json +++ b/developer/src/kmc-model/package.json @@ -32,11 +32,13 @@ "dependencies": { "@keymanapp/common-types": "*", "@keymanapp/keyman-version": "*", + "@keymanapp/models-templates": "*", "typescript": "^5.4.5" }, "devDependencies": { "@keymanapp/developer-test-helpers": "*", "@keymanapp/models-templates": "*", + "@types/node": "^20.4.1", "c8": "^7.12.0", "chalk": "^2.4.2", "esbuild": "^0.25.0" diff --git a/developer/src/kmc-model/src/build-trie.ts b/developer/src/kmc-model/src/build-trie.ts index e65b2fc42c..8473ef5c23 100644 --- a/developer/src/kmc-model/src/build-trie.ts +++ b/developer/src/kmc-model/src/build-trie.ts @@ -1,6 +1,8 @@ import { ModelCompilerError, ModelCompilerMessageContext, ModelCompilerMessages } from "./model-compiler-messages.js"; import { callbacks } from "./compiler-callbacks.js"; +import { TrieBuilder } from '@keymanapp/models-templates'; + // Supports LF or CRLF line terminators. const NEWLINE_SEPARATOR = /\u000d?\u000a/; @@ -29,7 +31,7 @@ export function createTrieDataStructure(filenames: string[], searchTermToKey?: ( const wordlist: WordList = {}; filenames.forEach(filename => parseWordListFromFilename(wordlist, filename)); - const trie = Trie.buildTrie(wordlist, searchTermToKey as Trie.SearchTermToKey); + const trie = buildTrie(wordlist, searchTermToKey as SearchTermToKey); return JSON.stringify(trie); } @@ -188,302 +190,51 @@ function* enumerateLines(lines: string[]): Generator { } } -namespace Trie { - /** - * An **opaque** type for a string that is exclusively used as a search key in - * the trie. There should be a function that converts arbitrary strings - * (queries) and converts them into a standard search key for a given language - * model. - * - * Fun fact: This opaque type has ALREADY saved my bacon and found a bug! - */ - type SearchKey = string & { _: 'SearchKey'}; +/** + * An **opaque** type for a string that is exclusively used as a search key in + * the trie. There should be a function that converts arbitrary strings + * (queries) and converts them into a standard search key for a given language + * model. + * + * Fun fact: This opaque type has ALREADY saved my bacon and found a bug! + */ +type SearchKey = string & { _: 'SearchKey'}; - /** - * A function that converts a string (word form or query) into a search key - * (secretly, this is also a string). - */ - export interface SearchTermToKey { - (wordform: string): SearchKey; +/** + * A function that converts a string (word form or query) into a search key + * (secretly, this is also a string). + */ +export interface SearchTermToKey { + (wordform: string): SearchKey; +} + +/** + * Builds a trie from a word list. + * + * @param wordlist The wordlist with non-negative weights. + * @param keyFunction Function that converts word forms into indexed search keys + * @returns A JSON-serialiable object that can be given to the TrieModel constructor. + */ +export function buildTrie(wordlist: WordList, keyFunction: SearchTermToKey): object { + const collater = new TrieBuilder(keyFunction); + + buildFromWordList(collater, wordlist); + return { + totalWeight: collater.getTotalWeight(), + root: collater.getRoot() } +} - // The following trie implementation has been (heavily) derived from trie-ing - // by Conrad Irwin. - // - // trie-ing is distributed under the terms of the MIT license, reproduced here: - // - // The MIT License - // Copyright (c) 2015-2017 Conrad Irwin - // Copyright (c) 2011 Marc Campbell - // - // Permission is hereby granted, free of charge, to any person obtaining a copy - // of this software and associated documentation files (the "Software"), to deal - // in the Software without restriction, including without limitation the rights - // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - // copies of the Software, and to permit persons to whom the Software is - // furnished to do so, subject to the following conditions: - // - // The above copyright notice and this permission notice shall be included in - // all copies or substantial portions of the Software. - // - // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - // THE SOFTWARE. - // - // See: https://github.com/ConradIrwin/trie-ing/blob/df55d7af7068d357829db9e0a7faa8a38add1d1d/LICENSE - - /** - * An entry in the prefix trie. The matched word is "content". - */ - interface Entry { - content: string; - key: SearchKey; - weight: number; - } - - /** - * The trie is made up of nodes. A node can be EITHER an internal node (whose - * only children are other nodes) OR a leaf, which actually contains the word - * form entries. - */ - type Node = InternalNode | Leaf; - - /** - * An internal node. - */ - interface InternalNode { - type: 'internal'; - weight: number; - // TODO: As an optimization, "values" can be a single string! - values: string[]; - children: { [codeunit: string]: Node }; - unsorted?: true; - } - - /** - * A leaf node. - */ - interface Leaf { - type: 'leaf'; - weight: number; - entries: Entry[]; - unsorted?: true; - } - - /** - * A sentinel value for when an internal node has contents and requires an - * "internal" leaf. That is, this internal node has content. Instead of placing - * entries as children in an internal node, a "fake" leaf is created, and its - * key is this special internal value. - * - * The value is a valid Unicode BMP code point, but it is a "non-character". - * Unicode will never assign semantics to these characters, as they are - * intended to be used internally as sentinel values. - */ - const INTERNAL_VALUE = '\uFDD0'; - - /** - * Builds a trie from a word list. - * - * @param wordlist The wordlist with non-negative weights. - * @param keyFunction Function that converts word forms into indexed search keys - * @returns A JSON-serialiable object that can be given to the TrieModel constructor. - */ - export function buildTrie(wordlist: WordList, keyFunction: SearchTermToKey): object { - const root = new Trie(keyFunction).buildFromWordList(wordlist).root; - return { - totalWeight: sumWeights(root), - root: root - } - } - - /** - * Wrapper class for the trie and its nodes and wordform to search - */ - class Trie { - readonly root = createRootNode(); - toKey: SearchTermToKey; - constructor(wordform2key: SearchTermToKey) { - this.toKey = wordform2key; - } - - /** - * Populates the trie with the contents of an entire wordlist. - * @param words a list of word and count pairs. - */ - buildFromWordList(words: WordList): Trie { - for (const [wordform, weight] of Object.entries(words)) { - const key = this.toKey(wordform); - addUnsorted(this.root, { key, weight, content: wordform }, 0); - } - sortTrie(this.root); - return this; - } - } - - // "Constructors" - function createRootNode(): Node { - return { - type: 'leaf', - weight: 0, - entries: [] - }; - } - - // Implement Trie creation. - - /** - * Adds an entry to the trie. - * - * Note that the trie will likely be unsorted after the add occurs. Before - * performing a lookup on the trie, use call sortTrie() on the root note! - * - * @param node Which node should the entry be added to? - * @param entry the wordform/weight/key to add to the trie - * @param index the index in the key and also the trie depth. Should be set to - * zero when adding onto the root node of the trie. - */ - function addUnsorted(node: Node, entry: Entry, index: number = 0) { - // Each node stores the MAXIMUM weight out of all of its decesdents, to - // enable a greedy search through the trie. - node.weight = Math.max(node.weight, entry.weight); - - // When should a leaf become an interior node? - // When it already has a value, but the key of the current value is longer - // than the prefix. - if (node.type === 'leaf' && index < entry.key.length && node.entries.length >= 1) { - convertLeafToInternalNode(node, index); - } - - if (node.type === 'leaf') { - // The key matches this leaf node, so add yet another entry. - addItemToLeaf(node, entry); - } else { - // Push the node down to a lower node. - addItemToInternalNode(node, entry, index); - } - - node.unsorted = true; - } - - /** - * Adds an item to the internal node at a given depth. - * @param item - * @param index - */ - function addItemToInternalNode(node: InternalNode, item: Entry, index: number) { - let char = item.key[index]; - // If an internal node is the proper site for item, it belongs under the - // corresponding (sentinel, internal-use) child node signifying this. - if(char == undefined) { - char = INTERNAL_VALUE; - } - if (!node.children[char]) { - node.children[char] = createRootNode(); - node.values.push(char); - } - addUnsorted(node.children[char], item, index + 1); - } - - function addItemToLeaf(leaf: Leaf, item: Entry) { - leaf.entries.push(item); - } - - /** - * Mutates the given Leaf to turn it into an InternalNode. - * - * NOTE: the node passed in will be DESTRUCTIVELY CHANGED into a different - * type when passed into this function! - * - * @param depth depth of the trie at this level. - */ - function convertLeafToInternalNode(leaf: Leaf, depth: number): void { - const entries = leaf.entries; - - // Alias the current node, as the desired type. - const internal = ( leaf) as InternalNode; - internal.type = 'internal'; - - delete leaf.entries; - internal.values = []; - internal.children = {}; - - // Convert the old values array into the format for interior nodes. - for (const item of entries) { - let char: string; - if (depth < item.key.length) { - char = item.key[depth]; - } else { - char = INTERNAL_VALUE; - } - - if (!internal.children[char]) { - internal.children[char] = createRootNode(); - internal.values.push(char); - } - addUnsorted(internal.children[char], item, depth + 1); - } - - internal.unsorted = true; - } - - /** - * Recursively sort the trie, in descending order of weight. - * @param node any node in the trie - */ - function sortTrie(node: Node) { - if (node.type === 'leaf') { - if (!node.unsorted) { - return; - } - - node.entries.sort(function (a, b) { return b.weight - a.weight; }); - } else { - // We MUST recurse and sort children before returning. - for (const char of node.values) { - sortTrie(node.children[char]); - } - - if (!node.unsorted) { - return; - } - - node.values.sort((a, b) => { - return node.children[b].weight - node.children[a].weight; - }); - } - - delete node.unsorted; - } - - /** - * O(n) recursive traversal to sum the total weight of all leaves in the - * trie, starting at the provided node. - * - * @param node The node to start summing weights. - */ - function sumWeights(node: Node): number { - let val: number; - if (node.type === 'leaf') { - val = node.entries - .map(entry => entry.weight) - //.map(entry => isNaN(entry.weight) ? 1 : entry.weight) - .reduce((acc, count) => acc + count, 0); - } else { - val = Object.keys(node.children) - .map((key) => sumWeights(node.children[key])) - .reduce((acc, count) => acc + count, 0); - } - - if(isNaN(val)) { - throw new Error("Unexpected NaN has appeared!"); - } - return val; +/** + * Populates the trie with the contents of an entire wordlist. + * @param words a list of word and count pairs. + */ +function buildFromWordList(trieCollator: TrieBuilder, words: WordList): TrieBuilder { + for (const [wordform, weight] of Object.entries(words)) { + trieCollator.addEntry(wordform, weight); } + trieCollator.sort(); + return trieCollator; } /** diff --git a/package-lock.json b/package-lock.json index c05360af53..70766e139d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -804,11 +804,13 @@ "dependencies": { "@keymanapp/common-types": "*", "@keymanapp/keyman-version": "*", + "@keymanapp/models-templates": "*", "typescript": "^5.4.5" }, "devDependencies": { "@keymanapp/developer-test-helpers": "*", "@keymanapp/models-templates": "*", + "@types/node": "^20.4.1", "c8": "^7.12.0", "chalk": "^2.4.2", "esbuild": "^0.25.0" diff --git a/web/src/engine/predictive-text/templates/src/common.ts b/web/src/engine/predictive-text/templates/src/common.ts index f23ad6095b..1595b3f2e9 100644 --- a/web/src/engine/predictive-text/templates/src/common.ts +++ b/web/src/engine/predictive-text/templates/src/common.ts @@ -157,3 +157,19 @@ export function defaultApplyCasing(casing: CasingForm, text: string): string { return text.substring(0, headUnitLength).toUpperCase() .concat(text.substring(headUnitLength)); } } + +/** + * An **opaque** type for a string that is exclusively used as a search key in + * the trie. There should be a function that converts arbitrary strings + * (queries) and converts them into a standard search key for a given language + * model. + */ +export type SearchKey = string & { _: 'SearchKey'}; + +/** + * A function that converts a string (word form or query) into a search key + * (which, internally, is also a string type). + */ +export interface Wordform2Key { + (wordform: string): SearchKey; +} diff --git a/web/src/engine/predictive-text/templates/src/index.ts b/web/src/engine/predictive-text/templates/src/index.ts index 0966fb4494..fb681472fa 100644 --- a/web/src/engine/predictive-text/templates/src/index.ts +++ b/web/src/engine/predictive-text/templates/src/index.ts @@ -1,7 +1,12 @@ export { - SENTINEL_CODE_UNIT, applyTransform, buildMergedTransform, isHighSurrogate, isLowSurrogate, isSentinel, - transformToSuggestion, defaultApplyCasing + applyTransform, buildMergedTransform, isHighSurrogate, isLowSurrogate, isSentinel, + SearchKey, SENTINEL_CODE_UNIT, transformToSuggestion, defaultApplyCasing } from "./common.js"; export { default as QuoteBehavior } from "./quote-behavior.js"; export { getLastPreCaretToken, Token, Tokenization, tokenize, wordbreak } from "./tokenization.js"; +export { + Entry, InternalNode, Leaf, Node +} from './trie.js'; +export { TrieBuilder } from './trie-builder.js'; +export * as trieConstruction from './trie-builder.js'; export { default as TrieModel, TrieModelOptions } from "./trie-model.js"; \ No newline at end of file diff --git a/web/src/engine/predictive-text/templates/src/traversal-model.ts b/web/src/engine/predictive-text/templates/src/traversal-model.ts new file mode 100644 index 0000000000..a81f45d277 --- /dev/null +++ b/web/src/engine/predictive-text/templates/src/traversal-model.ts @@ -0,0 +1,297 @@ +/* + * Copyright (c) 2019 National Research Council Canada (author: Eddie A. Santos) + * Copyright (c) 2019 SIL International + * Copyright (c) 2015–2017 Conrad Irwin + * Copyright (c) 2011–2015 Marc Campbell + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +// Allows the kmwstring bindings to resolve. +import { KMWString, PriorityQueue } from "@keymanapp/web-utils"; +import { default as defaultWordBreaker } from "@keymanapp/models-wordbreakers"; + +import { applyTransform, SearchKey, transformToSuggestion, Wordform2Key } from "./common.js"; +import { getLastPreCaretToken } from "./tokenization.js"; +import { LexicalModelTypes } from "@keymanapp/common-types"; +import Capabilities = LexicalModelTypes.Capabilities; +import CasingFunction = LexicalModelTypes.CasingFunction; +import Configuration = LexicalModelTypes.Configuration; +import Context = LexicalModelTypes.Context; +import Distribution = LexicalModelTypes.Distribution; +import LexicalModel = LexicalModelTypes.LexicalModel; +import LexicalModelPunctuation = LexicalModelTypes.LexicalModelPunctuation; +import LexiconTraversal = LexicalModelTypes.LexiconTraversal; +import Suggestion = LexicalModelTypes.Suggestion; +import TextWithProbability = LexicalModelTypes.TextWithProbability; +import Transform = LexicalModelTypes.Transform; +import WithOutcome = LexicalModelTypes.WithOutcome; +import WordBreakingFunction = LexicalModelTypes.WordBreakingFunction; + +/** + * @file traversal-model.ts + * + * Defines a lexical model based upon the implementation of a `LexiconTraversal`. + */ + +/** Upper bound on the amount of suggestions to generate. */ +const MAX_SUGGESTIONS = 12; + +/** + * Additional arguments to pass into the model, in addition to the model + * parameters themselves. + */ +export interface ModelOptions { + /** + * How to break words in a phrase. + */ + wordBreaker?: WordBreakingFunction; + /** + * This should simplify a search term into a key. + */ + searchTermToKey?: (searchTerm: string) => string; + + /** + * Indicates that the model's written form has 'casing' behavior. + */ + languageUsesCasing?: boolean; + + /** + * Specifies a function used to apply the language's casing rules to a word. + */ + applyCasing?: CasingFunction; + + /** + * Any punctuation to expose to the user. + */ + punctuation?: LexicalModelPunctuation; +} + +/** + * @class TraversalModel + * + * Defines a lexical model based upon the implementation of a `LexiconTraversal`. + */ +export class TraversalModel implements LexicalModel { + configuration?: Configuration; + private rootTraversal: LexiconTraversal; + readonly breakWords: WordBreakingFunction; + readonly punctuation?: LexicalModelPunctuation; + readonly languageUsesCasing?: boolean; + readonly _toKey: Wordform2Key; + + readonly applyCasing?: CasingFunction; + + constructor(rootTraversal: LexiconTraversal, options: ModelOptions = {}) { + this.languageUsesCasing = options.languageUsesCasing; + this.applyCasing = options.applyCasing; + + this.rootTraversal = rootTraversal; + this._toKey = options.searchTermToKey as Wordform2Key || defaultSearchTermToKey + this.breakWords = options.wordBreaker || defaultWordBreaker; + this.punctuation = options.punctuation; + } + + configure(capabilities: Capabilities): Configuration { + return this.configuration = { + leftContextCodePoints: capabilities.maxLeftContextCodePoints, + rightContextCodePoints: capabilities.maxRightContextCodePoints ?? 0 + }; + } + + toKey(text: string): string { + return this._toKey(text); + } + + predict(transform: Transform, context: Context): Distribution { + const makeDistribution = function(suggestions: WithOutcome[]): Distribution { + let distribution: Distribution = []; + + for(const s of suggestions) { + distribution.push({sample: s, p: s.p}); + } + + return distribution; + } + + // Special-case the empty buffer/transform: return the top suggestions. + if (!transform.insert && !context.left && !context.right && context.startOfBuffer && context.endOfBuffer) { + return makeDistribution(this.firstN(MAX_SUGGESTIONS).map(({text, p}) => ({ + transform: { + insert: text, + deleteLeft: 0 + }, + displayAs: text, + p: p + }))); + } + + // Compute the results of the keystroke: + const newContext = applyTransform(transform, context); + + // Computes the different in word length after applying the transform above. + const leftDelOffset = transform.deleteLeft - KMWString.length(transform.insert); + + // All text to the left of the cursor INCLUDING anything that has + // just been typed. + const prefix = getLastPreCaretToken(this.breakWords, newContext); + + // Return suggestions from the trie. + return makeDistribution(this.lookup(prefix).map(({text, p}) => + transformToSuggestion({ + insert: text, + // Delete whatever the prefix that the user wrote. + deleteLeft: leftDelOffset + KMWString.length(prefix) + // Note: a separate capitalization/orthography engine can take this + // result and transform it as needed. + }, + p + ))); + } + + get wordbreaker(): WordBreakingFunction { + return this.breakWords; + } + + public traverseFromRoot(): LexiconTraversal { + return this.rootTraversal; + } + + /** + * Returns the top N suggestions from the trie. + * @param n How many suggestions, maximum, to return. + */ + firstN(n: number): TextWithProbability[] { + return getSortedResults(this.rootTraversal, n); + } + + /** + * Lookups an arbitrary prefix (a query) in the trie. Returns the top 3 + * results in sorted order. + * + * @param prefix + */ + lookup(prefix: string): TextWithProbability[] { + const searchKey = this.toKey(prefix); + const rootTraversal = this.traverseFromRoot().child(searchKey); + + if(!rootTraversal) { + return []; + } + + const directEntries = rootTraversal.entries; + // `Set` requires Chrome 38+, which is more recent than Chrome 35. + const directSet: Record = {}; + for(const entry of directEntries) { + directSet[entry.text] = entry.text; + } + + const bestEntries = getSortedResults(rootTraversal); + const deduplicated = bestEntries.filter((entry) => !directSet[entry.text]); + + // Any entries directly hosted on the current node should get full display + // priority over anything from its descendants. + return directEntries.concat(deduplicated); + } +}; + +/** + * The priority queue will always pop the most probable item - be it a Traversal + * state or a lexical entry reached via Traversal. + */ +type TraversableWithProb = TextWithProbability | LexiconTraversal; + +/** + * Returns all entries matching the given prefix, in descending order of + * weight. + * + * @param prefix the prefix to match. + * @param results the current results + * @param queue + */ +function getSortedResults(traversal: LexiconTraversal, limit = MAX_SUGGESTIONS): TextWithProbability[] { + let queue = new PriorityQueue(function(a: TraversableWithProb, b: TraversableWithProb) { + // In case of Trie compilation issues that emit `null` or `undefined` + return (b ? b.p : 0) - (a ? a.p : 0); + }); + let results: TextWithProbability[] = []; + + queue.enqueue(traversal); + + while(queue.count > 0) { + const entry = queue.dequeue(); + + if((entry as TextWithProbability)!.text !== undefined) { + const lexicalEntry = entry as TextWithProbability; + results.push(lexicalEntry); + if(results.length >= limit) { + return results; + } + } else { + const traversal = entry as LexiconTraversal; + queue.enqueueAll(traversal.entries); + let children: LexiconTraversal[] = [] + for(let child of traversal.children()) { + children.push(child.traversal()); + } + queue.enqueueAll(children); + } + } + + return results; +} + +/** + * Converts wordforms into an indexable form. It does this by + * normalizing into NFD, removing diacritics, and then converting + * the result to lowercase. + * + * This is a very naïve implementation, that I only think will work on + * some languages that use the Latin script. As of 2020-04-08, only + * 4 out of 11 (36%) of published language models use the Latin script, + * so this might not actually be a great default. + * + * This uses String.prototype.normalize() to convert normalize into NFD. + * NFD is an easy way to separate a Latin character from its diacritics; + * Even then, some Latin-based orthographies use code points that, + * under NFD normalization, do NOT decompose into an ASCII letter and a + * combining diacritical mark (e.g., SENĆOŦEN). + * + * Use this only in early iterations of the model. For a production lexical + * model, you SHOULD write/generate your own key function, tailored to your + * language. + */ +function defaultSearchTermToKey(wordform: string): SearchKey { + /** + * N.B.: this is (slightly) DIFFERENT than the version in + * keymanapp/lexical-model-compiler/build-trie + * as this is for compatibility for models built + * BEFORE the searchTermToKey function was bundled with + * all models. + * + * This compatibility version lowercases AFTER removing diacritics; + * the new version (bundled in future models) lowercases, + * NFD normalizes, THEN removes diacritics. + */ + return wordform + .normalize('NFD') + // Remove all combining diacritics (if input is in NFD) + // common to Latin-orthographies. + .replace(/[\u0300-\u036f]/g, '') + .toLowerCase() as SearchKey; +} diff --git a/web/src/engine/predictive-text/templates/src/trie-builder.ts b/web/src/engine/predictive-text/templates/src/trie-builder.ts new file mode 100644 index 0000000000..790923a1e4 --- /dev/null +++ b/web/src/engine/predictive-text/templates/src/trie-builder.ts @@ -0,0 +1,146 @@ +import { SENTINEL_CODE_UNIT, Wordform2Key } from "./common.js"; +import { Entry, InternalNode, Leaf, Node, Trie, sortNode } from "./trie.js"; + +export function createRootNode(): Node { + return { + type: 'leaf', + weight: 0, + entries: [] + }; +} + +/** + * Adds an entry to the trie. Currently assumes there is no pre-existing match + * for the entry. + * + * Note that the trie will likely be unsorted after the add occurs. Before + * performing a lookup on the trie, use call sortTrie() on the root note! + * + * @param node Which node should the entry be added to? + * @param entry the wordform/weight/key to add to the trie + * @param index the index in the key and also the trie depth. Should be set to + * zero when adding onto the root node of the trie. + */ +export function addUnsorted(node: Node, entry: Entry, index: number = 0) { + // Each node stores the MAXIMUM weight out of all of its decesdents, to + // enable a greedy search through the trie. + node.weight = Math.max(node.weight, entry.weight); + + // When should a leaf become an interior node? + // When it already has a value, but the key of the current value is longer + // than the prefix. + if (node.type === 'leaf' && index < entry.key.length && node.entries.length >= 1) { + convertLeafToInternalNode(node, index); + } + + if (node.type === 'leaf') { + // The key matches this leaf node, so add yet another entry. + addItemToLeaf(node, entry); + } else { + // Push the node down to a lower node. + addItemToInternalNode(node, entry, index); + } + + node.unsorted = true; +} + +/** + * Adds an item to the internal node at a given depth. + * @param item + * @param index + */ +export function addItemToInternalNode(node: InternalNode, item: Entry, index: number) { + let char = item.key[index]; + // If an internal node is the proper site for item, it belongs under the + // corresponding (sentinel, internal-use) child node signifying this. + if(char == undefined) { + char = SENTINEL_CODE_UNIT; + } + if (!node.children[char]) { + node.children[char] = createRootNode(); + node.values.push(char); + } + addUnsorted(node.children[char], item, index + 1); +} + +export function addItemToLeaf(leaf: Leaf, item: Entry) { + leaf.entries.push(item); +} + +/** + * Mutates the given Leaf to turn it into an InternalNode. + * + * NOTE: the node passed in will be DESTRUCTIVELY CHANGED into a different + * type when passed into this function! + * + * @param depth depth of the trie at this level. + */ +export function convertLeafToInternalNode(leaf: Leaf, depth: number): void { + let entries = leaf.entries; + + // Alias the current node, as the desired type. + let internal = ( leaf) as InternalNode; + internal.type = 'internal'; + + delete (leaf as Partial).entries; + internal.values = []; + internal.children = {}; + + // Convert the old values array into the format for interior nodes. + for (let item of entries) { + let char: string; + if (depth < item.key.length) { + char = item.key[depth]; + } else { + char = SENTINEL_CODE_UNIT; + } + + if (!internal.children[char]) { + internal.children[char] = createRootNode(); + internal.values.push(char); + } + addUnsorted(internal.children[char], item, depth + 1); + } + + internal.unsorted = true; +} + +/** + * Wrapper class for the trie and its nodes. + */ +export class TrieBuilder extends Trie { + /** The total weight of the entire trie. */ + totalWeight: number; + + constructor(toKey: Wordform2Key) { + super(createRootNode(), 0, toKey); + this.totalWeight = 0; + } + + addEntry(word: string, weight?: number) { + weight = (isNaN(weight ?? NaN) || !weight) ? 1 : weight; + this.totalWeight += weight; + + addUnsorted( + this.root, { + key: this.toKey(word), + content: word, + weight: weight + }, + 0 + ); + } + + sort() { + // Sorts the full Trie, not just a part of it. + sortNode(this.root); + } + + getRoot(): Node { + return this.root; + } + + getTotalWeight(): number { + return this.totalWeight; + } +} \ No newline at end of file diff --git a/web/src/engine/predictive-text/templates/src/trie-model.ts b/web/src/engine/predictive-text/templates/src/trie-model.ts index 938553330d..f998d0b131 100644 --- a/web/src/engine/predictive-text/templates/src/trie-model.ts +++ b/web/src/engine/predictive-text/templates/src/trie-model.ts @@ -25,25 +25,15 @@ // Worth noting: we're starting to get quite a 'library' of common model/LMLayer functionality. // Should probably make a 'lm-utils' submodule. -import { KMWString, PriorityQueue } from "@keymanapp/web-utils"; -import { default as defaultWordBreaker } from "@keymanapp/models-wordbreakers"; - -import { applyTransform, isHighSurrogate, isSentinel, SENTINEL_CODE_UNIT, transformToSuggestion } from "./common.js"; -import { getLastPreCaretToken } from "./tokenization.js"; import { LexicalModelTypes } from "@keymanapp/common-types"; -import Capabilities = LexicalModelTypes.Capabilities; import CasingFunction = LexicalModelTypes.CasingFunction; import Configuration = LexicalModelTypes.Configuration; -import Context = LexicalModelTypes.Context; -import Distribution = LexicalModelTypes.Distribution; import LexicalModel = LexicalModelTypes.LexicalModel; import LexicalModelPunctuation = LexicalModelTypes.LexicalModelPunctuation; -import LexiconTraversal = LexicalModelTypes.LexiconTraversal; -import Suggestion = LexicalModelTypes.Suggestion; -import TextWithProbability = LexicalModelTypes.TextWithProbability; -import Transform = LexicalModelTypes.Transform; -import WithOutcome = LexicalModelTypes.WithOutcome; import WordBreakingFunction = LexicalModelTypes.WordBreakingFunction; +import { SearchKey, Wordform2Key } from "./common.js"; +import { Node, Trie, TrieTraversal } from './trie.js'; +import { TraversalModel } from './traversal-model.js'; /** * @file trie-model.ts @@ -51,9 +41,6 @@ import WordBreakingFunction = LexicalModelTypes.WordBreakingFunction; * Defines a simple word list (unigram) model. */ -/** Upper bound on the amount of suggestions to generate. */ -const MAX_SUGGESTIONS = 12; - /** * Additional arguments to pass into the model, in addition to the model * parameters themselves. @@ -84,182 +71,6 @@ export interface TrieModelOptions { punctuation?: LexicalModelPunctuation; } -export class Traversal implements LexiconTraversal { - /** - * The lexical prefix corresponding to the current traversal state. - */ - readonly prefix: string; - - /** - * The current traversal node. Serves as the 'root' of its own sub-Trie, - * and we cannot navigate back to its parent. - */ - readonly root: Node; - - /** - * The max weight for the Trie being 'traversed'. Needed for probability - * calculations. - */ - readonly totalWeight: number; - - constructor(root: Node, prefix: string, totalWeight: number) { - this.root = root; - this.prefix = prefix; - this.totalWeight = totalWeight; - } - - child(char: string): Traversal | undefined { - // May result for blank tokens resulting immediately after whitespace. - if(char == '') { - return this; - } - - // Split into individual code units. - let steps = char.split(''); - let traversal: Traversal | undefined = this; - - while(steps.length > 0 && traversal) { - const step: string = steps.shift()!; - traversal = traversal._child(step); - } - - return traversal; - } - - // Handles one code unit at a time. - private _child(char: string): Traversal | undefined { - const root = this.root; - const totalWeight = this.totalWeight; - const nextPrefix = this.prefix + char; - - if(root.type == 'internal') { - let childNode = root.children[char]; - if(!childNode) { - return undefined; - } - - return new Traversal(childNode, nextPrefix, totalWeight); - } else { - // root.type == 'leaf'; - const legalChildren = root.entries.filter(function(entry) { - return entry.key.indexOf(nextPrefix) == 0; - }); - - if(!legalChildren.length) { - return undefined; - } - - return new Traversal(root, nextPrefix, totalWeight); - } - } - - *children(): Generator<{char: string, traversal: () => Traversal}> { - let root = this.root; - - // We refer to the field multiple times in this method, and it doesn't change. - // This also assists minification a bit, since we can't minify when re-accessing - // through `this.`. - const totalWeight = this.totalWeight; - - if(root.type == 'internal') { - for(let entry of root.values) { - let entryNode = root.children[entry]; - - // UTF-16 astral plane check. - if(isHighSurrogate(entry)) { - // First code unit of a UTF-16 code point. - // For now, we'll just assume the second always completes such a char. - // - // Note: Things get nasty here if this is only sometimes true; in the future, - // we should compile-time enforce that this assumption is always true if possible. - if(entryNode.type == 'internal') { - let internalNode = entryNode; - for(let lowSurrogate of internalNode.values) { - let prefix = this.prefix + entry + lowSurrogate; - yield { - char: entry + lowSurrogate, - traversal: function() { return new Traversal(internalNode.children[lowSurrogate], prefix, totalWeight) } - } - } - } else { - // Determine how much of the 'leaf' entry has no Trie nodes, emulate them. - let fullText = entryNode.entries[0].key; - entry = entry + fullText[this.prefix.length + 1]; // The other half of the non-BMP char. - let prefix = this.prefix + entry; - - yield { - char: entry, - traversal: function () {return new Traversal(entryNode, prefix, totalWeight)} - } - } - } else if(isSentinel(entry)) { - continue; - } else if(!entry) { - // Prevent any accidental 'null' or 'undefined' entries from having an effect. - continue; - } else { - let prefix = this.prefix + entry; - yield { - char: entry, - traversal: function() { return new Traversal(entryNode, prefix, totalWeight)} - } - } - } - - return; - } else { // type == 'leaf' - let prefix = this.prefix; - - let children = root.entries.filter(function(entry) { - return entry.key != prefix && prefix.length < entry.key.length; - }) - - for(let {key} of children) { - let nodeKey = key[prefix.length]; - - if(isHighSurrogate(nodeKey)) { - // Merge the other half of an SMP char in! - nodeKey = nodeKey + key[prefix.length+1]; - } - yield { - char: nodeKey, - traversal: function() { return new Traversal(root, prefix + nodeKey, totalWeight)} - } - }; - return; - } - } - - get entries() { - const entryMapper = (value: Entry) => { - return { - text: value.content, - p: value.weight / this.totalWeight - } - } - - if(this.root.type == 'leaf') { - let prefix = this.prefix; - let matches = this.root.entries.filter(function(entry) { - return entry.key == prefix; - }); - - return matches.map(entryMapper); - } else { - let matchingLeaf = this.root.children[SENTINEL_CODE_UNIT]; - if(matchingLeaf && matchingLeaf.type == 'leaf') { - return matchingLeaf.entries.map(entryMapper); - } else { - return []; - } - } - } - - get p(): number { - return this.root.weight / this.totalWeight; - } -} - /** * @class TrieModel * @@ -269,267 +80,30 @@ export class Traversal implements LexiconTraversal { * prefix searches within words, however they are not very good * at predicting the next word. */ -export default class TrieModel implements LexicalModel { +export default class TrieModel extends TraversalModel implements LexicalModel { configuration?: Configuration; - private _trie: Trie; - readonly breakWords: WordBreakingFunction; + public readonly trie: Trie; readonly punctuation?: LexicalModelPunctuation; readonly languageUsesCasing?: boolean; readonly applyCasing?: CasingFunction; constructor(trieData: {root: Node, totalWeight: number}, options: TrieModelOptions = {}) { - this.languageUsesCasing = options.languageUsesCasing; - this.applyCasing = options.applyCasing; - - this._trie = new Trie( + const trie = new Trie( trieData['root'], trieData['totalWeight'], options.searchTermToKey as Wordform2Key || defaultSearchTermToKey ); - this.breakWords = options.wordBreaker || defaultWordBreaker; - this.punctuation = options.punctuation; + + super(trie.traverseFromRoot(), options); + this.trie = trie; } - configure(capabilities: Capabilities): Configuration { - return this.configuration = { - leftContextCodePoints: capabilities.maxLeftContextCodePoints, - rightContextCodePoints: capabilities.maxRightContextCodePoints ?? 0 - }; - } - - toKey(text: string): string { - return this._trie.toKey(text); - } - - predict(transform: Transform, context: Context): Distribution { - const makeDistribution = function(suggestions: WithOutcome[]): Distribution { - const distribution: Distribution = []; - - for (const s of suggestions) { - distribution.push({ sample: s, p: s.p }); - } - - return distribution; - } - - // Special-case the empty buffer/transform: return the top suggestions. - if (!transform.insert && !context.left && !context.right && context.startOfBuffer && context.endOfBuffer) { - return makeDistribution(this._trie.firstN(MAX_SUGGESTIONS).map(({text, p}) => ({ - transform: { - insert: text, - deleteLeft: 0 - }, - displayAs: text, - p: p - }))); - } - - // Compute the results of the keystroke: - const newContext = applyTransform(transform, context); - - // Computes the different in word length after applying the transform above. - const leftDelOffset = transform.deleteLeft - KMWString.length(transform.insert); - - // All text to the left of the cursor INCLUDING anything that has - // just been typed. - const prefix = getLastPreCaretToken(this.breakWords, newContext); - - // Return suggestions from the trie. - return makeDistribution(this._trie.lookup(prefix).map(({text, p}) => - transformToSuggestion({ - insert: text, - // Delete whatever the prefix that the user wrote. - deleteLeft: leftDelOffset + KMWString.length(prefix) - // Note: a separate capitalization/orthography engine can take this - // result and transform it as needed. - }, - p - ))); - } - - get wordbreaker(): WordBreakingFunction { - return this.breakWords; - } - - public traverseFromRoot(): Traversal { - return this._trie.traverseFromRoot(); + public traverseFromRoot(): TrieTraversal { + return super.traverseFromRoot() as TrieTraversal; } }; -///////////////////////////////////////////////////////////////////////////////// -// What remains in this file is the trie implementation proper. Note: to // -// reduce bundle size, any functions/methods related to creating the trie have // -// been removed. // -///////////////////////////////////////////////////////////////////////////////// - -/** - * An **opaque** type for a string that is exclusively used as a search key in - * the trie. There should be a function that converts arbitrary strings - * (queries) and converts them into a standard search key for a given language - * model. - * - * Fun fact: This opaque type has ALREADY saved my bacon and found a bug! - */ -type SearchKey = string & { _: 'SearchKey'}; - -/** - * The priority queue will always pop the most probable item - be it a Traversal - * state or a lexical entry reached via Traversal. - */ -type TraversableWithProb = TextWithProbability | LexiconTraversal; - -/** - * A function that converts a string (word form or query) into a search key - * (secretly, this is also a string). - */ -interface Wordform2Key { - (wordform: string): SearchKey; -} - -// The following trie implementation has been (heavily) derived from trie-ing -// by Conrad Irwin. -// trie-ing is copyright (C) 2015–2017 Conrad Irwin. -// Distributed under the terms of the MIT license: -// https://github.com/ConradIrwin/trie-ing/blob/df55d7af7068d357829db9e0a7faa8a38add1d1d/LICENSE - -type Node = InternalNode | Leaf; -/** - * An internal node in the trie. Internal nodes NEVER contain entries; if an - * internal node should contain an entry, then it has a dummy leaf node (see - * below), that can be accessed by node.children["\uFDD0"]. - */ -interface InternalNode { - type: 'internal'; - weight: number; - /** Maintains the keys of children in descending order of weight. */ - values: string[]; // TODO: As an optimization, "values" can be a single string! - /** - * Maps a single UTF-16 code unit to a child node in the trie. This child - * node may be a leaf or an internal node. The keys of this object are kept - * in sorted order in the .values array. - */ - children: { [codeunit: string]: Node }; -} -/** Only leaf nodes actually contain entries (i.e., the words proper). */ -interface Leaf { - type: 'leaf'; - weight: number; - entries: Entry[]; -} - -/** - * An entry in the prefix trie (stored in leaf nodes exclusively!) - */ -interface Entry { - /** The actual word form, stored in the trie. */ - content: string; - /** A search key that usually simplifies the word form, for ease of search. */ - key: SearchKey; - weight: number; -} - -/** - * Wrapper class for the trie and its nodes. - */ -class Trie { - public readonly root: Node; - /** The total weight of the entire trie. */ - readonly totalWeight: number; - /** - * Converts arbitrary strings to a search key. The trie is built up of - * search keys; not each entry's word form! - */ - toKey: Wordform2Key; - - constructor(root: Node, totalWeight: number, wordform2key: Wordform2Key) { - this.root = root; - this.toKey = wordform2key; - this.totalWeight = totalWeight; - } - - public traverseFromRoot(): Traversal { - return new Traversal(this.root, '', this.totalWeight); - } - - /** - * Lookups an arbitrary prefix (a query) in the trie. Returns the top 3 - * results in sorted order. - * - * @param prefix - */ - lookup(prefix: string): TextWithProbability[] { - const searchKey = this.toKey(prefix); - const rootTraversal = this.traverseFromRoot().child(searchKey); - - if(!rootTraversal) { - return []; - } - - const directEntries = rootTraversal.entries; - // `Set` requires Chrome 38+, which is more recent than Chrome 35. - const directSet: Record = {}; - for(const entry of directEntries) { - directSet[entry.text] = entry.text; - } - - const bestEntries = getSortedResults(rootTraversal); - const deduplicated = bestEntries.filter((entry) => !directSet[entry.text]); - - // Any entries directly hosted on the current node should get full display - // priority over anything from its descendants. - return directEntries.concat(deduplicated); - } - - /** - * Returns the top N suggestions from the trie. - * @param n How many suggestions, maximum, to return. - */ - firstN(n: number): TextWithProbability[] { - return getSortedResults(this.traverseFromRoot(), n); - } -} - -/** - * Returns all entries matching the given prefix, in descending order of - * weight. - * - * @param prefix the prefix to match. - * @param results the current results - * @param queue - */ -function getSortedResults(traversal: LexiconTraversal, limit = MAX_SUGGESTIONS): TextWithProbability[] { - let queue = new PriorityQueue(function(a: TraversableWithProb, b: TraversableWithProb) { - // In case of Trie compilation issues that emit `null` or `undefined` - return (b ? b.p : 0) - (a ? a.p : 0); - }); - let results: TextWithProbability[] = []; - - queue.enqueue(traversal); - - while(queue.count > 0) { - const entry = queue.dequeue(); - - if((entry as TextWithProbability)!.text !== undefined) { - const lexicalEntry = entry as TextWithProbability; - results.push(lexicalEntry); - if(results.length >= limit) { - return results; - } - } else { - const traversal = entry as LexiconTraversal; - queue.enqueueAll(traversal.entries); - let children: LexiconTraversal[] = [] - for(let child of traversal.children()) { - children.push(child.traversal()); - } - queue.enqueueAll(children); - } - } - - return results; -} - /** * Converts wordforms into an indexable form. It does this by * normalizing into NFD, removing diacritics, and then converting diff --git a/web/src/engine/predictive-text/templates/src/trie.ts b/web/src/engine/predictive-text/templates/src/trie.ts new file mode 100644 index 0000000000..6c5be6c3c1 --- /dev/null +++ b/web/src/engine/predictive-text/templates/src/trie.ts @@ -0,0 +1,308 @@ +import { isHighSurrogate, isSentinel, SearchKey, SENTINEL_CODE_UNIT, Wordform2Key } from "./common.js"; +import { LexicalModelTypes } from "@keymanapp/common-types"; +import LexiconTraversal = LexicalModelTypes.LexiconTraversal; + +// The following trie implementation has been (heavily) derived from trie-ing +// by Conrad Irwin. +// trie-ing is copyright (C) 2015–2017 Conrad Irwin. +// Distributed under the terms of the MIT license: +// https://github.com/ConradIrwin/trie-ing/blob/df55d7af7068d357829db9e0a7faa8a38add1d1d/LICENSE + +export type Node = InternalNode | Leaf; +/** + * An internal node in the trie. Internal nodes NEVER contain entries; if an + * internal node should contain an entry, then it has a dummy leaf node (see + * below), that can be accessed by node.children["\uFDD0"]. + */ +export interface InternalNode { + type: 'internal'; + weight: number; + /** Maintains the keys of children in descending order of weight. */ + values: string[]; // TODO: As an optimization, "values" can be a single string! + /** + * Maps a single UTF-16 code unit to a child node in the trie. This child + * node may be a leaf or an internal node. The keys of this object are kept + * in sorted order in the .values array. + */ + children: { [codeunit: string]: Node }; + + /** + * Used during compilation. + */ + unsorted?: boolean; +} +/** Only leaf nodes actually contain entries (i.e., the words proper). */ +export interface Leaf { + type: 'leaf'; + weight: number; + entries: Entry[]; + + /** + * Used during compilation. + */ + unsorted?: boolean; +} + +/** + * An entry in the prefix trie (stored in leaf nodes exclusively!) + */ +export interface Entry { + /** The actual word form, stored in the trie. */ + content: string; + /** A search key that usually simplifies the word form, for ease of search. */ + key: SearchKey; + weight: number; +} + +/** + * Recursively sort the trie, in descending order of weight. + * @param node any node in the trie + */ +export function sortNode(node: Node, onlyLocal?: boolean) { + if (node.type === 'leaf') { + if (!node.unsorted) { + return; + } + + node.entries.sort(function (a, b) { return b.weight - a.weight; }); + } else { + if(!onlyLocal) { + // We recurse and sort children before returning if sorting the full Trie. + for (let char of node.values) { + sortNode(node.children[char], onlyLocal); + } + } + + if (!node.unsorted) { + return; + } + + node.values.sort((a, b) => { + return node.children[b].weight - node.children[a].weight; + }); + } + + delete node.unsorted; +} + +export class TrieTraversal implements LexiconTraversal { + /** + * The lexical prefix corresponding to the current traversal state. + */ + readonly prefix: string; + + /** + * The current traversal node. Serves as the 'root' of its own sub-Trie, + * and we cannot navigate back to its parent. + */ + readonly root: Node; + + /** + * The max weight for the Trie being 'traversed'. Needed for probability + * calculations. + */ + readonly totalWeight: number; + + constructor(root: Node, prefix: string, totalWeight: number) { + this.root = root; + this.prefix = prefix; + this.totalWeight = totalWeight; + } + + child(char: string): LexiconTraversal | undefined { + // May result for blank tokens resulting immediately after whitespace. + if(char == '') { + return this; + } + + // Split into individual code units. + let steps = char.split(''); + let traversal: ReturnType = this; + + while(steps.length > 0 && traversal) { + const step: string = steps.shift()!; + traversal = traversal._child(step); + } + + return traversal; + } + + // Handles one code unit at a time. + private _child(char: string): TrieTraversal | undefined { + const root = this.root; + const totalWeight = this.totalWeight; + const nextPrefix = this.prefix + char; + + // Sorts _just_ the current level, and only if needed. + // We only care about sorting parts that we're actually accessing. + sortNode(root, true); + + if(root.type == 'internal') { + let childNode = root.children[char]; + if(!childNode) { + return undefined; + } + + return new TrieTraversal(childNode, nextPrefix, totalWeight); + } else { + // root.type == 'leaf'; + const legalChildren = root.entries.filter(function(entry) { + return entry.key.indexOf(nextPrefix) == 0; + }); + + if(!legalChildren.length) { + return undefined; + } + + return new TrieTraversal(root, nextPrefix, totalWeight); + } + } + + children(): {char: string, p: number, traversal: () => LexiconTraversal}[] { + let root = this.root; + + // We refer to the field multiple times in this method, and it doesn't change. + // This also assists minification a bit, since we can't minify when re-accessing + // through `this.`. + const totalWeight = this.totalWeight; + + const results: {char: string, p: number, traversal: () => LexiconTraversal}[] = []; + + // Sorts _just_ the current level, and only if needed. + // We only care about sorting parts that we're actually accessing. + sortNode(root, true); + + if(root.type == 'internal') { + for(let entry of root.values) { + let entryNode = root.children[entry]; + + // UTF-16 astral plane check. + if(isHighSurrogate(entry)) { + // First code unit of a UTF-16 code point. + // For now, we'll just assume the second always completes such a char. + // + // Note: Things get nasty here if this is only sometimes true; in the future, + // we should compile-time enforce that this assumption is always true if possible. + if(entryNode.type == 'internal') { + let internalNode = entryNode; + for(let lowSurrogate of internalNode.values) { + let prefix = this.prefix + entry + lowSurrogate; + results.push({ + char: entry + lowSurrogate, + p: internalNode.children[lowSurrogate].weight / totalWeight, + traversal: function() { return new TrieTraversal(internalNode.children[lowSurrogate], prefix, totalWeight) } + }); + } + } else { + // No deduplication needed. If there were multiple keys available, + // we wouldn't be in a leaf node. + let fullText = entryNode.entries[0].key; + entry = entry + fullText[this.prefix.length + 1]; // The other half of the non-BMP char. + let prefix = this.prefix + entry; + + const weight = entryNode.entries.reduce((best, current) => Math.max(best, current.weight), 0); + + results.push({ + char: entry, + p: weight / totalWeight, + traversal: function () {return new TrieTraversal(entryNode, prefix, totalWeight)} + }); + } + } else if(isSentinel(entry)) { + continue; + } else if(!entry) { + // Prevent any accidental 'null' or 'undefined' entries from having an effect. + continue; + } else { + let prefix = this.prefix + entry; + results.push({ + char: entry, + p: entryNode.weight / totalWeight, + traversal: function() { return new TrieTraversal(entryNode, prefix, totalWeight)} + }); + } + } + + return results; + } else { // type == 'leaf' + let prefix = this.prefix; + + // No actual deduplication needed. If there were multiple keys available, + // we wouldn't be in a leaf node. + let children = root.entries.filter(function(entry) { + return entry.key != prefix && prefix.length < entry.key.length; + }); + + let weight = children.reduce((best, current) => Math.max(current.weight, best), 0); + + if(children.length > 0) { + const key = children[0].content; + let nodeKey = key[prefix.length]; + + if(isHighSurrogate(nodeKey)) { + // Merge the other half of an SMP char in! + nodeKey = nodeKey + key[prefix.length+1]; + } + results.push({ + char: nodeKey, + p: weight / totalWeight, + traversal: function() { return new TrieTraversal(root, prefix + nodeKey, totalWeight)} + }); + }; + return results; + } + } + + get entries() { + const entryMapper = (value: Entry) => { + return { + text: value.content, + p: value.weight / this.totalWeight + } + } + + if(this.root.type == 'leaf') { + let prefix = this.prefix; + let matches = this.root.entries.filter(function(entry) { + return entry.key == prefix; + }); + + return matches.map(entryMapper); + } else { + let matchingLeaf = this.root.children[SENTINEL_CODE_UNIT]; + if(matchingLeaf && matchingLeaf.type == 'leaf') { + return matchingLeaf.entries.map(entryMapper); + } else { + return []; + } + } + } + + get p(): number { + return this.root.weight / this.totalWeight; + } +} + +/** + * Wrapper class for the trie and its nodes. + */ +export class Trie { + protected root: Node; + /** The total weight of the entire trie. */ + readonly totalWeight: number; + /** + * Converts arbitrary strings to a search key. The trie is built up of + * search keys; not each entry's word form! + */ + toKey: Wordform2Key; + + constructor(root: Node, totalWeight: number, wordform2key: Wordform2Key) { + this.root = root; + this.toKey = wordform2key; + this.totalWeight = totalWeight; + } + + public traverseFromRoot(): LexiconTraversal { + return new TrieTraversal(this.root, '', this.totalWeight); + } +} \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-main/README.md b/web/src/engine/predictive-text/worker-main/README.md index b44aba5016..0271650580 100644 --- a/web/src/engine/predictive-text/worker-main/README.md +++ b/web/src/engine/predictive-text/worker-main/README.md @@ -17,11 +17,13 @@ Build Run `build.sh`. This will also automatically install dependencies with `npm`. ```sh -./build.sh +./build.sh configure build ``` ### Two-stage compilation process +TODO: this is well and truly out of date... + Since the primary LMLayer code runs within a [Web Worker][], `build.sh` compiles the LMLayer in two stages: @@ -43,19 +45,7 @@ This will run both headless unit tests, and in-browser unit tests and integratio tests: ```sh -./build.sh -test +./build.sh test ``` -### Test-Driven Development - -I like to use [entr]() to automatically build and re-run the unit tests anytime I -change a source code file. Here's the command I run in separate window: - -```sh -git ls-files | entr -c ./build.sh -tdd -``` - -Importantly, `./build.sh -tdd` skips running the in-browser tests, and skips -downloading/updating `npm` dependencies. - -[entr]: http://eradman.com/entrproject/ +### TODO: epic/user-dict \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/models/composited-traversal.ts b/web/src/engine/predictive-text/worker-thread/src/main/models/composited-traversal.ts new file mode 100644 index 0000000000..e743bbb638 --- /dev/null +++ b/web/src/engine/predictive-text/worker-thread/src/main/models/composited-traversal.ts @@ -0,0 +1,262 @@ +import { LexicalModelTypes } from "@keymanapp/common-types"; +import Distribution = LexicalModelTypes.Distribution; +import LexiconTraversal = LexicalModelTypes.LexiconTraversal; +import TextWithProbability = LexicalModelTypes.TextWithProbability; +import { PriorityQueue } from "@keymanapp/web-utils"; + +type ChildEdge = { char: string; p: number, traversal: () => LexiconTraversal }; + +export class CompositedTraversal implements LexiconTraversal { + private readonly componentTraversals: Distribution; + + // Used both internally and externally - and we don't want to re-weight internal calls! + constructor(traversalsWithWeighting: Distribution) { + // let totalP = traversalsWithWeighting.reduce((accum, current) => accum + current.p, 0); + this.componentTraversals = traversalsWithWeighting.map((entry) => { + // Shallow-copy the traversals while normalizing the associated weightings. + // The sum of all `p` values should sum to 1. + return { + sample: entry.sample, + p: entry.p // totalP + }; + }); + } + + children(): ChildEdge[] { + // Easy case: when there's no source on the current Traversal path, there + // are no children. + // + // Though... we shouldn't even reach this point; if a child path lacks + // support; `undefined` should have been returned instead of a class + // instance. + + /* c8 ignore start */ + if(this.componentTraversals.length == 0) { + return []; + } + /* c8 ignore stop */ + + // If there were no duplicated children, merging the arrays returned from the + // following statement, as is, would be the correct answer. + const unblendedComponents = this.componentTraversals.map((component) => { + const weight = component.p; + return { + sample: component.sample.children().map((entry) => { + return { + char: entry.char, + p: entry.p, + traversal: () => entry.traversal() + }; + }), + // Make sure to track the model-source's weighting! + weight: weight + }; + }); + + // If there's only a single active component, well, there actually are + // no duplicates - thus it is the right answer. + if(unblendedComponents.length == 1) { + const singleSource = unblendedComponents[0]; + return singleSource.sample.map((child) => { + return { + char: child.char, + p: singleSource.weight * child.p, + traversal: () => new CompositedTraversal([{ sample: child.traversal(), p: singleSource.weight }]) + }; + }); + } + + // Now, the tricky part: actual blending. + + // Find sets of traversals that traverse the same char path after the represented prefix. + const componentBuckets: Record = {}; + unblendedComponents.forEach((componentChildren) => { + componentChildren.sample.forEach((entry) => { + componentBuckets[entry.char] ??= []; + componentBuckets[entry.char].push({ + component: entry, + weight: componentChildren.weight + }); + }); + }); + + // For each char path... + const keys = Object.keys(componentBuckets); + return keys.map((key) => { + const bucket = componentBuckets[key]; + // Build the blended traversal for that path + const traversalFunc = () => { + const traversalsWithWeighting = bucket.map((entry) => { + return { + sample: entry.component.traversal(), + // Apply the model's weighting! + // (This is not the `p` for that model's traversal path.) + p: entry.weight + }; + }); + return new CompositedTraversal(traversalsWithWeighting); + }; + + // If there's only one contributing model, there's no need to worry about duplicates + // affecting the "highest probability" value for the path; no blending needed. + // + // It's thus safe to just return the model's probability for the path. + if(bucket.length == 1) { + return { + char: key, + p: bucket[0].weight * bucket[0].component.p, + traversal: traversalFunc + }; + } else { + // Here, there's a non-zero chance that duplicate entries may exist. + // We thus need to worry about the potential for blending... + // which means we're best off delegating "highest probability" calculation + // to the child, which is the level at which the issue may be resolved. + const traversal = traversalFunc(); + + return { + char: key, + // Use the traversal early to ensure duplicate entries aren't improperly + // neglected due to their probability sources being split. + p: traversal.p, + // We've actually built it already, so just re-use the instance. + traversal: () => traversal + }; + } + }); + } + + child(char: string): LexiconTraversal { + let children = this.componentTraversals.map((traversalTuple) => { + return { + sample: traversalTuple.sample.child(char), + p: traversalTuple.p + }; + }); + + children = children.filter((entry) => !!entry.sample); + + if(children.length == 0) { + return undefined; + } else { + // Will sort out blending on its own. + return new CompositedTraversal(children); + } + } + + get entries(): TextWithProbability[] { + // If there were guaranteed to be no duplicates, .flatMap instead of .map would be fine here. + const entrySets = this.componentTraversals.map((traversalTuple) => { + return traversalTuple.sample.entries.map((entry) => { + const reweightedEntry: TextWithProbability = { + text: entry.text, + p: entry.p * traversalTuple.p + }; + + return reweightedEntry; + }); + }); + + // Combine any duplicated entries. + const resultMap: Record = {}; + entrySets.forEach((set) => { + set.forEach((entry) => { + if(resultMap[entry.text]) { + resultMap[entry.text].p += entry.p; + } else { + resultMap[entry.text] = entry; + } + }); + }); + + // Finalize as array + sort by descending probability + const results = Object.values(resultMap); + results.sort((a, b) => b.p - a.p); + + return results; + } + + get p(): number { + // If it is impossible for a blended traversal path to yield a higher- + // probability entry than the best unblended result, there's no point + // in considering it here. + const bestUnblendedProb = this.componentTraversals + // to 'traversal weighting' * 'traversal node's max probability' + .map((entry) => entry.p * entry.sample.p) + // find the best entry post-weighting + .reduce((best, current) => (current > best) ? current : best, 0); + + // Easy case: 1 component at this level? No blending is possible, + // so just return the best entry found this way. + if(this.componentTraversals.length == 1) { + return bestUnblendedProb; + } + + // Check for traversal children that travel the same path. O(sum of child + // nodes across all components) + // + // If we wanted to try optimizing this... + // - priority queue for each componentTraversal, descending prob order + // - track which sources have contributed while accumulating path probs + // - if best entry from sources not contributed + current accumulation can't + // win, we can stop blending early. + // - further iterations can no longer exceed bestUnblendedProb. + // + // This likely avoids iterating over all paths... but invoking priority + // queues means worst-case O(sum N_i log N_i), which is worse. (With N_i = + // path count per component traversal) + // - constructable "worst case": fully balanced pMaxes? + // - for "good cases"... would the complexity even be worth it? + // - suppose 3 sources + English: worst-case iteration count with current + // setup is 4 * 26 = 104. That's not "terrible"; fairly low to be + // considering big-O optimizations. + // - +1 multiplier due to the following `filter` block. + const childBuckets: Record = {}; + const entryBuckets: Record = {}; + this.componentTraversals.forEach((entry) => { + const weight = entry.p; + const component = entry.sample; + const entries = component.entries; + + component.children().forEach((child) => { + const record = childBuckets[child.char] ??= { pMax: 0, char: child.char }; + record.pMax += weight * child.p; + }); + + entries.forEach((entry) => entryBuckets[entry.text] = (entryBuckets[entry.text] ?? 0) + entry.p * weight); + }); + + // If a path's maximum potential probability does not exceed the best entry, + // it can't affect our answer; skip it. + // Is O(alphabet count) + const potentialBlendedChildren = Object.keys(childBuckets).map((char) => { + const record = childBuckets[char]; + if(record.pMax <= bestUnblendedProb) { + return null; + } else { + return record; + } + }).filter((record) => !!record); + + let bestBlendedEntry = Object.values(entryBuckets).reduce((best, current) => Math.max(best, current), 0); + + // Prep the path-search for a potential winner. + let bestProb = Math.max(bestUnblendedProb, bestBlendedEntry); + // May have zero entries if no blended path can exceed the best non-blended path's probability. + // (Assumption: this is the common case.) + const queue = new PriorityQueue((a, b) => b.pMax - a.pMax, potentialBlendedChildren); + + while(queue.peek()?.pMax > bestProb) { + const entry = queue.dequeue(); + const childPath = this.child(entry.char); + + // Note: this pattern, as-is, does not allow cross-node optimizations + // (such as seeing a better potential here once we've started evaluating + // potentials within the child) + // + // It's a reasonable "first pass" though. + bestProb = Math.max(childPath.p, bestProb); + } + return bestProb; + } +} \ No newline at end of file diff --git a/web/src/engine/predictive-text/worker-thread/src/main/models/index.ts b/web/src/engine/predictive-text/worker-thread/src/main/models/index.ts index 4a6f20da15..6c51a33e0f 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/models/index.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/models/index.ts @@ -1,2 +1,3 @@ export * from '@keymanapp/models-templates'; +export { CompositedTraversal } from './composited-traversal.js'; export { default as DummyModel, DummyOptions } from './dummy-model.js'; diff --git a/web/src/test/auto/headless/engine/predictive-text/templates/trie-builder.tests.ts b/web/src/test/auto/headless/engine/predictive-text/templates/trie-builder.tests.ts new file mode 100644 index 0000000000..f7cadd2d48 --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/templates/trie-builder.tests.ts @@ -0,0 +1,264 @@ +/* + * Unit tests for the Trie prediction model. + */ + +import { assert } from 'chai'; +import { InternalNode, Leaf, SearchKey, SENTINEL_CODE_UNIT, trieConstruction, TrieBuilder } from '@keymanapp/models-templates'; + +describe('trie construction', () => { + it('default root node', () => { + const defaultRoot = trieConstruction.createRootNode() as Leaf; + assert.equal(defaultRoot.type, 'leaf'); + assert.equal(defaultRoot.weight, 0); + assert.sameMembers(defaultRoot.entries, []); + }); + + it('addItemToLeaf', () => { + // `weight` is managed by addUnsorted, not by addItemToLeaf. + // As a result, we don't test for it here. + const defaultRoot = trieConstruction.createRootNode() as Leaf; + trieConstruction.addItemToLeaf(defaultRoot, { + content: 'cafe', + weight: 2, + key: 'cafe' as SearchKey + }); + + assert.equal(defaultRoot.type, 'leaf'); + assert.equal(defaultRoot.entries.length, 1); + + trieConstruction.addItemToLeaf(defaultRoot, { + content: 'café', + weight: 1, + key: 'cafe' as SearchKey + }); + + assert.equal(defaultRoot.type, 'leaf'); + assert.equal(defaultRoot.entries.length, 2); + }); + + it('convertLeafToInternalNode', () => { + const defaultRoot = trieConstruction.createRootNode() as InternalNode; + const cafeEntry = { + content: 'cafe', + weight: 5, + key: 'cafe' as SearchKey + }; + trieConstruction.addItemToLeaf(defaultRoot as unknown as Leaf, cafeEntry); + + defaultRoot.weight = 5; + trieConstruction.convertLeafToInternalNode(defaultRoot as unknown as Leaf, 4); + + assert.equal(defaultRoot.type, 'internal'); + assert.sameMembers(defaultRoot.values, [SENTINEL_CODE_UNIT]); + assert.sameMembers(Object.keys(defaultRoot.children), defaultRoot.values); + assert.equal(defaultRoot.weight, 5); + + const sentinelChild = defaultRoot.children[SENTINEL_CODE_UNIT] as Leaf; + assert.equal(sentinelChild.type, 'leaf'); + assert.equal(sentinelChild.weight, 5); + assert.equal(sentinelChild.entries.length, 1); + // Should be strict-equal too, but we only really care if it's deep-equal. + assert.deepEqual(sentinelChild.entries[0], cafeEntry); + }); + + it('addUnsorted - simple initial entry', () => { + // `weight` is managed by addUnsorted, not by addItemToLeaf. + // As a result, we don't test for it here. + const defaultRoot = trieConstruction.createRootNode() as Leaf; + trieConstruction.addUnsorted(defaultRoot, { + content: 'cafe', + weight: 2, + key: 'cafe' as SearchKey + }, 4); + + assert.equal(defaultRoot.type, 'leaf'); + assert.equal(defaultRoot.weight, 2); + assert.equal(defaultRoot.entries.length, 1); + + // smaller weight, will not override. Is not additive. + trieConstruction.addUnsorted(defaultRoot, { + content: 'café', + weight: 1, + key: 'cafe' as SearchKey + }, 4); + + assert.equal(defaultRoot.type, 'leaf'); + assert.equal(defaultRoot.weight, 2); + assert.equal(defaultRoot.entries.length, 2); + }); + + it('addUnsorted - short word, then longer word with same prefix', () => { + const defaultRoot = trieConstruction.createRootNode(); + trieConstruction.addUnsorted(defaultRoot, { + content: 'cafe', + weight: 5, + key: 'cafe' as SearchKey + }, 4); + + const rootAsLeaf = defaultRoot as Leaf; + + assert.equal(rootAsLeaf.type, 'leaf'); + assert.equal(rootAsLeaf.weight, 5); + assert.equal(rootAsLeaf.entries.length, 1); + + // smaller weight, will not override. Is not additive. + trieConstruction.addUnsorted(defaultRoot, { + content: 'cafeteria', + weight: 3, + key: 'cafeteria' as SearchKey + }, 4); + + const rootAsInternal = defaultRoot as InternalNode; + + assert.equal(rootAsInternal.type, 'internal'); + assert.equal(rootAsInternal.weight, 5); + assert.deepEqual(rootAsInternal.values, [SENTINEL_CODE_UNIT, 't']); + assert.equal((rootAsInternal.children[SENTINEL_CODE_UNIT] as Leaf).weight, 5); + assert.equal((rootAsInternal.children['t'] as Leaf).weight, 3); + }); + + it('addUnsorted', () => { + // Perspective: current root is actually representing `ca`. + const defaultRoot = trieConstruction.createRootNode(); + trieConstruction.addUnsorted(defaultRoot, { + content: 'cafe', + weight: 5, + key: 'cafe' as SearchKey + }, 2); + + // There's only one child, so there's no reason to start making + // internal structure just yet. + assert.equal(defaultRoot.type, 'leaf'); + assert.equal(defaultRoot.weight, 5); + + // Adds a second child; that child has more than one letter not-in-common with the first. + trieConstruction.addUnsorted(defaultRoot, { + content: 'call', + weight: 8, + key: 'call' as SearchKey + }, 2); + + const rootAsInternal = defaultRoot as InternalNode; + + assert.equal(rootAsInternal.type, 'internal'); + assert.equal(rootAsInternal.weight, 8); + assert.sameMembers(rootAsInternal.values, ['f', 'l']); + assert.equal((rootAsInternal.children['f'] as Leaf).weight, 5); + assert.equal((rootAsInternal.children['l'] as Leaf).weight, 8); + }); + + it('addUnsorted - lower frequency prefix', () => { + // Will correspond to 'thin'. + const root = trieConstruction.createRootNode(); + trieConstruction.addUnsorted(root, { + key: 'think' as SearchKey, + content: 'think', + weight: 100 + }, 4); + trieConstruction.addUnsorted(root, { + key: 'thing' as SearchKey, + content: 'thing', + weight: 40 + }, 4); + + const rootAsInternal = root as InternalNode; + assert.equal(rootAsInternal.type, 'internal'); + assert.sameMembers(rootAsInternal.values, ['k', 'g']); + + // Interesting noted behavior: if just 'think' into 'thin', and nothing else... + // it remains a leaf! That doesn't seem... entirely proper. With 'thing', it + // will at least be an 'internal' node already. + trieConstruction.addUnsorted(rootAsInternal, { + key: 'thin' as SearchKey, + content: 'thin', + weight: 20 + }, 4); + + assert.sameMembers(rootAsInternal.values, ['k', 'g', SENTINEL_CODE_UNIT]); + assert.isOk(rootAsInternal.children[SENTINEL_CODE_UNIT]); + assert.equal((rootAsInternal.children[SENTINEL_CODE_UNIT] as Leaf).weight, 20); + }); + + describe('TrieBuilder', () => { + it('standard Trie construction', () => { + const builder = new TrieBuilder((text) => text as SearchKey); + builder.addEntry('caffeine', 2); + builder.addEntry('cafe', 5); + builder.addEntry('calm', 3); + builder.addEntry('calf', 4); + builder.addEntry('call', 6); // total: 20 + builder.addEntry('can', 10); // total: 30 + builder.addEntry('and', 20); // total: 50 + + assert.equal(builder.getTotalWeight(), 50); + + const root = builder.getRoot() as InternalNode; + const aNode = (root.children['c'] as InternalNode).children['a'] as InternalNode; + + // As the nodes were not added in sorted order, they should not currently be ordered. + assert.sameMembers(aNode.values, ['n', 'l', 'f']); + assert.notSameOrderedMembers(aNode.values, ['n', 'l', 'f']); + + const lNode = aNode.children['l'] as InternalNode; + assert.sameMembers(lNode.values, ['l', 'f', 'm']); + assert.notSameOrderedMembers(lNode.values, ['l', 'f', 'm']); + + builder.sort(); + + assert.sameOrderedMembers(aNode.values, ['n', 'l', 'f']); + assert.sameOrderedMembers(lNode.values, ['l', 'f', 'm']); + }); + + // In case of high startup-time for user-dictionary processing, this could allow use + // of a 'partially' processed user wordlist. (We'd prioritize predicting when the + // user is interacting with text, then resume processing once 'idle'.) + it('interspersed construction + lookup', () => { + const builder = new TrieBuilder((text) => text as SearchKey); + builder.addEntry('caffeine', 2); + builder.addEntry('cafe', 5); + builder.addEntry('calm', 3); + builder.addEntry('calf', 4); + builder.addEntry('call', 6); // total: 20 + + // ------------------------------------------ + // Pause construction; actually use the Trie. + + assert.equal(builder.getTotalWeight(), 20); + const root = builder.getRoot() as InternalNode; + const caNode = (root.children['c'] as InternalNode).children['a'] as InternalNode; + + // As the nodes were not added in sorted order, they should not currently be ordered. + assert.sameMembers(caNode.values, ['l', 'f']); + assert.notSameOrderedMembers(caNode.values, ['l', 'f']); + + const cal = builder.traverseFromRoot().child('cal'); + // Actually traversing through the node should auto-sort the entries. + assert.sameOrderedMembers(caNode.values, ['l', 'f']); + // Including the reached node's children. + assert.sameOrderedMembers([...cal.children()].map((entry) => entry.char), ['l', 'f', 'm']); + + const cafNode = caNode.children['f'] as InternalNode; + + // 'caffeine' was added before 'cafe'. + // Parts not 'traversed' should not be unnecessarily sorted. + assert.notSameOrderedMembers(cafNode.values, ['e', 'f']); + + // ------------------- + // Resume construction + + builder.addEntry('can', 10); // total: 30 + builder.addEntry('and', 20); // total: 50 + + assert.equal(builder.getTotalWeight(), 50); + + // As the nodes were not added in sorted order, they should not currently be ordered. + assert.sameMembers(caNode.values, ['n', 'l', 'f']); + // 'n' was added later, thus will be out of sorted order. + assert.notSameOrderedMembers(caNode.values, ['n', 'l', 'f']); + + builder.sort(); + + assert.sameOrderedMembers(caNode.values, ['n', 'l', 'f']); + }); + }); +}); \ No newline at end of file diff --git a/web/src/test/auto/headless/engine/predictive-text/templates/trie-traversal.tests.ts b/web/src/test/auto/headless/engine/predictive-text/templates/trie-traversal.tests.ts index 6c3239f884..7f8947ca16 100644 --- a/web/src/test/auto/headless/engine/predictive-text/templates/trie-traversal.tests.ts +++ b/web/src/test/auto/headless/engine/predictive-text/templates/trie-traversal.tests.ts @@ -175,15 +175,13 @@ describe('Trie traversal abstractions', function() { do { assert.isNotEmpty(leafChildSequence); - let iter = curChild.traversal().children(); - let curr = iter.next(); - curChild = curr.value; + let children = curChild.traversal().children(); + assert.equal(children.length, 1); + curChild = children[0]; // Test generator behavior - there should be one child, then the 'done' state. assert.isDefined(curChild); assert.equal(curChild.char, leafChildSequence[0]); - curr = iter.next(); - assert.isTrue(curr.done); // Prepare for iteration. leafChildSequence.shift(); @@ -276,15 +274,13 @@ describe('Trie traversal abstractions', function() { do { assert.isNotEmpty(leafChildSequence); - let iter = curChild.traversal().children(); - let curr = iter.next(); - curChild = curr.value; + let children = curChild.traversal().children(); + assert.equal(children.length, 1); + curChild = children[0]; // Test generator behavior - there should be one child, then the 'done' state. assert.isDefined(curChild); assert.equal(curChild.char, leafChildSequence[0]); - curr = iter.next(); - assert.isTrue(curr.done); // Prepare for iteration. leafChildSequence.shift(); diff --git a/web/src/test/auto/headless/engine/predictive-text/worker-thread/composited-traversal.ts b/web/src/test/auto/headless/engine/predictive-text/worker-thread/composited-traversal.ts new file mode 100644 index 0000000000..de6a547def --- /dev/null +++ b/web/src/test/auto/headless/engine/predictive-text/worker-thread/composited-traversal.ts @@ -0,0 +1,174 @@ +import { assert } from 'chai'; + +import { SearchKey, TrieBuilder } from '@keymanapp/models-templates'; +import { models } from '@keymanapp/lm-worker/test-index'; + +import CompositedTraversal = models.CompositedTraversal; + +const simpleLexicalModel = (function() { + const builder = new TrieBuilder((str) => str as SearchKey); + + // Sum total: 10. + builder.addEntry('are', 1); + builder.addEntry('and', 2); + + builder.addEntry('not', 1); + + builder.addEntry('the', 3); + builder.addEntry('try', 1); + builder.addEntry('to', 1); + builder.addEntry('tom', 1); + + return builder; +}()); + +const namesModel = (function() { + const builder = new TrieBuilder((str) => str as SearchKey); + + builder.addEntry('amy', 1); + builder.addEntry('bob', 1); + builder.addEntry('mark', 1); + builder.addEntry('tom', 1); + + return builder; +}()); + +const blendedModel = new CompositedTraversal([ + { sample: simpleLexicalModel.traverseFromRoot(), p: 0.5 }, + { sample: namesModel.traverseFromRoot(), p: 0.5 } +]); + +describe('CompositedTraversal', () => { + describe('direct prefix lookup', () => { + it('returns undefined for non-existent paths', () => { + assert.isUndefined(blendedModel.child('tomcat')); + assert.isUndefined(blendedModel.child('tone')); + assert.isUndefined(blendedModel.child('ton')); + assert.isUndefined(blendedModel.child('apple')); + assert.isUndefined(blendedModel.child('aim')); + assert.isUndefined(blendedModel.child('xylophone')); + }); + + it('reweighs entries found in only one source', () => { + const andNode = blendedModel.child('and'); + assert.equal(andNode.p, 0.1); + + assert.equal(andNode.entries.length, 1); + const and = andNode.entries[0]; + assert.equal(and.text, 'and'); + assert.equal(and.p, 0.1); + }); + + it('merges entries found in multiple sources, combining their probability and entries', () => { + const tomNode = blendedModel.child('tom'); + assert.equal(tomNode.p, 0.175); + + assert.equal(tomNode.entries.length, 1); + const tom = tomNode.entries[0]; + assert.equal(tom.text, 'tom'); + assert.equal(tom.p, 0.175); + }); + + it('correctly returns the probability of intermediate child paths', () => { + // simpleLexical: best is 'and', 0.2 + // names: best is 'amy', 0.25 + // Weight of each is equal at 0.5; 0.25 * 0.5 = 0.125. + // Should not combine. + const aNode = blendedModel.child('a'); + assert.equal(aNode.p, 0.125, "probability of paths were merged despite lack of duplicate"); + assert.equal(aNode.entries.length, 0); + + // 'tom' is on both paths, and is highest-frequency entry when merged. + // We want the combined, net probability to show up on its predecessor paths. + const toNode = blendedModel.child('to'); + assert.equal(toNode.p, 0.175, "probability of path with winning duplicate not merged"); + + // simpleLexical - p('to') = 0.1; is then multiplied by model weight of 0.5. + assert.equal(toNode.entries.length, 1); + const to = toNode.entries[0]; + assert.equal(to.text, 'to'); + assert.equal(to.p, 0.05); + }); + }); + + describe('iteration for correction-search', () => { + it('correctly enumerates child paths from root', () => { + const topChildren = blendedModel.children(); + // 'n' from "simpleLexical", 'b' and 'm' from "names" + assert.sameMembers(topChildren.map((child) => child.char), ['a', 'b', 'm', 'n', 't']); + assert.sameDeepMembers(topChildren.map((child) => { + return { + char: child.char, + p: child.p + } + }), [ + { char: 'a', p: .125 }, // highest: 'amy' + { char: 'b', p: .125 }, // highest: 'bob' + { char: 'm', p: .125 }, // highest: 'mark' + { char: 'n', p: .05 }, // highest: 'not' + { char: 't', p: .175 } // highest: 'tom' + ]); + topChildren.forEach((entry) => assert.equal(entry.p, entry.traversal().p)); + }); + + it('correctly enumerates child paths from section supported by a single member', () => { + const topChildren = blendedModel.children(); + const bNode = topChildren.find((entry) => entry.char == 'b').traversal(); + const bChildren = bNode.children(); + assert.sameDeepMembers(bChildren.map((child) => { + return { + char: child.char, + p: child.p + }; + }), [ + { char: 'o', p: 0.125 } + ]); + bChildren.forEach((entry) => assert.equal(entry.p, entry.traversal().p)); + + const boNode = bChildren.find((entry) => entry.char == 'o').traversal(); + const boChildren = boNode.children(); + assert.sameDeepMembers(boChildren.map((child) => { + return { + char: child.char, + p: child.p + }; + }), [ + { char: 'b', p: 0.125 } + ]); + boChildren.forEach((entry) => assert.equal(entry.p, entry.traversal().p)); + }); + + it('correctly enumerates child paths from section supported by multiple members', () => { + const topChildren = blendedModel.children(); + const tNode = topChildren.find((entry) => entry.char == 't').traversal(); + const tChildren = tNode.children(); + + assert.sameMembers(tChildren.map((child) => child.char), ['h', 'o', 'r']); + assert.sameDeepMembers(tChildren.map((child) => { + return { + char: child.char, + p: child.p + } + }), [ + { char: 'h', p: 0.15 }, + { char: 'o', p: 0.175 }, + { char: 'r', p: 0.05 } + ]); + tChildren.forEach((entry) => assert.equal(entry.p, entry.traversal().p)); + + const toNode = tChildren.find((entry) => entry.char == 'o').traversal(); + const toChildren = toNode.children(); + + assert.sameMembers(toChildren.map((child) => child.char), ['m']); + assert.sameDeepMembers(toChildren.map((child) => { + return { + char: child.char, + p: child.p + }; + }), [ + { char: 'm', p: 0.175 } + ]); + toChildren.forEach((entry) => assert.equal(entry.p, entry.traversal().p)); + }); + }); +}); \ No newline at end of file