From fe31917bb5f1de71cdde1fa8dfbd26c8c3a510b5 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 15 Mar 2024 20:26:19 +0700 Subject: [PATCH 1/3] change(common/models/templates): predict methods now utilize traversals --- common/models/templates/src/trie-model.ts | 124 ++++------------------ common/models/types/index.d.ts | 38 ++++--- 2 files changed, 43 insertions(+), 119 deletions(-) diff --git a/common/models/templates/src/trie-model.ts b/common/models/templates/src/trie-model.ts index df94e5cb83..605bbc7c3f 100644 --- a/common/models/templates/src/trie-model.ts +++ b/common/models/templates/src/trie-model.ts @@ -73,15 +73,6 @@ export interface TrieModelOptions { punctuation?: LexicalModelPunctuation; } -/** - * Used to determine the probability of an entry from the trie. - */ -type TextWithProbability = { - text: string; - // TODO: use negative-log scaling instead? - p: number; // real-number weight, from 0 to 1 -} - class Traversal implements LexiconTraversal { /** * The lexical prefix corresponding to the current traversal state. @@ -375,11 +366,10 @@ export default class TrieModel implements LexicalModel { type SearchKey = string & { _: 'SearchKey'}; /** - * The priority queue will always pop the most weighted item. There can only - * be two kinds of items right now: nodes, and entries; both having a weight - * attribute. + * The priority queue will always pop the most probable item - be it a Traversal + * state or a lexical entry reached via Traversal. */ -type Weighted = Node | Entry; +type TraversableWithProb = TextWithProbability | LexiconTraversal; /** * A function that converts a string (word form or query) into a search key @@ -462,12 +452,8 @@ class Trie { */ lookup(prefix: string): TextWithProbability[] { let searchKey = this.toKey(prefix); - let lowestCommonNode = findPrefix(this.root, searchKey); - if (lowestCommonNode === null) { - return []; - } - - return getSortedResults(lowestCommonNode, searchKey, this.totalWeight); + let rootTraversal = this.traverseFromRoot().child(searchKey); + return rootTraversal ? getSortedResults(rootTraversal) : []; } /** @@ -475,36 +461,10 @@ class Trie { * @param n How many suggestions, maximum, to return. */ firstN(n: number): TextWithProbability[] { - return getSortedResults(this.root, '' as SearchKey, this.totalWeight, n); + return getSortedResults(this.traverseFromRoot(), n); } } -/** - * Finds the deepest descendent in the trie with the given prefix key. - * - * This means that a search in the trie for a given prefix has a best-case - * complexity of O(m) where m is the length of the prefix. - * - * @param key The prefix to search for. - * @param index The index in the prefix. Initially 0. - */ -function findPrefix(node: Node, key: SearchKey, index: number = 0): Node | null { - // An important note - the Trie itself is built on a per-JS-character basis, - // not on a UTF-8 character-code basis. - if (node.type === 'leaf' || index === key.length) { - return node; - } - - // So, for SMP models, we need to match each char of the supplementary pair - // in sequence. Each has its own node in the Trie. - let char = key[index]; - if (node.children[char]) { - return findPrefix(node.children[char], key, index + 1); - } - - return null; -} - /** * Returns all entries matching the given prefix, in descending order of * weight. @@ -513,72 +473,32 @@ function findPrefix(node: Node, key: SearchKey, index: number = 0): Node | null * @param results the current results * @param queue */ -function getSortedResults(node: Node, prefix: SearchKey, N: number, limit = MAX_SUGGESTIONS): TextWithProbability[] { - let queue = new PriorityQueue(function(a: Weighted, b: Weighted) { +function getSortedResults(traversal: LexiconTraversal, limit = MAX_SUGGESTIONS): TextWithProbability[] { + let queue = new PriorityQueue(function(a: TraversableWithProb, b: TraversableWithProb) { // In case of Trie compilation issues that emit `null` or `undefined` - return (b ? b.weight : 0) - (a ? a.weight : 0); + return (b ? b.p : 0) - (a ? a.p : 0); }); let results: TextWithProbability[] = []; - if (node.type === 'leaf') { - // Assuming the values are sorted, we can just add all of the values in the - // leaf, until we reach the limit. - for (let item of node.entries) { - // String.startsWith is not supported on certain Android (5.0) devices we wish to support. - // Requires a minimum of Chrome 36, as opposed to 5.0's default of 35. - if (item.key.indexOf(prefix) == 0) { - let { content, weight } = item; - results.push({ - text: content, - p: weight / N - }); + queue.enqueue(traversal); - if (results.length >= limit) { - return results; - } - } - } - } else { - queue.enqueue(node); - let next: Weighted | undefined; + while(queue.count > 0) { + const entry = queue.dequeue(); - while (next = queue.dequeue()) { - if (isNode(next)) { - // When a node is next up in the queue, that means that next least - // likely suggestion is among its decsendants. - // So we search all of its descendants! - if (next.type === 'leaf') { - queue.enqueueAll(next.entries); - } else { - // XXX: alias `next` so that TypeScript can be SURE that internal is - // in fact an internal node. Because of the callback binding to the - // original definition of node (i.e., a Node | Entry), this will not - // type-check otherwise. - let internal = next; - queue.enqueueAll(next.values.map(char => { - return internal.children[char]; - })); - } - } else { - // When an entry is up next in the queue, we just add its contents to - // the results! - results.push({ - text: next.content, - p: next.weight / N - }); - if (results.length >= limit) { - return results; - } + if((entry as TextWithProbability)!.text !== undefined) { + const lexicalEntry = entry as TextWithProbability; + results.push(lexicalEntry); + if(results.length >= limit) { + return results; } + } else { + const traversal = entry as LexiconTraversal; + queue.enqueueAll(traversal.entries); + queue.enqueueAll(Array.from(traversal.children()).map((entry) => entry.traversal())); } } + return results; - -} - -/** TypeScript type guard that returns whether the thing is a Node. */ -function isNode(x: Entry | Node): x is Node { - return 'type' in x; } /** diff --git a/common/models/types/index.d.ts b/common/models/types/index.d.ts index e94d758b0b..2fc1c92cb3 100644 --- a/common/models/types/index.d.ts +++ b/common/models/types/index.d.ts @@ -19,6 +19,23 @@ declare type USVString = string; declare type CasingForm = 'lower' | 'initial' | 'upper'; +/** + * Represents one lexical entry and its probability.. + */ +type TextWithProbability = { + /** + * A lexical entry (word) offered by the model. + * + * Note: not the search-term keyed part. This will match the actual, unkeyed form. + */ + text: string; + + /** + * The probability of the lexical entry, directly based upon its frequency. + */ + p: number; // real-number weight, from 0 to 1 +} + /** * Used to facilitate edit-distance calculations by allowing the LMLayer to * efficiently search the model's lexicon in a Trie-like manner. @@ -52,13 +69,11 @@ declare interface LexiconTraversal { /** * Allows direct access to the traversal state that results when appending a - * `char` representing a single UTF-16 codepoint to the current traversal - * state's prefix. This bypasses the need to iterate among all legal child - * Traversals. + * `char` representing one or more individual UTF-16 codepoints to the + * current traversal state's prefix. This bypasses the need to iterate + * among all legal child Traversals. * * If such a traversal state is not supported, returns `undefined`. - * Implementations may choose to return `undefined` if more than one UTF-16 - * codepoint is appended, even if such a descendant exists. * * Note: traversals navigate and represent the lexicon in its "keyed" state, * as produced by use of the search-term keying function defined for the model. @@ -87,18 +102,7 @@ declare interface LexiconTraversal { * - prefix of 'crepe': ['crêpe', 'crêpé'] * - other examples: https://www.thoughtco.com/french-accent-homographs-1371072 */ - entries: { - /** - * A lexical entry (word) offered by the model. - * - * Note: not the search-term keyed part. This will match the actual, unkeyed form. - */ - text: USVString, - /** - * The probability of the lexical entry, directly based upon its frequency. - */ - p: number - }[]; + entries: TextWithProbability[]; // Note: `p`, not `maxP` - we want to see the same name for `this.entries.p` and `this.p` /** From 3423c63bc0a16ddcee6a6021327ea5360be6aa62 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 26 Jun 2024 11:48:20 +0700 Subject: [PATCH 2/3] change(web): avoid use of Array.from with iterators (in TrieModel.predict) --- common/models/templates/src/trie-model.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/common/models/templates/src/trie-model.ts b/common/models/templates/src/trie-model.ts index 605bbc7c3f..cbbe918162 100644 --- a/common/models/templates/src/trie-model.ts +++ b/common/models/templates/src/trie-model.ts @@ -494,7 +494,11 @@ function getSortedResults(traversal: LexiconTraversal, limit = MAX_SUGGESTIONS): } else { const traversal = entry as LexiconTraversal; queue.enqueueAll(traversal.entries); - queue.enqueueAll(Array.from(traversal.children()).map((entry) => entry.traversal())); + let children: LexiconTraversal[] = [] + for(let child of traversal.children()) { + children.push(child.traversal()); + } + queue.enqueueAll(children); } } From a7365abd66d6475ef635bf10024ca3212c862cac Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 8 Jul 2024 10:26:40 +0700 Subject: [PATCH 3/3] docs(common/models): comment format adjustment per PR review --- common/models/types/index.d.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/common/models/types/index.d.ts b/common/models/types/index.d.ts index a75e6a046f..5de561ef47 100644 --- a/common/models/types/index.d.ts +++ b/common/models/types/index.d.ts @@ -32,8 +32,10 @@ type TextWithProbability = { /** * The probability of the lexical entry, directly based upon its frequency. + * + * A real-number weight, from 0 to 1. */ - p: number; // real-number weight, from 0 to 1 + p: number; } /**