mirror of
https://github.com/keymanapp/keyman.git
synced 2026-10-15 02:17:32 +00:00
Merge branch 'epic/user-dict' into chore/merge-master-into-user-dict
This commit is contained in:
Joshua Horton
commit
71a5a9bdd4
8 changed files with 707 additions and 31 deletions
|
|
@ -10,8 +10,10 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")"
|
|||
|
||||
builder_describe "Keyman kmc Lexical Model Compiler module" \
|
||||
"@/common/web/keyman-version" \
|
||||
"@/common/web/types" \
|
||||
"@/developer/src/common/web/test-helpers" \
|
||||
"@/web/src/engine/predictive-text/templates/ test" \
|
||||
"@/developer/src/common/web/utils" \
|
||||
"@/web/src/engine/predictive-text/templates/" \
|
||||
"clean" \
|
||||
"configure" \
|
||||
"build" \
|
||||
|
|
|
|||
|
|
@ -32,11 +32,11 @@
|
|||
"dependencies": {
|
||||
"@keymanapp/common-types": "*",
|
||||
"@keymanapp/keyman-version": "*",
|
||||
"@keymanapp/models-templates": "*",
|
||||
"typescript": "^5.4.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@keymanapp/developer-test-helpers": "*",
|
||||
"@keymanapp/models-templates": "*",
|
||||
"@types/mocha": "^5.2.7",
|
||||
"@types/node": "^20.4.1",
|
||||
"c8": "^7.12.0",
|
||||
|
|
|
|||
2
package-lock.json
generated
2
package-lock.json
generated
|
|
@ -1473,11 +1473,11 @@
|
|||
"dependencies": {
|
||||
"@keymanapp/common-types": "*",
|
||||
"@keymanapp/keyman-version": "*",
|
||||
"@keymanapp/models-templates": "*",
|
||||
"typescript": "^5.4.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@keymanapp/developer-test-helpers": "*",
|
||||
"@keymanapp/models-templates": "*",
|
||||
"@types/mocha": "^5.2.7",
|
||||
"@types/node": "^20.4.1",
|
||||
"c8": "^7.12.0",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,257 @@
|
|||
/*
|
||||
* Unit tests for the Trie prediction model.
|
||||
*/
|
||||
|
||||
import { assert } from 'chai';
|
||||
import { SENTINEL_CODE_UNIT, trieConstruction, TrieBuilder } from '@keymanapp/models-templates';
|
||||
|
||||
describe('trie construction', () => {
|
||||
it('default root node', () => {
|
||||
const defaultRoot = trieConstruction.createRootNode();
|
||||
assert.equal(defaultRoot.type, 'leaf');
|
||||
assert.equal(defaultRoot.weight, 0);
|
||||
assert.sameMembers(defaultRoot.entries, []);
|
||||
});
|
||||
|
||||
it('addItemToLeaf', () => {
|
||||
// `weight` is managed by addUnsorted, not by addItemToLeaf.
|
||||
// As a result, we don't test for it here.
|
||||
const defaultRoot = trieConstruction.createRootNode();
|
||||
trieConstruction.addItemToLeaf(defaultRoot, {
|
||||
content: 'cafe',
|
||||
weight: 2,
|
||||
key: 'cafe'
|
||||
});
|
||||
|
||||
assert.equal(defaultRoot.type, 'leaf');
|
||||
assert.equal(defaultRoot.entries.length, 1);
|
||||
|
||||
trieConstruction.addItemToLeaf(defaultRoot, {
|
||||
content: 'café',
|
||||
weight: 1,
|
||||
key: 'cafe'
|
||||
});
|
||||
|
||||
assert.equal(defaultRoot.type, 'leaf');
|
||||
assert.equal(defaultRoot.entries.length, 2);
|
||||
});
|
||||
|
||||
it('convertLeafToInternalNode', () => {
|
||||
const defaultRoot = trieConstruction.createRootNode();
|
||||
const cafeEntry = {
|
||||
content: 'cafe',
|
||||
weight: 5,
|
||||
key: 'cafe'
|
||||
};
|
||||
trieConstruction.addItemToLeaf(defaultRoot, cafeEntry);
|
||||
|
||||
defaultRoot.weight = 5;
|
||||
trieConstruction.convertLeafToInternalNode(defaultRoot, 4);
|
||||
|
||||
assert.equal(defaultRoot.type, 'internal');
|
||||
assert.sameMembers(defaultRoot.values, [SENTINEL_CODE_UNIT]);
|
||||
assert.sameMembers(Object.keys(defaultRoot.children), defaultRoot.values);
|
||||
assert.equal(defaultRoot.weight, 5);
|
||||
|
||||
const sentinelChild = defaultRoot.children[SENTINEL_CODE_UNIT];
|
||||
assert.equal(sentinelChild.type, 'leaf');
|
||||
assert.equal(sentinelChild.weight, 5);
|
||||
assert.equal(sentinelChild.entries.length, 1);
|
||||
// Should be strict-equal too, but we only really care if it's deep-equal.
|
||||
assert.deepEqual(sentinelChild.entries[0], cafeEntry);
|
||||
});
|
||||
|
||||
it('addUnsorted - simple initial entry', () => {
|
||||
// `weight` is managed by addUnsorted, not by addItemToLeaf.
|
||||
// As a result, we don't test for it here.
|
||||
const defaultRoot = trieConstruction.createRootNode();
|
||||
trieConstruction.addUnsorted(defaultRoot, {
|
||||
content: 'cafe',
|
||||
weight: 2,
|
||||
key: 'cafe'
|
||||
}, 4);
|
||||
|
||||
assert.equal(defaultRoot.type, 'leaf');
|
||||
assert.equal(defaultRoot.weight, 2);
|
||||
assert.equal(defaultRoot.entries.length, 1);
|
||||
|
||||
// smaller weight, will not override. Is not additive.
|
||||
trieConstruction.addUnsorted(defaultRoot, {
|
||||
content: 'café',
|
||||
weight: 1,
|
||||
key: 'cafe'
|
||||
}, 4);
|
||||
|
||||
assert.equal(defaultRoot.type, 'leaf');
|
||||
assert.equal(defaultRoot.weight, 2);
|
||||
assert.equal(defaultRoot.entries.length, 2);
|
||||
});
|
||||
|
||||
it('addUnsorted - short word, then longer word with same prefix', () => {
|
||||
const defaultRoot = trieConstruction.createRootNode();
|
||||
trieConstruction.addUnsorted(defaultRoot, {
|
||||
content: 'cafe',
|
||||
weight: 5,
|
||||
key: 'cafe'
|
||||
}, 4);
|
||||
|
||||
assert.equal(defaultRoot.type, 'leaf');
|
||||
assert.equal(defaultRoot.weight, 5);
|
||||
assert.equal(defaultRoot.entries.length, 1);
|
||||
|
||||
// smaller weight, will not override. Is not additive.
|
||||
trieConstruction.addUnsorted(defaultRoot, {
|
||||
content: 'cafeteria',
|
||||
weight: 3,
|
||||
key: 'cafeteria'
|
||||
}, 4);
|
||||
|
||||
assert.equal(defaultRoot.type, 'internal');
|
||||
assert.equal(defaultRoot.weight, 5);
|
||||
assert.deepEqual(defaultRoot.values, [SENTINEL_CODE_UNIT, 't']);
|
||||
assert.equal(defaultRoot.children[SENTINEL_CODE_UNIT].weight, 5);
|
||||
assert.equal(defaultRoot.children['t'].weight, 3);
|
||||
});
|
||||
|
||||
it('addUnsorted', () => {
|
||||
// Perspective: current root is actually representing `ca`.
|
||||
const defaultRoot = trieConstruction.createRootNode();
|
||||
trieConstruction.addUnsorted(defaultRoot, {
|
||||
content: 'cafe',
|
||||
weight: 5,
|
||||
key: 'cafe'
|
||||
}, 2);
|
||||
|
||||
// There's only one child, so there's no reason to start making
|
||||
// internal structure just yet.
|
||||
assert.equal(defaultRoot.type, 'leaf');
|
||||
assert.equal(defaultRoot.weight, 5);
|
||||
|
||||
// Adds a second child; that child has more than one letter not-in-common with the first.
|
||||
trieConstruction.addUnsorted(defaultRoot, {
|
||||
content: 'call',
|
||||
weight: 8,
|
||||
key: 'call'
|
||||
}, 2);
|
||||
|
||||
assert.equal(defaultRoot.type, 'internal');
|
||||
assert.equal(defaultRoot.weight, 8);
|
||||
assert.sameMembers(defaultRoot.values, ['f', 'l']);
|
||||
assert.equal(defaultRoot.children['f'].weight, 5);
|
||||
assert.equal(defaultRoot.children['l'].weight, 8);
|
||||
});
|
||||
|
||||
it('addUnsorted - lower frequency prefix', () => {
|
||||
// Will correspond to 'thin'.
|
||||
const root = trieConstruction.createRootNode();
|
||||
trieConstruction.addUnsorted(root, {
|
||||
key: 'think',
|
||||
content: 'think',
|
||||
weight: 100
|
||||
}, 4);
|
||||
trieConstruction.addUnsorted(root, {
|
||||
key: 'thing',
|
||||
content: 'thing',
|
||||
weight: 40
|
||||
}, 4);
|
||||
|
||||
assert.equal(root.type, 'internal');
|
||||
assert.sameMembers(root.values, ['k', 'g']);
|
||||
|
||||
// Interesting noted behavior: if just 'think' into 'thin', and nothing else...
|
||||
// it remains a leaf! That doesn't seem... entirely proper. With 'thing', it
|
||||
// will at least be an 'internal' node already.
|
||||
trieConstruction.addUnsorted(root, {
|
||||
key: 'thin',
|
||||
content: 'thin',
|
||||
weight: 20
|
||||
}, 4);
|
||||
|
||||
assert.sameMembers(root.values, ['k', 'g', SENTINEL_CODE_UNIT]);
|
||||
assert.isOk(root.children[SENTINEL_CODE_UNIT]);
|
||||
assert.equal(root.children[SENTINEL_CODE_UNIT].weight, 20);
|
||||
});
|
||||
|
||||
describe('TrieBuilder', () => {
|
||||
it('standard Trie construction', () => {
|
||||
const builder = new TrieBuilder((text) => text);
|
||||
builder.addEntry('caffeine', 2);
|
||||
builder.addEntry('cafe', 5);
|
||||
builder.addEntry('calm', 3);
|
||||
builder.addEntry('calf', 4);
|
||||
builder.addEntry('call', 6); // total: 20
|
||||
builder.addEntry('can', 10); // total: 30
|
||||
builder.addEntry('and', 20); // total: 50
|
||||
|
||||
assert.equal(builder.getTotalWeight(), 50);
|
||||
|
||||
const root = builder.getRoot();
|
||||
const aNode = root.children['c'].children['a'];
|
||||
|
||||
// As the nodes were not added in sorted order, they should not currently be ordered.
|
||||
assert.sameMembers(aNode.values, ['n', 'l', 'f']);
|
||||
assert.notSameOrderedMembers(aNode.values, ['n', 'l', 'f']);
|
||||
|
||||
const lNode = aNode.children['l'];
|
||||
assert.sameMembers(lNode.values, ['l', 'f', 'm']);
|
||||
assert.notSameOrderedMembers(lNode.values, ['l', 'f', 'm']);
|
||||
|
||||
builder.sort();
|
||||
|
||||
assert.sameOrderedMembers(aNode.values, ['n', 'l', 'f']);
|
||||
assert.sameOrderedMembers(lNode.values, ['l', 'f', 'm']);
|
||||
});
|
||||
|
||||
// In case of high startup-time for user-dictionary processing, this could allow use
|
||||
// of a 'partially' processed user wordlist. (We'd prioritize predicting when the
|
||||
// user is interacting with text, then resume processing once 'idle'.)
|
||||
it('interspersed construction + lookup', () => {
|
||||
const builder = new TrieBuilder((text) => text);
|
||||
builder.addEntry('caffeine', 2);
|
||||
builder.addEntry('cafe', 5);
|
||||
builder.addEntry('calm', 3);
|
||||
builder.addEntry('calf', 4);
|
||||
builder.addEntry('call', 6); // total: 20
|
||||
|
||||
// ------------------------------------------
|
||||
// Pause construction; actually use the Trie.
|
||||
|
||||
assert.equal(builder.getTotalWeight(), 20);
|
||||
const root = builder.getRoot();
|
||||
const caNode = root.children['c'].children['a'];
|
||||
|
||||
// As the nodes were not added in sorted order, they should not currently be ordered.
|
||||
assert.sameMembers(caNode.values, ['l', 'f']);
|
||||
assert.notSameOrderedMembers(caNode.values, ['l', 'f']);
|
||||
|
||||
const cal = builder.traverseFromRoot().child('cal');
|
||||
// Actually traversing through the node should auto-sort the entries.
|
||||
assert.sameOrderedMembers(caNode.values, ['l', 'f']);
|
||||
// Including the reached node's children.
|
||||
assert.sameOrderedMembers([...cal.children()].map((entry) => entry.char), ['l', 'f', 'm']);
|
||||
|
||||
const cafNode = caNode.children['f'];
|
||||
|
||||
// 'caffeine' was added before 'cafe'.
|
||||
// Parts not 'traversed' should not be unnecessarily sorted.
|
||||
assert.notSameOrderedMembers(cafNode.values, ['e', 'f']);
|
||||
|
||||
// -------------------
|
||||
// Resume construction
|
||||
|
||||
builder.addEntry('can', 10); // total: 30
|
||||
builder.addEntry('and', 20); // total: 50
|
||||
|
||||
assert.equal(builder.getTotalWeight(), 50);
|
||||
|
||||
// As the nodes were not added in sorted order, they should not currently be ordered.
|
||||
assert.sameMembers(caNode.values, ['n', 'l', 'f']);
|
||||
// 'n' was added later, thus will be out of sorted order.
|
||||
assert.notSameOrderedMembers(caNode.values, ['n', 'l', 'f']);
|
||||
|
||||
builder.sort();
|
||||
|
||||
assert.sameOrderedMembers(caNode.values, ['n', 'l', 'f']);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -174,15 +174,13 @@ describe('Trie traversal abstractions', function() {
|
|||
do {
|
||||
assert.isNotEmpty(leafChildSequence);
|
||||
|
||||
let iter = curChild.traversal().children();
|
||||
let curr = iter.next();
|
||||
curChild = curr.value;
|
||||
let children = curChild.traversal().children();
|
||||
assert.equal(children.length, 1);
|
||||
curChild = children[0];
|
||||
|
||||
// Test generator behavior - there should be one child, then the 'done' state.
|
||||
assert.isDefined(curChild);
|
||||
assert.equal(curChild.char, leafChildSequence[0]);
|
||||
curr = iter.next();
|
||||
assert.isTrue(curr.done);
|
||||
|
||||
// Prepare for iteration.
|
||||
leafChildSequence.shift();
|
||||
|
|
@ -275,15 +273,13 @@ describe('Trie traversal abstractions', function() {
|
|||
do {
|
||||
assert.isNotEmpty(leafChildSequence);
|
||||
|
||||
let iter = curChild.traversal().children();
|
||||
let curr = iter.next();
|
||||
curChild = curr.value;
|
||||
let children = curChild.traversal().children();
|
||||
assert.equal(children.length, 1);
|
||||
curChild = children[0];
|
||||
|
||||
// Test generator behavior - there should be one child, then the 'done' state.
|
||||
assert.isDefined(curChild);
|
||||
assert.equal(curChild.char, leafChildSequence[0]);
|
||||
curr = iter.next();
|
||||
assert.isTrue(curr.done);
|
||||
|
||||
// Prepare for iteration.
|
||||
leafChildSequence.shift();
|
||||
|
|
@ -337,9 +333,6 @@ describe('Trie traversal abstractions', function() {
|
|||
// Just to be sure our utility function is working right.
|
||||
assert.equal(smpA + smpP + 'pl' + smpE, "𝖺𝗉pl𝖾");
|
||||
|
||||
let pKeys = ['p', smpP];
|
||||
let leafChildSequence = ['l', smpE];
|
||||
|
||||
const aNode = rootTraversal.child(smpA);
|
||||
assert.isOk(aNode);
|
||||
assert.isNotOk(rootTraversal.child('a'));
|
||||
|
|
|
|||
|
|
@ -17,11 +17,13 @@ Build
|
|||
Run `build.sh`. This will also automatically install dependencies with `npm`.
|
||||
|
||||
```sh
|
||||
./build.sh
|
||||
./build.sh configure build
|
||||
```
|
||||
|
||||
### Two-stage compilation process
|
||||
|
||||
TODO: this is well and truly out of date...
|
||||
|
||||
Since the primary LMLayer code runs within a [Web Worker][], `build.sh` compiles the
|
||||
LMLayer in two stages:
|
||||
|
||||
|
|
@ -43,19 +45,7 @@ This will run both headless unit tests, and in-browser unit tests and integratio
|
|||
tests:
|
||||
|
||||
```sh
|
||||
./build.sh -test
|
||||
./build.sh test
|
||||
```
|
||||
|
||||
### Test-Driven Development
|
||||
|
||||
I like to use [entr]() to automatically build and re-run the unit tests anytime I
|
||||
change a source code file. Here's the command I run in separate window:
|
||||
|
||||
```sh
|
||||
git ls-files | entr -c ./build.sh -tdd
|
||||
```
|
||||
|
||||
Importantly, `./build.sh -tdd` skips running the in-browser tests, and skips
|
||||
downloading/updating `npm` dependencies.
|
||||
|
||||
[entr]: http://eradman.com/entrproject/
|
||||
### TODO: epic/user-dict
|
||||
|
|
@ -0,0 +1,262 @@
|
|||
import { LexicalModelTypes } from "@keymanapp/common-types";
|
||||
import Distribution = LexicalModelTypes.Distribution;
|
||||
import LexiconTraversal = LexicalModelTypes.LexiconTraversal;
|
||||
import TextWithProbability = LexicalModelTypes.TextWithProbability;
|
||||
import { PriorityQueue } from "@keymanapp/web-utils";
|
||||
|
||||
type ChildEdge = { char: string; p: number, traversal: () => LexiconTraversal };
|
||||
|
||||
export class CompositedTraversal implements LexiconTraversal {
|
||||
private readonly componentTraversals: Distribution<LexiconTraversal>;
|
||||
|
||||
// Used both internally and externally - and we don't want to re-weight internal calls!
|
||||
constructor(traversalsWithWeighting: Distribution<LexiconTraversal>) {
|
||||
// let totalP = traversalsWithWeighting.reduce((accum, current) => accum + current.p, 0);
|
||||
this.componentTraversals = traversalsWithWeighting.map((entry) => {
|
||||
// Shallow-copy the traversals while normalizing the associated weightings.
|
||||
// The sum of all `p` values should sum to 1.
|
||||
return {
|
||||
sample: entry.sample,
|
||||
p: entry.p // totalP
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
children(): ChildEdge[] {
|
||||
// Easy case: when there's no source on the current Traversal path, there
|
||||
// are no children.
|
||||
//
|
||||
// Though... we shouldn't even reach this point; if a child path lacks
|
||||
// support; `undefined` should have been returned instead of a class
|
||||
// instance.
|
||||
|
||||
/* c8 ignore start */
|
||||
if(this.componentTraversals.length == 0) {
|
||||
return [];
|
||||
}
|
||||
/* c8 ignore stop */
|
||||
|
||||
// If there were no duplicated children, merging the arrays returned from the
|
||||
// following statement, as is, would be the correct answer.
|
||||
const unblendedComponents = this.componentTraversals.map((component) => {
|
||||
const weight = component.p;
|
||||
return {
|
||||
sample: component.sample.children().map((entry) => {
|
||||
return {
|
||||
char: entry.char,
|
||||
p: entry.p,
|
||||
traversal: () => entry.traversal()
|
||||
};
|
||||
}),
|
||||
// Make sure to track the model-source's weighting!
|
||||
weight: weight
|
||||
};
|
||||
});
|
||||
|
||||
// If there's only a single active component, well, there actually are
|
||||
// no duplicates - thus it is the right answer.
|
||||
if(unblendedComponents.length == 1) {
|
||||
const singleSource = unblendedComponents[0];
|
||||
return singleSource.sample.map((child) => {
|
||||
return {
|
||||
char: child.char,
|
||||
p: singleSource.weight * child.p,
|
||||
traversal: () => new CompositedTraversal([{ sample: child.traversal(), p: singleSource.weight }])
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
// Now, the tricky part: actual blending.
|
||||
|
||||
// Find sets of traversals that traverse the same char path after the represented prefix.
|
||||
const componentBuckets: Record<string, { component: ChildEdge, weight: number }[]> = {};
|
||||
unblendedComponents.forEach((componentChildren) => {
|
||||
componentChildren.sample.forEach((entry) => {
|
||||
componentBuckets[entry.char] ??= [];
|
||||
componentBuckets[entry.char].push({
|
||||
component: entry,
|
||||
weight: componentChildren.weight
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// For each char path...
|
||||
const keys = Object.keys(componentBuckets);
|
||||
return keys.map((key) => {
|
||||
const bucket = componentBuckets[key];
|
||||
// Build the blended traversal for that path
|
||||
const traversalFunc = () => {
|
||||
const traversalsWithWeighting = bucket.map((entry) => {
|
||||
return {
|
||||
sample: entry.component.traversal(),
|
||||
// Apply the model's weighting!
|
||||
// (This is not the `p` for that model's traversal path.)
|
||||
p: entry.weight
|
||||
};
|
||||
});
|
||||
return new CompositedTraversal(traversalsWithWeighting);
|
||||
};
|
||||
|
||||
// If there's only one contributing model, there's no need to worry about duplicates
|
||||
// affecting the "highest probability" value for the path; no blending needed.
|
||||
//
|
||||
// It's thus safe to just return the model's probability for the path.
|
||||
if(bucket.length == 1) {
|
||||
return {
|
||||
char: key,
|
||||
p: bucket[0].weight * bucket[0].component.p,
|
||||
traversal: traversalFunc
|
||||
};
|
||||
} else {
|
||||
// Here, there's a non-zero chance that duplicate entries may exist.
|
||||
// We thus need to worry about the potential for blending...
|
||||
// which means we're best off delegating "highest probability" calculation
|
||||
// to the child, which is the level at which the issue may be resolved.
|
||||
const traversal = traversalFunc();
|
||||
|
||||
return {
|
||||
char: key,
|
||||
// Use the traversal early to ensure duplicate entries aren't improperly
|
||||
// neglected due to their probability sources being split.
|
||||
p: traversal.p,
|
||||
// We've actually built it already, so just re-use the instance.
|
||||
traversal: () => traversal
|
||||
};
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
child(char: string): LexiconTraversal {
|
||||
let children = this.componentTraversals.map((traversalTuple) => {
|
||||
return {
|
||||
sample: traversalTuple.sample.child(char),
|
||||
p: traversalTuple.p
|
||||
};
|
||||
});
|
||||
|
||||
children = children.filter((entry) => !!entry.sample);
|
||||
|
||||
if(children.length == 0) {
|
||||
return undefined;
|
||||
} else {
|
||||
// Will sort out blending on its own.
|
||||
return new CompositedTraversal(children);
|
||||
}
|
||||
}
|
||||
|
||||
get entries(): TextWithProbability[] {
|
||||
// If there were guaranteed to be no duplicates, .flatMap instead of .map would be fine here.
|
||||
const entrySets = this.componentTraversals.map((traversalTuple) => {
|
||||
return traversalTuple.sample.entries.map((entry) => {
|
||||
const reweightedEntry: TextWithProbability = {
|
||||
text: entry.text,
|
||||
p: entry.p * traversalTuple.p
|
||||
};
|
||||
|
||||
return reweightedEntry;
|
||||
});
|
||||
});
|
||||
|
||||
// Combine any duplicated entries.
|
||||
const resultMap: Record<string, TextWithProbability> = {};
|
||||
entrySets.forEach((set) => {
|
||||
set.forEach((entry) => {
|
||||
if(resultMap[entry.text]) {
|
||||
resultMap[entry.text].p += entry.p;
|
||||
} else {
|
||||
resultMap[entry.text] = entry;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Finalize as array + sort by descending probability
|
||||
const results = Object.values(resultMap);
|
||||
results.sort((a, b) => b.p - a.p);
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
get p(): number {
|
||||
// If it is impossible for a blended traversal path to yield a higher-
|
||||
// probability entry than the best unblended result, there's no point
|
||||
// in considering it here.
|
||||
const bestUnblendedProb = this.componentTraversals
|
||||
// to 'traversal weighting' * 'traversal node's max probability'
|
||||
.map((entry) => entry.p * entry.sample.p)
|
||||
// find the best entry post-weighting
|
||||
.reduce((best, current) => (current > best) ? current : best, 0);
|
||||
|
||||
// Easy case: 1 component at this level? No blending is possible,
|
||||
// so just return the best entry found this way.
|
||||
if(this.componentTraversals.length == 1) {
|
||||
return bestUnblendedProb;
|
||||
}
|
||||
|
||||
// Check for traversal children that travel the same path. O(sum of child
|
||||
// nodes across all components)
|
||||
//
|
||||
// If we wanted to try optimizing this...
|
||||
// - priority queue for each componentTraversal, descending prob order
|
||||
// - track which sources have contributed while accumulating path probs
|
||||
// - if best entry from sources not contributed + current accumulation can't
|
||||
// win, we can stop blending early.
|
||||
// - further iterations can no longer exceed bestUnblendedProb.
|
||||
//
|
||||
// This likely avoids iterating over all paths... but invoking priority
|
||||
// queues means worst-case O(sum N_i log N_i), which is worse. (With N_i =
|
||||
// path count per component traversal)
|
||||
// - constructable "worst case": fully balanced pMaxes?
|
||||
// - for "good cases"... would the complexity even be worth it?
|
||||
// - suppose 3 sources + English: worst-case iteration count with current
|
||||
// setup is 4 * 26 = 104. That's not "terrible"; fairly low to be
|
||||
// considering big-O optimizations.
|
||||
// - +1 multiplier due to the following `filter` block.
|
||||
const childBuckets: Record<string, { p?: number, pMax: number, char: string }> = {};
|
||||
const entryBuckets: Record<string, number> = {};
|
||||
this.componentTraversals.forEach((entry) => {
|
||||
const weight = entry.p;
|
||||
const component = entry.sample;
|
||||
const entries = component.entries;
|
||||
|
||||
component.children().forEach((child) => {
|
||||
const record = childBuckets[child.char] ??= { pMax: 0, char: child.char };
|
||||
record.pMax += weight * child.p;
|
||||
});
|
||||
|
||||
entries.forEach((entry) => entryBuckets[entry.text] = (entryBuckets[entry.text] ?? 0) + entry.p * weight);
|
||||
});
|
||||
|
||||
// If a path's maximum potential probability does not exceed the best entry,
|
||||
// it can't affect our answer; skip it.
|
||||
// Is O(alphabet count)
|
||||
const potentialBlendedChildren = Object.keys(childBuckets).map((char) => {
|
||||
const record = childBuckets[char];
|
||||
if(record.pMax <= bestUnblendedProb) {
|
||||
return null;
|
||||
} else {
|
||||
return record;
|
||||
}
|
||||
}).filter((record) => !!record);
|
||||
|
||||
let bestBlendedEntry = Object.values(entryBuckets).reduce((best, current) => Math.max(best, current), 0);
|
||||
|
||||
// Prep the path-search for a potential winner.
|
||||
let bestProb = Math.max(bestUnblendedProb, bestBlendedEntry);
|
||||
// May have zero entries if no blended path can exceed the best non-blended path's probability.
|
||||
// (Assumption: this is the common case.)
|
||||
const queue = new PriorityQueue((a, b) => b.pMax - a.pMax, potentialBlendedChildren);
|
||||
|
||||
while(queue.peek()?.pMax > bestProb) {
|
||||
const entry = queue.dequeue();
|
||||
const childPath = this.child(entry.char);
|
||||
|
||||
// Note: this pattern, as-is, does not allow cross-node optimizations
|
||||
// (such as seeing a better potential here once we've started evaluating
|
||||
// potentials within the child)
|
||||
//
|
||||
// It's a reasonable "first pass" though.
|
||||
bestProb = Math.max(childPath.p, bestProb);
|
||||
}
|
||||
return bestProb;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,172 @@
|
|||
import { assert } from 'chai';
|
||||
|
||||
import { CompositedTraversal } from '#./models/composited-traversal.js';
|
||||
import { TrieBuilder } from '@keymanapp/models-templates';
|
||||
|
||||
const simpleLexicalModel = (function() {
|
||||
const builder = new TrieBuilder((str) => str);
|
||||
|
||||
// Sum total: 10.
|
||||
builder.addEntry('are', 1);
|
||||
builder.addEntry('and', 2);
|
||||
|
||||
builder.addEntry('not', 1);
|
||||
|
||||
builder.addEntry('the', 3);
|
||||
builder.addEntry('try', 1);
|
||||
builder.addEntry('to', 1);
|
||||
builder.addEntry('tom', 1);
|
||||
|
||||
return builder;
|
||||
}());
|
||||
|
||||
const namesModel = (function() {
|
||||
const builder = new TrieBuilder((str) => str);
|
||||
|
||||
builder.addEntry('amy', 1);
|
||||
builder.addEntry('bob', 1);
|
||||
builder.addEntry('mark', 1);
|
||||
builder.addEntry('tom', 1);
|
||||
|
||||
return builder;
|
||||
}());
|
||||
|
||||
const blendedModel = new CompositedTraversal([
|
||||
{ sample: simpleLexicalModel.traverseFromRoot(), p: 0.5 },
|
||||
{ sample: namesModel.traverseFromRoot(), p: 0.5 }
|
||||
]);
|
||||
|
||||
describe('CompositedTraversal', () => {
|
||||
describe('direct prefix lookup', () => {
|
||||
it('returns undefined for non-existent paths', () => {
|
||||
assert.isUndefined(blendedModel.child('tomcat'));
|
||||
assert.isUndefined(blendedModel.child('tone'));
|
||||
assert.isUndefined(blendedModel.child('ton'));
|
||||
assert.isUndefined(blendedModel.child('apple'));
|
||||
assert.isUndefined(blendedModel.child('aim'));
|
||||
assert.isUndefined(blendedModel.child('xylophone'));
|
||||
});
|
||||
|
||||
it('reweighs entries found in only one source', () => {
|
||||
const andNode = blendedModel.child('and');
|
||||
assert.equal(andNode.p, 0.1);
|
||||
|
||||
assert.equal(andNode.entries.length, 1);
|
||||
const and = andNode.entries[0];
|
||||
assert.equal(and.text, 'and');
|
||||
assert.equal(and.p, 0.1);
|
||||
});
|
||||
|
||||
it('merges entries found in multiple sources, combining their probability and entries', () => {
|
||||
const tomNode = blendedModel.child('tom');
|
||||
assert.equal(tomNode.p, 0.175);
|
||||
|
||||
assert.equal(tomNode.entries.length, 1);
|
||||
const tom = tomNode.entries[0];
|
||||
assert.equal(tom.text, 'tom');
|
||||
assert.equal(tom.p, 0.175);
|
||||
});
|
||||
|
||||
it('correctly returns the probability of intermediate child paths', () => {
|
||||
// simpleLexical: best is 'and', 0.2
|
||||
// names: best is 'amy', 0.25
|
||||
// Weight of each is equal at 0.5; 0.25 * 0.5 = 0.125.
|
||||
// Should not combine.
|
||||
const aNode = blendedModel.child('a');
|
||||
assert.equal(aNode.p, 0.125, "probability of paths were merged despite lack of duplicate");
|
||||
assert.equal(aNode.entries.length, 0);
|
||||
|
||||
// 'tom' is on both paths, and is highest-frequency entry when merged.
|
||||
// We want the combined, net probability to show up on its predecessor paths.
|
||||
const toNode = blendedModel.child('to');
|
||||
assert.equal(toNode.p, 0.175, "probability of path with winning duplicate not merged");
|
||||
|
||||
// simpleLexical - p('to') = 0.1; is then multiplied by model weight of 0.5.
|
||||
assert.equal(toNode.entries.length, 1);
|
||||
const to = toNode.entries[0];
|
||||
assert.equal(to.text, 'to');
|
||||
assert.equal(to.p, 0.05);
|
||||
});
|
||||
});
|
||||
|
||||
describe('iteration for correction-search', () => {
|
||||
it('correctly enumerates child paths from root', () => {
|
||||
const topChildren = blendedModel.children();
|
||||
// 'n' from "simpleLexical", 'b' and 'm' from "names"
|
||||
assert.sameMembers(topChildren.map((child) => child.char), ['a', 'b', 'm', 'n', 't']);
|
||||
assert.sameDeepMembers(topChildren.map((child) => {
|
||||
return {
|
||||
char: child.char,
|
||||
p: child.p
|
||||
}
|
||||
}), [
|
||||
{ char: 'a', p: .125 }, // highest: 'amy'
|
||||
{ char: 'b', p: .125 }, // highest: 'bob'
|
||||
{ char: 'm', p: .125 }, // highest: 'mark'
|
||||
{ char: 'n', p: .05 }, // highest: 'not'
|
||||
{ char: 't', p: .175 } // highest: 'tom'
|
||||
]);
|
||||
topChildren.forEach((entry) => assert.equal(entry.p, entry.traversal().p));
|
||||
});
|
||||
|
||||
it('correctly enumerates child paths from section supported by a single member', () => {
|
||||
const topChildren = blendedModel.children();
|
||||
const bNode = topChildren.find((entry) => entry.char == 'b').traversal();
|
||||
const bChildren = bNode.children();
|
||||
assert.sameDeepMembers(bChildren.map((child) => {
|
||||
return {
|
||||
char: child.char,
|
||||
p: child.p
|
||||
};
|
||||
}), [
|
||||
{ char: 'o', p: 0.125 }
|
||||
]);
|
||||
bChildren.forEach((entry) => assert.equal(entry.p, entry.traversal().p));
|
||||
|
||||
const boNode = bChildren.find((entry) => entry.char == 'o').traversal();
|
||||
const boChildren = boNode.children();
|
||||
assert.sameDeepMembers(boChildren.map((child) => {
|
||||
return {
|
||||
char: child.char,
|
||||
p: child.p
|
||||
};
|
||||
}), [
|
||||
{ char: 'b', p: 0.125 }
|
||||
]);
|
||||
boChildren.forEach((entry) => assert.equal(entry.p, entry.traversal().p));
|
||||
});
|
||||
|
||||
it('correctly enumerates child paths from section supported by multiple members', () => {
|
||||
const topChildren = blendedModel.children();
|
||||
const tNode = topChildren.find((entry) => entry.char == 't').traversal();
|
||||
const tChildren = tNode.children();
|
||||
|
||||
assert.sameMembers(tChildren.map((child) => child.char), ['h', 'o', 'r']);
|
||||
assert.sameDeepMembers(tChildren.map((child) => {
|
||||
return {
|
||||
char: child.char,
|
||||
p: child.p
|
||||
}
|
||||
}), [
|
||||
{ char: 'h', p: 0.15 },
|
||||
{ char: 'o', p: 0.175 },
|
||||
{ char: 'r', p: 0.05 }
|
||||
]);
|
||||
tChildren.forEach((entry) => assert.equal(entry.p, entry.traversal().p));
|
||||
|
||||
const toNode = tChildren.find((entry) => entry.char == 'o').traversal();
|
||||
const toChildren = toNode.children();
|
||||
|
||||
assert.sameMembers(toChildren.map((child) => child.char), ['m']);
|
||||
assert.sameDeepMembers(toChildren.map((child) => {
|
||||
return {
|
||||
char: child.char,
|
||||
p: child.p
|
||||
};
|
||||
}), [
|
||||
{ char: 'm', p: 0.175 }
|
||||
]);
|
||||
toChildren.forEach((entry) => assert.equal(entry.p, entry.traversal().p));
|
||||
});
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Reference in a new issue