mirror of
https://github.com/keymanapp/keyman.git
synced 2026-08-13 04:09:25 +00:00
448 lines
15 KiB
TypeScript
448 lines
15 KiB
TypeScript
import { readFileSync } from "fs";
|
||
|
||
/**
|
||
* A word list is an array of pairs: the concrete word form itself, followed by
|
||
* a non-negative count.
|
||
*/
|
||
type WordList = [string, number][];
|
||
|
||
/**
|
||
* Returns a data structure that can be loaded by the TrieModel.
|
||
*
|
||
* It implements a **weighted** trie, whose indices (paths down the trie) are
|
||
* generated by a search key, and not concrete wordforms themselves.
|
||
*
|
||
* @param sourceFiles an array of source files that will be read to generate the trie.
|
||
*/
|
||
export function createTrieDataStructure(filenames: string[], searchTermToKey?: (wf: string) => string): string {
|
||
if (typeof searchTermToKey !== "function") {
|
||
throw new TypeError("searchTermToKey must be explicitly specified")
|
||
}
|
||
// Make one big word list out of all of the filenames provided.
|
||
let wordlist = filenames
|
||
.map(parseWordListFromFilename)
|
||
.reduce((bigWordlist, current) => bigWordlist.concat(current), []);
|
||
let trie = Trie.buildTrie(wordlist, searchTermToKey as Trie.SearchTermToKey);
|
||
return JSON.stringify(trie);
|
||
}
|
||
|
||
/**
|
||
* Parses a word list from its filename.
|
||
*
|
||
* The word list may be encoded in:
|
||
*
|
||
* - UTF-8, with or without BOM [exported by most software]
|
||
* - UTF-16, little endian, with BOM [exported by Microsoft Excel]
|
||
*
|
||
* @param filename filename of the word list
|
||
*/
|
||
export function parseWordListFromFilename(filename: string): WordList {
|
||
let contents = readFileSync(filename, detectEncoding(filename));
|
||
return parseWordList(contents);
|
||
}
|
||
|
||
/**
|
||
* Reads a tab-separated values file into a word list.
|
||
*
|
||
* Format specification:
|
||
*
|
||
* - the file is a UTF-8 encoded text file.
|
||
* - new lines are either LF or CRLF.
|
||
* - the file MAY start with the UTF-8 byte-order mark (BOM); that is, if the
|
||
* first three bytes of the file are EF BB BF, these will be interepreted as
|
||
* the BOM and will be ignored.
|
||
* - the file either consists of a comment or an entry.
|
||
* - comment lines MUST start with the '#' character on the very first column.
|
||
* - entries are one to three columns, separated by the (horizontal) tab
|
||
* character.
|
||
* - column 1 (REQUIRED): the wordform: can have any character except tab, CR,
|
||
* LF. Surrounding whitespace characters are trimmed.
|
||
* - column 2 (optional): the count: a non-negative integer specifying how many
|
||
* times this entry has appeared in the corpus. Blank means 'indeterminate'.
|
||
* - column 3 (optional): comment: an informative comment, ignored by the tool.
|
||
*/
|
||
export function parseWordList(contents: string): WordList {
|
||
// Supports LF or CRLF line terminators.
|
||
const NEWLINE_SEPARATOR = /\u000d?\u000a/;
|
||
const TAB = "\t";
|
||
// TODO: format validation.
|
||
let lines = contents.split(NEWLINE_SEPARATOR);
|
||
|
||
let result: WordList = [];
|
||
for (let line of lines) {
|
||
// Remove the byte-order mark (BOM) from the beginning of the string.
|
||
// Because `contents` can be the concatenation of several files, we have to remove
|
||
// the BOM from every possible start of file -- i.e., beginning of every line.
|
||
line = line.replace(/^\uFEFF/, '');
|
||
|
||
if (line.startsWith('#') || line === "") {
|
||
continue; // skip comments and empty lines
|
||
}
|
||
|
||
// The third column is the comment. Always ignored!
|
||
let [wordform, countText] = line.split(TAB);
|
||
|
||
// Clean the word form.
|
||
// TODO: what happens if we get duplicate forms?
|
||
wordform = wordform.normalize('NFC').trim();
|
||
countText = (countText || '').trim();
|
||
let count = parseInt(countText, 10);
|
||
|
||
// When parsing a decimal integer fails (e.g., blank or something else):
|
||
if (!isFinite(count)) {
|
||
// TODO: is this the right thing to do?
|
||
// Treat it like a hapax legonmenom -- it exist, but only once.
|
||
count = 1;
|
||
}
|
||
result.push([wordform, count]);
|
||
}
|
||
return result;
|
||
}
|
||
|
||
namespace Trie {
|
||
/**
|
||
* An **opaque** type for a string that is exclusively used as a search key in
|
||
* the trie. There should be a function that converts arbitrary strings
|
||
* (queries) and converts them into a standard search key for a given language
|
||
* model.
|
||
*
|
||
* Fun fact: This opaque type has ALREADY saved my bacon and found a bug!
|
||
*/
|
||
type SearchKey = string & { _: 'SearchKey'};
|
||
|
||
/**
|
||
* A function that converts a string (word form or query) into a search key
|
||
* (secretly, this is also a string).
|
||
*/
|
||
export interface SearchTermToKey {
|
||
(wordform: string): SearchKey;
|
||
}
|
||
|
||
// The following trie implementation has been (heavily) derived from trie-ing
|
||
// by Conrad Irwin.
|
||
//
|
||
// trie-ing is distributed under the terms of the MIT license, reproduced here:
|
||
//
|
||
// The MIT License
|
||
// Copyright (c) 2015-2017 Conrad Irwin <conrad.irwin@gmail.com>
|
||
// Copyright (c) 2011 Marc Campbell <marc.e.campbell@gmail.com>
|
||
//
|
||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||
// of this software and associated documentation files (the "Software"), to deal
|
||
// in the Software without restriction, including without limitation the rights
|
||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||
// copies of the Software, and to permit persons to whom the Software is
|
||
// furnished to do so, subject to the following conditions:
|
||
//
|
||
// The above copyright notice and this permission notice shall be included in
|
||
// all copies or substantial portions of the Software.
|
||
//
|
||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||
// THE SOFTWARE.
|
||
//
|
||
// See: https://github.com/ConradIrwin/trie-ing/blob/df55d7af7068d357829db9e0a7faa8a38add1d1d/LICENSE
|
||
|
||
/**
|
||
* An entry in the prefix trie. The matched word is "content".
|
||
*/
|
||
interface Entry {
|
||
content: string;
|
||
key: SearchKey;
|
||
weight: number;
|
||
}
|
||
|
||
/**
|
||
* The trie is made up of nodes. A node can be EITHER an internal node (whose
|
||
* only children are other nodes) OR a leaf, which actually contains the word
|
||
* form entries.
|
||
*/
|
||
type Node = InternalNode | Leaf;
|
||
|
||
/**
|
||
* An internal node.
|
||
*/
|
||
interface InternalNode {
|
||
type: 'internal';
|
||
weight: number;
|
||
// TODO: As an optimization, "values" can be a single string!
|
||
values: string[];
|
||
children: { [codeunit: string]: Node };
|
||
unsorted?: true;
|
||
}
|
||
|
||
/**
|
||
* A leaf node.
|
||
*/
|
||
interface Leaf {
|
||
type: 'leaf';
|
||
weight: number;
|
||
entries: Entry[];
|
||
unsorted?: true;
|
||
}
|
||
|
||
/**
|
||
* A sentinel value for when an internal node has contents and requires an
|
||
* "internal" leaf. That is, this internal node has content. Instead of placing
|
||
* entries as children in an internal node, a "fake" leaf is created, and its
|
||
* key is this special internal value.
|
||
*
|
||
* The value is a valid Unicode BMP code point, but it is a "non-character".
|
||
* Unicode will never assign semantics to these characters, as they are
|
||
* intended to be used internally as sentinel values.
|
||
*/
|
||
const INTERNAL_VALUE = '\uFDD0';
|
||
|
||
/**
|
||
* Builds a trie from a word list.
|
||
*
|
||
* @param wordlist The wordlist with non-negative weights.
|
||
* @param keyFunction Function that converts word forms into indexed search keys
|
||
* @returns A JSON-serialiable object that can be given to the TrieModel constructor.
|
||
*/
|
||
export function buildTrie(wordlist: WordList, keyFunction: SearchTermToKey): object {
|
||
let root = new Trie(keyFunction).buildFromWordList(wordlist).root;
|
||
return {
|
||
totalWeight: sumWeights(root),
|
||
root: root
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Wrapper class for the trie and its nodes and wordform to search
|
||
*/
|
||
class Trie {
|
||
readonly root = createRootNode();
|
||
toKey: SearchTermToKey;
|
||
constructor(wordform2key: SearchTermToKey) {
|
||
this.toKey = wordform2key;
|
||
}
|
||
|
||
/**
|
||
* Populates the trie with the contents of an entire wordlist.
|
||
* @param words a list of word and count pairs.
|
||
*/
|
||
buildFromWordList(words: WordList): Trie {
|
||
for (let [wordform, weight] of words) {
|
||
let key = this.toKey(wordform);
|
||
addUnsorted(this.root, { key, weight, content: wordform }, 0);
|
||
}
|
||
sortTrie(this.root);
|
||
return this;
|
||
}
|
||
}
|
||
|
||
// "Constructors"
|
||
function createRootNode(): Node {
|
||
return {
|
||
type: 'leaf',
|
||
weight: 0,
|
||
entries: []
|
||
};
|
||
}
|
||
|
||
// Implement Trie creation.
|
||
|
||
/**
|
||
* Adds an entry to the trie.
|
||
*
|
||
* Note that the trie will likely be unsorted after the add occurs. Before
|
||
* performing a lookup on the trie, use call sortTrie() on the root note!
|
||
*
|
||
* @param node Which node should the entry be added to?
|
||
* @param entry the wordform/weight/key to add to the trie
|
||
* @param index the index in the key and also the trie depth. Should be set to
|
||
* zero when adding onto the root node of the trie.
|
||
*/
|
||
function addUnsorted(node: Node, entry: Entry, index: number = 0) {
|
||
// Each node stores the MAXIMUM weight out of all of its decesdents, to
|
||
// enable a greedy search through the trie.
|
||
node.weight = Math.max(node.weight, entry.weight);
|
||
|
||
// When should a leaf become an interior node?
|
||
// When it already has a value, but the key of the current value is longer
|
||
// than the prefix.
|
||
if (node.type === 'leaf' && index < entry.key.length && node.entries.length >= 1) {
|
||
convertLeafToInternalNode(node, index);
|
||
}
|
||
|
||
if (node.type === 'leaf') {
|
||
// The key matches this leaf node, so add yet another entry.
|
||
addItemToLeaf(node, entry);
|
||
} else {
|
||
// Push the node down to a lower node.
|
||
addItemToInternalNode(node, entry, index);
|
||
}
|
||
|
||
node.unsorted = true;
|
||
}
|
||
|
||
/**
|
||
* Adds an item to the internal node at a given depth.
|
||
* @param item
|
||
* @param index
|
||
*/
|
||
function addItemToInternalNode(node: InternalNode, item: Entry, index: number) {
|
||
let char = item.key[index];
|
||
if (!node.children[char]) {
|
||
node.children[char] = createRootNode();
|
||
node.values.push(char);
|
||
}
|
||
addUnsorted(node.children[char], item, index + 1);
|
||
}
|
||
|
||
function addItemToLeaf(leaf: Leaf, item: Entry) {
|
||
leaf.entries.push(item);
|
||
}
|
||
|
||
/**
|
||
* Mutates the given Leaf to turn it into an InternalNode.
|
||
*
|
||
* NOTE: the node passed in will be DESTRUCTIVELY CHANGED into a different
|
||
* type when passed into this function!
|
||
*
|
||
* @param depth depth of the trie at this level.
|
||
*/
|
||
function convertLeafToInternalNode(leaf: Leaf, depth: number): void {
|
||
let entries = leaf.entries;
|
||
|
||
// Alias the current node, as the desired type.
|
||
let internal = (<unknown> leaf) as InternalNode;
|
||
internal.type = 'internal';
|
||
|
||
delete leaf.entries;
|
||
internal.values = [];
|
||
internal.children = {};
|
||
|
||
// Convert the old values array into the format for interior nodes.
|
||
for (let item of entries) {
|
||
let char: string;
|
||
if (depth < item.key.length) {
|
||
char = item.key[depth];
|
||
} else {
|
||
char = INTERNAL_VALUE;
|
||
}
|
||
|
||
if (!internal.children[char]) {
|
||
internal.children[char] = createRootNode();
|
||
internal.values.push(char);
|
||
}
|
||
addUnsorted(internal.children[char], item, depth + 1);
|
||
}
|
||
|
||
internal.unsorted = true;
|
||
}
|
||
|
||
/**
|
||
* Recursively sort the trie, in descending order of weight.
|
||
* @param node any node in the trie
|
||
*/
|
||
function sortTrie(node: Node) {
|
||
if (node.type === 'leaf') {
|
||
if (!node.unsorted) {
|
||
return;
|
||
}
|
||
|
||
node.entries.sort(function (a, b) { return b.weight - a.weight; });
|
||
} else {
|
||
// We MUST recurse and sort children before returning.
|
||
for (let char of node.values) {
|
||
sortTrie(node.children[char]);
|
||
}
|
||
|
||
if (!node.unsorted) {
|
||
return;
|
||
}
|
||
|
||
node.values.sort((a, b) => {
|
||
return node.children[b].weight - node.children[a].weight;
|
||
});
|
||
}
|
||
|
||
delete node.unsorted;
|
||
}
|
||
|
||
/**
|
||
* O(n) recursive traversal to sum the total weight of all leaves in the
|
||
* trie, starting at the provided node.
|
||
*
|
||
* @param node The node to start summing weights.
|
||
*/
|
||
function sumWeights(node: Node): number {
|
||
if (node.type === 'leaf') {
|
||
return node.entries
|
||
.map(entry => entry.weight)
|
||
.reduce((acc, count) => acc + count, 0);
|
||
} else {
|
||
return Object.keys(node.children)
|
||
.map((key) => sumWeights(node.children[key]))
|
||
.reduce((acc, count) => acc + count, 0);
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Converts wordforms into an indexable form. It does this by
|
||
* normalizing the letter case of characters INDIVIDUALLY (to disregard
|
||
* context-sensitive case transformations), normalizing to NFKD form,
|
||
* and removing common diacritical marks.
|
||
*
|
||
* This is a very speculative implementation, that might work with
|
||
* your language. We don't guarantee that this will be perfect for your
|
||
* language, but it's a start.
|
||
*
|
||
* This uses String.prototype.normalize() to convert normalize into NFKD.
|
||
* NFKD neutralizes some funky distinctions, e.g., ꬲ, ｅ, e should all be the
|
||
* same character; plus, it's an easy way to separate a Latin character from
|
||
* its diacritics; Even then, orthographies regularly use code points
|
||
* that, under NFKD normalization, do NOT decompose appropriately for your
|
||
* language (e.g., SENĆOŦEN, Plains Cree in syllabics).
|
||
*
|
||
* Use this in early iterations of the model. For a production lexical model,
|
||
* you will probably write/generate your own key function, tailored to your
|
||
* language. There is a chance the default will work properly out of the box.
|
||
*/
|
||
export function defaultSearchTermToKey(wordform: string): string {
|
||
return Array.from(wordform)
|
||
.map(c => c.toLowerCase())
|
||
.join('')
|
||
.normalize('NFKD')
|
||
// Remove any combining diacritics (if input is in NFKD)
|
||
.replace(/[\u0300-\u036F]/g, '');
|
||
}
|
||
|
||
|
||
/**
|
||
* Detects the encoding of a text file.
|
||
*
|
||
* Supported encodings are:
|
||
*
|
||
* - UTF-8, with or without BOM
|
||
* - UTF-16, little endian, with BOM
|
||
*
|
||
* UTF-16 in big endian is explicitly NOT supported! The reason is two-fold:
|
||
* 1) Node does not support it without resorting to an external library (or
|
||
* swapping every byte in the file!); and 2) I'm not sure anything actually
|
||
* outputs in this format anyway!
|
||
*
|
||
* @param filename filename of the file to detect encoding
|
||
*/
|
||
function detectEncoding(filename: string): 'utf8' | 'utf16le' {
|
||
let buffer = readFileSync(filename);
|
||
// Note: BOM is U+FEFF
|
||
// In little endian, this is 0xFF 0xFE
|
||
if (buffer[0] == 0xFF && buffer[1] == 0xFE) {
|
||
return 'utf16le';
|
||
} else if (buffer[0] == 0xFE && buffer[1] == 0xFF) {
|
||
// Big Endian, is NOT supported because Node does not support it (???)
|
||
// See: https://stackoverflow.com/a/14551669/6626414
|
||
throw new Error('UTF-16BE is unsupported')
|
||
} else {
|
||
// Assume its in UTF-8, with or without a BOM.
|
||
return 'utf8';
|
||
}
|
||
}
|