Merge pull request #3340 from keymanapp/refactor/developer/hashmap-based-wordlist-compile

refactor(developer): hashmap-based wordlist compilation

developer/js/source/lexical-model-compiler/build-trie.ts

@@ -1,10 +1,13 @@
 import { readFileSync } from "fs";
 
 /**
- * A word list is an array of pairs: the concrete word form itself, followed by
- * a non-negative count.
+ * A word list is (conceptually) an array of pairs: the concrete word form itself + a
+ * non-negative count.
+ * 
+ * Since each word should only appear once within the list, we represent it with
+ * an associative array pattern keyed by the wordform.
  */
-export type WordList = [string, number][];
+export type WordList = {[wordform: string]: number};
 
 /**
  * Returns a data structure that can be loaded by the TrieModel.
@@ -19,7 +22,7 @@ export function createTrieDataStructure(filenames: string[], searchTermToKey?: (
     throw new TypeError("searchTermToKey must be explicitly specified")
   }
   // Make one big word list out of all of the filenames provided.
-  let wordlist: WordList = [];
+  let wordlist: WordList = {};
   filenames.forEach(filename => parseWordListFromFilename(wordlist, filename));
 
   let trie = Trie.buildTrie(wordlist, searchTermToKey as Trie.SearchTermToKey);
@@ -101,17 +104,7 @@ export function parseWordList(wordlist: WordList, contents: string): void {
       count = 1;
     }
 
-    // TODO: this merge is very naive. We should consider whether the merge
-    // needs to be a little more aggressive. This may also be slow for large
-    // wordlists; probably O(n log n). We could improve this with a hash table
-    // if it becomes a performance problem.
-    const item = wordlist.find(value => value[0] === wordform);
-    if(item) {
-      item[1] += count;
-    }
-    else {
-      wordlist.push([wordform, count]);
-    }
+    wordlist[wordform] = (wordlist[wordform] || 0) + count;
   }
 }
 
@@ -243,7 +236,7 @@ namespace Trie {
      * @param words a list of word and count pairs.
      */
     buildFromWordList(words: WordList): Trie {
-      for (let [wordform, weight] of words) {
+      for (let [wordform, weight] of Object.entries(words)) {
         let key = this.toKey(wordform);
         addUnsorted(this.root, { key, weight, content: wordform }, 0);
       }

developer/js/tests/test-parse-wordlist.ts

@@ -4,31 +4,31 @@ import 'mocha';
 import { makePathToFixture } from './helpers';
 
 const BOM = '\ufeff';
-const SENCOTEN_WORDLIST = [
-  ['TŦE', 13644],
-  ['E', 9134],
-  ['SEN', 4816],
-  ['Ȼ', 3479],
-  ['SW̱', 2621],
-  ['NIȽ', 2314],
-  ['U¸', 2298],
-  ['I¸', 1988],
-  ['ȻSE', 1925],
-  ['I', 1884]
-];
+const SENCOTEN_WORDLIST = {
+  'TŦE': 13644,
+  'E': 9134,
+  'SEN': 4816,
+  'Ȼ': 3479,
+  'SW̱': 2621,
+  'NIȽ': 2314,
+  'U¸': 2298,
+  'I¸': 1988,
+  'ȻSE': 1925,
+  'I': 1884
+};
 
 describe('parseWordList', function () {
   it('should remove the UTF-8 byte order mark from files', function () {
     let word = 'hello';
     let count = 1;
-    let expected = [
-      [word, count]
-    ];
+    let expected: WordList = {};
+    expected[word] = count;
+
     let file = `# this is a comment\n${word}\t${count}`;
-    let withoutBOM: WordList = [];
+    let withoutBOM: WordList = {};
     parseWordList(withoutBOM, file);
     assert.deepEqual(withoutBOM, expected, "expected regular file to parse properly");
-    let withBOM: WordList = [];
+    let withBOM: WordList = {};
     parseWordList(withBOM, `${BOM}${file}`)
     assert.deepEqual(withBOM, expected, "expected BOM to be ignored");
   });
@@ -36,7 +36,7 @@ describe('parseWordList', function () {
   it('should read word lists in UTF-8', function () {
     // N.B.: this is the format exported by Google Drive when selecting "TSV".
     const filename = makePathToFixture('example.qaa.sencoten', 'wordlist.tsv');
-    let wordlist: WordList = [];
+    let wordlist: WordList = {};
     parseWordListFromFilename(wordlist, filename);
     assert.deepEqual(wordlist, SENCOTEN_WORDLIST);
   });
@@ -45,7 +45,7 @@ describe('parseWordList', function () {
     // N.B.: this is the format exported by MS Excel when selecting
     // "UTF-16" text (tested on Excel for macOS).
     const filename = makePathToFixture('example.qaa.utf16le', 'wordlist.txt');
-    let wordlist: WordList = [];
+    let wordlist: WordList = {};
     parseWordListFromFilename(wordlist, filename);
     assert.deepEqual(wordlist, SENCOTEN_WORDLIST);
   });
@@ -53,7 +53,7 @@ describe('parseWordList', function () {
   it('should NOT read word lists in UTF-16 big-endian (with BOM)', function () {
     // N.B.: Does anything output this format...?
     const filename = makePathToFixture('example.qaa.utf16be', 'wordlist.txt');
-    let wordlist: WordList = [];
+    let wordlist: WordList = {};
     assert.throws(() => {
       parseWordListFromFilename(wordlist, filename);
     }, 'UTF-16BE is unsupported');
@@ -69,20 +69,19 @@ describe('parseWordList', function () {
       ' hello ', //4, expect to trim whitespace
       'hello']; //5
 
-    const expected = [
-      [ 'hello', 10 /* 1+4+5 trimmed and identical */ ],
-      [ 'hell'+String.fromCharCode(0x00F3), 5 /* 2+3 normalised to NFC */ ]
-    ];
+    const expected: WordList = {
+        'hello': 10, /* 1+4+5 trimmed and identical */
+        'hell\u00f3': 5, /* 2+3 normalised to NFC */
+    };
 
     // Build a wordlist from the array
     let file = `# this is a comment\n`;
     for(let i = 0; i < words.length; i++) {
       file += `${words[i]}\t${i+1}\n`;
     }
-    let repeatedWords: WordList = [];
+    let repeatedWords: WordList = {};
     parseWordList(repeatedWords, file);
 
     assert.deepEqual(repeatedWords, expected);
   });
 });
-