From 3f4aa0e9d91c861c4596ee19e9ba8aff69d7f02d Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Tue, 4 Jun 2019 09:37:45 -0600 Subject: [PATCH 1/2] Remove String.prototype.normalize() in favor of a lookup table :/ --- .../worker/models/trie-model.ts | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/common/predictive-text/worker/models/trie-model.ts b/common/predictive-text/worker/models/trie-model.ts index f04b037aac..cec1070fac 100644 --- a/common/predictive-text/worker/models/trie-model.ts +++ b/common/predictive-text/worker/models/trie-model.ts @@ -428,13 +428,26 @@ * language. */ function defaultWordform2Key(wordform: string): SearchKey { - // Use this pattern to remove common diacritical marks. - // See: https://www.compart.com/en/unicode/block/U+0300 - const COMBINING_DIACRITICAL_MARKS = /[\u0300-\u036f]/g; return wordform - .normalize('NFD') - .toLowerCase() - // remove diacritical marks. - .replace(COMBINING_DIACRITICAL_MARKS, '') as SearchKey; + .replace(/[\u0100-\u2200]/, function (c) { + if (c in PARTIAL_NFD_LOOKUP) { + return PARTIAL_NFD_LOOKUP[c]; + } + return c; + }) + .toLowerCase() as SearchKey; } + + /** + * String.prototype.normalize() is not available on all platforms (*cough* IE + * 11 *cough cough*). We want to use NFD to take off diacritical marks from + * characters so that they are not used in key searches. + * + * This table is of all characters in the range of U+0100 to U+2200 that + * have a canonical decomposition in NFD form. For some characters, this + * translates them into their canonical characters (e.g., K -> K). For + * characters with combining diacritical marks, this leaves behind the base + * character, removing its diacritics (e.g., É -> E). + */ + const PARTIAL_NFD_LOOKUP = {"À":"A","Á":"A","Â":"A","Ã":"A","Ä":"A","Å":"A","Ç":"C","È":"E","É":"E","Ê":"E","Ë":"E","Ì":"I","Í":"I","Î":"I","Ï":"I","Ñ":"N","Ò":"O","Ó":"O","Ô":"O","Õ":"O","Ö":"O","Ù":"U","Ú":"U","Û":"U","Ü":"U","Ý":"Y","à":"a","á":"a","â":"a","ã":"a","ä":"a","å":"a","ç":"c","è":"e","é":"e","ê":"e","ë":"e","ì":"i","í":"i","î":"i","ï":"i","ñ":"n","ò":"o","ó":"o","ô":"o","õ":"o","ö":"o","ù":"u","ú":"u","û":"u","ü":"u","ý":"y","ÿ":"y","Ā":"A","ā":"a","Ă":"A","ă":"a","Ą":"A","ą":"a","Ć":"C","ć":"c","Ĉ":"C","ĉ":"c","Ċ":"C","ċ":"c","Č":"C","č":"c","Ď":"D","ď":"d","Ē":"E","ē":"e","Ĕ":"E","ĕ":"e","Ė":"E","ė":"e","Ę":"E","ę":"e","Ě":"E","ě":"e","Ĝ":"G","ĝ":"g","Ğ":"G","ğ":"g","Ġ":"G","ġ":"g","Ģ":"G","ģ":"g","Ĥ":"H","ĥ":"h","Ĩ":"I","ĩ":"i","Ī":"I","ī":"i","Ĭ":"I","ĭ":"i","Į":"I","į":"i","İ":"I","Ĵ":"J","ĵ":"j","Ķ":"K","ķ":"k","Ĺ":"L","ĺ":"l","Ļ":"L","ļ":"l","Ľ":"L","ľ":"l","Ń":"N","ń":"n","Ņ":"N","ņ":"n","Ň":"N","ň":"n","Ō":"O","ō":"o","Ŏ":"O","ŏ":"o","Ő":"O","ő":"o","Ŕ":"R","ŕ":"r","Ŗ":"R","ŗ":"r","Ř":"R","ř":"r","Ś":"S","ś":"s","Ŝ":"S","ŝ":"s","Ş":"S","ş":"s","Š":"S","š":"s","Ţ":"T","ţ":"t","Ť":"T","ť":"t","Ũ":"U","ũ":"u","Ū":"U","ū":"u","Ŭ":"U","ŭ":"u","Ů":"U","ů":"u","Ű":"U","ű":"u","Ų":"U","ų":"u","Ŵ":"W","ŵ":"w","Ŷ":"Y","ŷ":"y","Ÿ":"Y","Ź":"Z","ź":"z","Ż":"Z","ż":"z","Ž":"Z","ž":"z","Ơ":"O","ơ":"o","Ư":"U","ư":"u","Ǎ":"A","ǎ":"a","Ǐ":"I","ǐ":"i","Ǒ":"O","ǒ":"o","Ǔ":"U","ǔ":"u","Ǖ":"U","ǖ":"u","Ǘ":"U","ǘ":"u","Ǚ":"U","ǚ":"u","Ǜ":"U","ǜ":"u","Ǟ":"A","ǟ":"a","Ǡ":"A","ǡ":"a","Ǣ":"Æ","ǣ":"æ","Ǧ":"G","ǧ":"g","Ǩ":"K","ǩ":"k","Ǫ":"O","ǫ":"o","Ǭ":"O","ǭ":"o","Ǯ":"Ʒ","ǯ":"ʒ","ǰ":"j","Ǵ":"G","ǵ":"g","Ǹ":"N","ǹ":"n","Ǻ":"A","ǻ":"a","Ǽ":"Æ","ǽ":"æ","Ǿ":"Ø","ǿ":"ø","Ȁ":"A","ȁ":"a","Ȃ":"A","ȃ":"a","Ȅ":"E","ȅ":"e","Ȇ":"E","ȇ":"e","Ȉ":"I","ȉ":"i","Ȋ":"I","ȋ":"i","Ȍ":"O","ȍ":"o","Ȏ":"O","ȏ":"o","Ȑ":"R","ȑ":"r","Ȓ":"R","ȓ":"r","Ȕ":"U","ȕ":"u","Ȗ":"U","ȗ":"u","Ș":"S","ș":"s","Ț":"T","ț":"t","Ȟ":"H","ȟ":"h","Ȧ":"A","ȧ":"a","Ȩ":"E","ȩ":"e","Ȫ":"O","ȫ":"o","Ȭ":"O","ȭ":"o","Ȯ":"O","ȯ":"o","Ȱ":"O","ȱ":"o","Ȳ":"Y","ȳ":"y","̀":"̀","́":"́","̓":"̓","̈́":"̈","ʹ":"ʹ",";":";","΅":"¨","Ά":"Α","·":"·","Έ":"Ε","Ή":"Η","Ί":"Ι","Ό":"Ο","Ύ":"Υ","Ώ":"Ω","ΐ":"ι","Ϊ":"Ι","Ϋ":"Υ","ά":"α","έ":"ε","ή":"η","ί":"ι","ΰ":"υ","ϊ":"ι","ϋ":"υ","ό":"ο","ύ":"υ","ώ":"ω","ϓ":"ϒ","ϔ":"ϒ","Ѐ":"Е","Ё":"Е","Ѓ":"Г","Ї":"І","Ќ":"К","Ѝ":"И","Ў":"У","Й":"И","й":"и","ѐ":"е","ё":"е","ѓ":"г","ї":"і","ќ":"к","ѝ":"и","ў":"у","Ѷ":"Ѵ","ѷ":"ѵ","Ӂ":"Ж","ӂ":"ж","Ӑ":"А","ӑ":"а","Ӓ":"А","ӓ":"а","Ӗ":"Е","ӗ":"е","Ӛ":"Ә","ӛ":"ә","Ӝ":"Ж","ӝ":"ж","Ӟ":"З","ӟ":"з","Ӣ":"И","ӣ":"и","Ӥ":"И","ӥ":"и","Ӧ":"О","ӧ":"о","Ӫ":"Ө","ӫ":"ө","Ӭ":"Э","ӭ":"э","Ӯ":"У","ӯ":"у","Ӱ":"У","ӱ":"у","Ӳ":"У","ӳ":"у","Ӵ":"Ч","ӵ":"ч","Ӹ":"Ы","ӹ":"ы","Ḁ":"A","ḁ":"a","Ḃ":"B","ḃ":"b","Ḅ":"B","ḅ":"b","Ḇ":"B","ḇ":"b","Ḉ":"C","ḉ":"c","Ḋ":"D","ḋ":"d","Ḍ":"D","ḍ":"d","Ḏ":"D","ḏ":"d","Ḑ":"D","ḑ":"d","Ḓ":"D","ḓ":"d","Ḕ":"E","ḕ":"e","Ḗ":"E","ḗ":"e","Ḙ":"E","ḙ":"e","Ḛ":"E","ḛ":"e","Ḝ":"E","ḝ":"e","Ḟ":"F","ḟ":"f","Ḡ":"G","ḡ":"g","Ḣ":"H","ḣ":"h","Ḥ":"H","ḥ":"h","Ḧ":"H","ḧ":"h","Ḩ":"H","ḩ":"h","Ḫ":"H","ḫ":"h","Ḭ":"I","ḭ":"i","Ḯ":"I","ḯ":"i","Ḱ":"K","ḱ":"k","Ḳ":"K","ḳ":"k","Ḵ":"K","ḵ":"k","Ḷ":"L","ḷ":"l","Ḹ":"L","ḹ":"l","Ḻ":"L","ḻ":"l","Ḽ":"L","ḽ":"l","Ḿ":"M","ḿ":"m","Ṁ":"M","ṁ":"m","Ṃ":"M","ṃ":"m","Ṅ":"N","ṅ":"n","Ṇ":"N","ṇ":"n","Ṉ":"N","ṉ":"n","Ṋ":"N","ṋ":"n","Ṍ":"O","ṍ":"o","Ṏ":"O","ṏ":"o","Ṑ":"O","ṑ":"o","Ṓ":"O","ṓ":"o","Ṕ":"P","ṕ":"p","Ṗ":"P","ṗ":"p","Ṙ":"R","ṙ":"r","Ṛ":"R","ṛ":"r","Ṝ":"R","ṝ":"r","Ṟ":"R","ṟ":"r","Ṡ":"S","ṡ":"s","Ṣ":"S","ṣ":"s","Ṥ":"S","ṥ":"s","Ṧ":"S","ṧ":"s","Ṩ":"S","ṩ":"s","Ṫ":"T","ṫ":"t","Ṭ":"T","ṭ":"t","Ṯ":"T","ṯ":"t","Ṱ":"T","ṱ":"t","Ṳ":"U","ṳ":"u","Ṵ":"U","ṵ":"u","Ṷ":"U","ṷ":"u","Ṹ":"U","ṹ":"u","Ṻ":"U","ṻ":"u","Ṽ":"V","ṽ":"v","Ṿ":"V","ṿ":"v","Ẁ":"W","ẁ":"w","Ẃ":"W","ẃ":"w","Ẅ":"W","ẅ":"w","Ẇ":"W","ẇ":"w","Ẉ":"W","ẉ":"w","Ẋ":"X","ẋ":"x","Ẍ":"X","ẍ":"x","Ẏ":"Y","ẏ":"y","Ẑ":"Z","ẑ":"z","Ẓ":"Z","ẓ":"z","Ẕ":"Z","ẕ":"z","ẖ":"h","ẗ":"t","ẘ":"w","ẙ":"y","ẛ":"ſ","Ạ":"A","ạ":"a","Ả":"A","ả":"a","Ấ":"A","ấ":"a","Ầ":"A","ầ":"a","Ẩ":"A","ẩ":"a","Ẫ":"A","ẫ":"a","Ậ":"A","ậ":"a","Ắ":"A","ắ":"a","Ằ":"A","ằ":"a","Ẳ":"A","ẳ":"a","Ẵ":"A","ẵ":"a","Ặ":"A","ặ":"a","Ẹ":"E","ẹ":"e","Ẻ":"E","ẻ":"e","Ẽ":"E","ẽ":"e","Ế":"E","ế":"e","Ề":"E","ề":"e","Ể":"E","ể":"e","Ễ":"E","ễ":"e","Ệ":"E","ệ":"e","Ỉ":"I","ỉ":"i","Ị":"I","ị":"i","Ọ":"O","ọ":"o","Ỏ":"O","ỏ":"o","Ố":"O","ố":"o","Ồ":"O","ồ":"o","Ổ":"O","ổ":"o","Ỗ":"O","ỗ":"o","Ộ":"O","ộ":"o","Ớ":"O","ớ":"o","Ờ":"O","ờ":"o","Ở":"O","ở":"o","Ỡ":"O","ỡ":"o","Ợ":"O","ợ":"o","Ụ":"U","ụ":"u","Ủ":"U","ủ":"u","Ứ":"U","ứ":"u","Ừ":"U","ừ":"u","Ử":"U","ử":"u","Ữ":"U","ữ":"u","Ự":"U","ự":"u","Ỳ":"Y","ỳ":"y","Ỵ":"Y","ỵ":"y","Ỷ":"Y","ỷ":"y","Ỹ":"Y","ỹ":"y","ἀ":"α","ἁ":"α","ἂ":"α","ἃ":"α","ἄ":"α","ἅ":"α","ἆ":"α","ἇ":"α","Ἀ":"Α","Ἁ":"Α","Ἂ":"Α","Ἃ":"Α","Ἄ":"Α","Ἅ":"Α","Ἆ":"Α","Ἇ":"Α","ἐ":"ε","ἑ":"ε","ἒ":"ε","ἓ":"ε","ἔ":"ε","ἕ":"ε","Ἐ":"Ε","Ἑ":"Ε","Ἒ":"Ε","Ἓ":"Ε","Ἔ":"Ε","Ἕ":"Ε","ἠ":"η","ἡ":"η","ἢ":"η","ἣ":"η","ἤ":"η","ἥ":"η","ἦ":"η","ἧ":"η","Ἠ":"Η","Ἡ":"Η","Ἢ":"Η","Ἣ":"Η","Ἤ":"Η","Ἥ":"Η","Ἦ":"Η","Ἧ":"Η","ἰ":"ι","ἱ":"ι","ἲ":"ι","ἳ":"ι","ἴ":"ι","ἵ":"ι","ἶ":"ι","ἷ":"ι","Ἰ":"Ι","Ἱ":"Ι","Ἲ":"Ι","Ἳ":"Ι","Ἴ":"Ι","Ἵ":"Ι","Ἶ":"Ι","Ἷ":"Ι","ὀ":"ο","ὁ":"ο","ὂ":"ο","ὃ":"ο","ὄ":"ο","ὅ":"ο","Ὀ":"Ο","Ὁ":"Ο","Ὂ":"Ο","Ὃ":"Ο","Ὄ":"Ο","Ὅ":"Ο","ὐ":"υ","ὑ":"υ","ὒ":"υ","ὓ":"υ","ὔ":"υ","ὕ":"υ","ὖ":"υ","ὗ":"υ","Ὑ":"Υ","Ὓ":"Υ","Ὕ":"Υ","Ὗ":"Υ","ὠ":"ω","ὡ":"ω","ὢ":"ω","ὣ":"ω","ὤ":"ω","ὥ":"ω","ὦ":"ω","ὧ":"ω","Ὠ":"Ω","Ὡ":"Ω","Ὢ":"Ω","Ὣ":"Ω","Ὤ":"Ω","Ὥ":"Ω","Ὦ":"Ω","Ὧ":"Ω","ὰ":"α","ά":"α","ὲ":"ε","έ":"ε","ὴ":"η","ή":"η","ὶ":"ι","ί":"ι","ὸ":"ο","ό":"ο","ὺ":"υ","ύ":"υ","ὼ":"ω","ώ":"ω","ᾀ":"α","ᾁ":"α","ᾂ":"α","ᾃ":"α","ᾄ":"α","ᾅ":"α","ᾆ":"α","ᾇ":"α","ᾈ":"Α","ᾉ":"Α","ᾊ":"Α","ᾋ":"Α","ᾌ":"Α","ᾍ":"Α","ᾎ":"Α","ᾏ":"Α","ᾐ":"η","ᾑ":"η","ᾒ":"η","ᾓ":"η","ᾔ":"η","ᾕ":"η","ᾖ":"η","ᾗ":"η","ᾘ":"Η","ᾙ":"Η","ᾚ":"Η","ᾛ":"Η","ᾜ":"Η","ᾝ":"Η","ᾞ":"Η","ᾟ":"Η","ᾠ":"ω","ᾡ":"ω","ᾢ":"ω","ᾣ":"ω","ᾤ":"ω","ᾥ":"ω","ᾦ":"ω","ᾧ":"ω","ᾨ":"Ω","ᾩ":"Ω","ᾪ":"Ω","ᾫ":"Ω","ᾬ":"Ω","ᾭ":"Ω","ᾮ":"Ω","ᾯ":"Ω","ᾰ":"α","ᾱ":"α","ᾲ":"α","ᾳ":"α","ᾴ":"α","ᾶ":"α","ᾷ":"α","Ᾰ":"Α","Ᾱ":"Α","Ὰ":"Α","Ά":"Α","ᾼ":"Α","ι":"ι","῁":"¨","ῂ":"η","ῃ":"η","ῄ":"η","ῆ":"η","ῇ":"η","Ὲ":"Ε","Έ":"Ε","Ὴ":"Η","Ή":"Η","ῌ":"Η","῍":"᾿","῎":"᾿","῏":"᾿","ῐ":"ι","ῑ":"ι","ῒ":"ι","ΐ":"ι","ῖ":"ι","ῗ":"ι","Ῐ":"Ι","Ῑ":"Ι","Ὶ":"Ι","Ί":"Ι","῝":"῾","῞":"῾","῟":"῾","ῠ":"υ","ῡ":"υ","ῢ":"υ","ΰ":"υ","ῤ":"ρ","ῥ":"ρ","ῦ":"υ","ῧ":"υ","Ῠ":"Υ","Ῡ":"Υ","Ὺ":"Υ","Ύ":"Υ","Ῥ":"Ρ","῭":"¨","΅":"¨","`":"`","ῲ":"ω","ῳ":"ω","ῴ":"ω","ῶ":"ω","ῷ":"ω","Ὸ":"Ο","Ό":"Ο","Ὼ":"Ω","Ώ":"Ω","ῼ":"Ω","´":"´"," ":" "," ":" ","Ω":"Ω","K":"K","Å":"A"}; } From 784555b7dbd9502c5967ea151de1eaf97e697ed7 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Tue, 4 Jun 2019 11:08:41 -0600 Subject: [PATCH 2/2] Add a test for the default wordform to key function. --- .../headless/worker-predict-trie.js | 31 ++++++++++++++++++- .../in_browser/json/tries/accented.json | 1 + .../worker/models/trie-model.ts | 2 +- 3 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 common/predictive-text/unit_tests/in_browser/json/tries/accented.json diff --git a/common/predictive-text/unit_tests/headless/worker-predict-trie.js b/common/predictive-text/unit_tests/headless/worker-predict-trie.js index 889fcae959..a19994f803 100644 --- a/common/predictive-text/unit_tests/headless/worker-predict-trie.js +++ b/common/predictive-text/unit_tests/headless/worker-predict-trie.js @@ -154,7 +154,36 @@ describe('LMLayerWorker trie model for word lists', function() { firstResults.map(s => s.displayAs) ); }); - }) + }); + + describe('The default key function', function () { + it('uses the default key function', function () { + var model = new TrieModel(jsonFixture('tries/accented')); + + // «nai| » [Send] + // [ ??? ] [ naïve ] [ ??? ] + var [suggestion] = model.predict({ + insert: 'i', deleteLeft: 0 + }, { + left: 'na', + startOfBuffer: false, + endOfBuffer: true + }).map(getSample); + assert.strictEqual(suggestion.displayAs, 'naïve'); + + // «Let‘s get some pho| » [Send] + // [ ??? ] [ phở ] [ ??? ] + var [suggestion] = model.predict({ + insert: 'o', deleteLeft: 0 + }, { + left: 'Let‘s get some ph', + startOfBuffer: false, + endOfBuffer: true + }).map(getSample); + assert.strictEqual(suggestion.displayAs, 'phở'); + + }); + }); it('replaces the entire typed word when a suggestion is accepted', function () { // Ensure that all input is lower-cased when we try to look it up. diff --git a/common/predictive-text/unit_tests/in_browser/json/tries/accented.json b/common/predictive-text/unit_tests/in_browser/json/tries/accented.json new file mode 100644 index 0000000000..37213e1048 --- /dev/null +++ b/common/predictive-text/unit_tests/in_browser/json/tries/accented.json @@ -0,0 +1 @@ +{"type":"internal","weight":1,"values":["p","n"],"children":{"p":{"type":"leaf","weight":1,"entries":[{"key":"pho","weight":1,"content":"phở"}]},"n":{"type":"leaf","weight":1,"entries":[{"key":"naive","weight":1,"content":"naïve"}]}}} \ No newline at end of file diff --git a/common/predictive-text/worker/models/trie-model.ts b/common/predictive-text/worker/models/trie-model.ts index cec1070fac..39e3a33a1c 100644 --- a/common/predictive-text/worker/models/trie-model.ts +++ b/common/predictive-text/worker/models/trie-model.ts @@ -429,7 +429,7 @@ */ function defaultWordform2Key(wordform: string): SearchKey { return wordform - .replace(/[\u0100-\u2200]/, function (c) { + .replace(/[\u0100-\u2200]/g, function (c) { if (c in PARTIAL_NFD_LOOKUP) { return PARTIAL_NFD_LOOKUP[c]; }