From fea6026f719408b960f25092254aa06cf28f7038 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Mon, 6 Apr 2020 09:25:50 -0600 Subject: [PATCH 1/5] test(common/lmlayer): add regression test for searchTermToKey --- .../cases/worker-trie-integration.js | 37 ++++++++++++++++++- .../in_browser/resources/models/naive-trie.js | 5 +++ 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 common/predictive-text/unit_tests/in_browser/resources/models/naive-trie.js diff --git a/common/predictive-text/unit_tests/in_browser/cases/worker-trie-integration.js b/common/predictive-text/unit_tests/in_browser/cases/worker-trie-integration.js index d5441823f9..5b486aef49 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/worker-trie-integration.js +++ b/common/predictive-text/unit_tests/in_browser/cases/worker-trie-integration.js @@ -42,6 +42,41 @@ describe('LMLayer using the trie model', function () { }); }); + describe('Regression tests', function () { + // Building the model with the default searchTermToKey implementation + // should also use the default implementation in the frontend. + // + // I've compiled a model with the defaults for the word 'naïve', and its + // search key is 'naive' accordingly (default searchTermToKey transforms + // to NFD and removes common diacritics). + // + // The lookup should also do the same! + // + // https://community.software.sil.org/t/search-term-to-key-in-lexical-model-not-working-both-ways-by-default/3133 + it('should use the default searchTermToKey()', function () { + var lmLayer = new LMLayer(helpers.defaultCapabilities); + + return lmLayer.loadModel( + // We need to provide an absolute path since the worker is based within a blob. + document.location.protocol + '//' + document.location.host + "/resources/models/naive-trie.js" + ).then(function (_actualConfiguration) { + return Promise.resolve(); + }).then(function () { + // Pretend we are typing 'na|i' + return lmLayer.predict(type('i'), atEndOfBuffer('na')); + }).then(function (rawSuggestions) { + // Discard the keep suggestion + let suggestions = rawSuggestions.filter(s => s.tag != 'keep') + assert.isAtLeast(suggestions.length, 1) + + // We SHOULD get 'naïve' suggested, despite never typing a diaeresis. + let topSuggestion = suggestions[0]; + assert.equal(topSuggestion.displayAs, "naïve") + assert.include(topSuggestion.transform.insert, "ïve") + }); + }); + }); + /** * Type some text (as a transform). * @param {string} text to type @@ -53,7 +88,7 @@ describe('LMLayer using the trie model', function () { /** * Create a context at the end of the buffer with the given text. - * @param {string} buffer + * @param {string} buffer * @return {Context} */ function atEndOfBuffer(buffer) { diff --git a/common/predictive-text/unit_tests/in_browser/resources/models/naive-trie.js b/common/predictive-text/unit_tests/in_browser/resources/models/naive-trie.js new file mode 100644 index 0000000000..58fa5af3c8 --- /dev/null +++ b/common/predictive-text/unit_tests/in_browser/resources/models/naive-trie.js @@ -0,0 +1,5 @@ +(function() { +'use strict'; +LMLayerWorker.loadModel(new models.TrieModel({"totalWeight":1,"root":{"type":"leaf","weight":1,"entries":[{"key":"naive","weight":1,"content":"naïve"}]}}, { +})); +})(); \ No newline at end of file From 77596b2c5c8c0f79d9be5cedaabe03d68b1ef732 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Mon, 6 Apr 2020 09:30:28 -0600 Subject: [PATCH 2/5] test(common/lmlayer): reproduce issue reported in community forum https://community.software.sil.org/t/search-term-to-key-in-lexical-model-not-working-both-ways-by-default/3133 --- .../unit_tests/in_browser/cases/worker-trie-integration.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/common/predictive-text/unit_tests/in_browser/cases/worker-trie-integration.js b/common/predictive-text/unit_tests/in_browser/cases/worker-trie-integration.js index 5b486aef49..361a3f4aa2 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/worker-trie-integration.js +++ b/common/predictive-text/unit_tests/in_browser/cases/worker-trie-integration.js @@ -50,7 +50,7 @@ describe('LMLayer using the trie model', function () { // search key is 'naive' accordingly (default searchTermToKey transforms // to NFD and removes common diacritics). // - // The lookup should also do the same! + // The lookup should also do the same to the input! // // https://community.software.sil.org/t/search-term-to-key-in-lexical-model-not-working-both-ways-by-default/3133 it('should use the default searchTermToKey()', function () { @@ -62,8 +62,9 @@ describe('LMLayer using the trie model', function () { ).then(function (_actualConfiguration) { return Promise.resolve(); }).then(function () { - // Pretend we are typing 'na|i' - return lmLayer.predict(type('i'), atEndOfBuffer('na')); + // Pretend we are typing 'na|ï' + // Notice that we are typing the diacritic! + return lmLayer.predict(type('ï'), atEndOfBuffer('na')); }).then(function (rawSuggestions) { // Discard the keep suggestion let suggestions = rawSuggestions.filter(s => s.tag != 'keep') From e659d4df0d7c544fa99095e9ed4c5ad594b14794 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Mon, 6 Apr 2020 10:24:33 -0600 Subject: [PATCH 3/5] fix(common/lmlayer): correct Unicode range in default key fn --- common/predictive-text/worker/models/trie-model.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/common/predictive-text/worker/models/trie-model.ts b/common/predictive-text/worker/models/trie-model.ts index 99b8da4590..05ce5e6273 100644 --- a/common/predictive-text/worker/models/trie-model.ts +++ b/common/predictive-text/worker/models/trie-model.ts @@ -429,7 +429,10 @@ */ function defaultWordform2Key(wordform: string): SearchKey { return wordform - .replace(/[\u0100-\u2200]/g, function (c) { + // remove all combining diacritics (if input is in NFD) + .replace(/[\u0300-\u036f]/g, '') + // remove all composed diacritics (if input is in NFC) LATIN-ONLY! + .replace(/[\u00C0-\u212A]/g, function (c) { if (c in PARTIAL_NFD_LOOKUP) { return PARTIAL_NFD_LOOKUP[c]; } From aa1552fbe8f130214721c81d9d428a8972167fa2 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Mon, 6 Apr 2020 11:29:39 -0600 Subject: [PATCH 4/5] fix(common/lmlayer): use IE11-friendly syntax --- .../unit_tests/in_browser/cases/worker-trie-integration.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/predictive-text/unit_tests/in_browser/cases/worker-trie-integration.js b/common/predictive-text/unit_tests/in_browser/cases/worker-trie-integration.js index 361a3f4aa2..322d5e2c89 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/worker-trie-integration.js +++ b/common/predictive-text/unit_tests/in_browser/cases/worker-trie-integration.js @@ -67,11 +67,11 @@ describe('LMLayer using the trie model', function () { return lmLayer.predict(type('ï'), atEndOfBuffer('na')); }).then(function (rawSuggestions) { // Discard the keep suggestion - let suggestions = rawSuggestions.filter(s => s.tag != 'keep') + var suggestions = rawSuggestions.filter(s => s.tag != 'keep') assert.isAtLeast(suggestions.length, 1) - // We SHOULD get 'naïve' suggested, despite never typing a diaeresis. - let topSuggestion = suggestions[0]; + // We SHOULD get 'naïve' suggested + var topSuggestion = suggestions[0]; assert.equal(topSuggestion.displayAs, "naïve") assert.include(topSuggestion.transform.insert, "ïve") }); From 42bc0197fbcb260a76e7fbb4be95d85ecff062df Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Mon, 6 Apr 2020 11:52:14 -0600 Subject: [PATCH 5/5] fix(common/lmlayer): arrow fn -> fn expr (IE11 compat) --- .../unit_tests/in_browser/cases/worker-trie-integration.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/common/predictive-text/unit_tests/in_browser/cases/worker-trie-integration.js b/common/predictive-text/unit_tests/in_browser/cases/worker-trie-integration.js index 322d5e2c89..68426acb7a 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/worker-trie-integration.js +++ b/common/predictive-text/unit_tests/in_browser/cases/worker-trie-integration.js @@ -67,7 +67,9 @@ describe('LMLayer using the trie model', function () { return lmLayer.predict(type('ï'), atEndOfBuffer('na')); }).then(function (rawSuggestions) { // Discard the keep suggestion - var suggestions = rawSuggestions.filter(s => s.tag != 'keep') + var suggestions = rawSuggestions.filter(function skimKeepSuggestions(s) { + return s.tag !== 'keep' + }) assert.isAtLeast(suggestions.length, 1) // We SHOULD get 'naïve' suggested