diff --git a/common/predictive-text/unit_tests/in_browser/cases/worker-trie-integration.js b/common/predictive-text/unit_tests/in_browser/cases/worker-trie-integration.js index d5441823f9..68426acb7a 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/worker-trie-integration.js +++ b/common/predictive-text/unit_tests/in_browser/cases/worker-trie-integration.js @@ -42,6 +42,44 @@ describe('LMLayer using the trie model', function () { }); }); + describe('Regression tests', function () { + // Building the model with the default searchTermToKey implementation + // should also use the default implementation in the frontend. + // + // I've compiled a model with the defaults for the word 'naïve', and its + // search key is 'naive' accordingly (default searchTermToKey transforms + // to NFD and removes common diacritics). + // + // The lookup should also do the same to the input! + // + // https://community.software.sil.org/t/search-term-to-key-in-lexical-model-not-working-both-ways-by-default/3133 + it('should use the default searchTermToKey()', function () { + var lmLayer = new LMLayer(helpers.defaultCapabilities); + + return lmLayer.loadModel( + // We need to provide an absolute path since the worker is based within a blob. + document.location.protocol + '//' + document.location.host + "/resources/models/naive-trie.js" + ).then(function (_actualConfiguration) { + return Promise.resolve(); + }).then(function () { + // Pretend we are typing 'na|ï' + // Notice that we are typing the diacritic! + return lmLayer.predict(type('ï'), atEndOfBuffer('na')); + }).then(function (rawSuggestions) { + // Discard the keep suggestion + var suggestions = rawSuggestions.filter(function skimKeepSuggestions(s) { + return s.tag !== 'keep' + }) + assert.isAtLeast(suggestions.length, 1) + + // We SHOULD get 'naïve' suggested + var topSuggestion = suggestions[0]; + assert.equal(topSuggestion.displayAs, "naïve") + assert.include(topSuggestion.transform.insert, "ïve") + }); + }); + }); + /** * Type some text (as a transform). * @param {string} text to type @@ -53,7 +91,7 @@ describe('LMLayer using the trie model', function () { /** * Create a context at the end of the buffer with the given text. - * @param {string} buffer + * @param {string} buffer * @return {Context} */ function atEndOfBuffer(buffer) { diff --git a/common/predictive-text/unit_tests/in_browser/resources/models/naive-trie.js b/common/predictive-text/unit_tests/in_browser/resources/models/naive-trie.js new file mode 100644 index 0000000000..58fa5af3c8 --- /dev/null +++ b/common/predictive-text/unit_tests/in_browser/resources/models/naive-trie.js @@ -0,0 +1,5 @@ +(function() { +'use strict'; +LMLayerWorker.loadModel(new models.TrieModel({"totalWeight":1,"root":{"type":"leaf","weight":1,"entries":[{"key":"naive","weight":1,"content":"naïve"}]}}, { +})); +})(); \ No newline at end of file diff --git a/common/predictive-text/worker/models/trie-model.ts b/common/predictive-text/worker/models/trie-model.ts index 99b8da4590..05ce5e6273 100644 --- a/common/predictive-text/worker/models/trie-model.ts +++ b/common/predictive-text/worker/models/trie-model.ts @@ -429,7 +429,10 @@ */ function defaultWordform2Key(wordform: string): SearchKey { return wordform - .replace(/[\u0100-\u2200]/g, function (c) { + // remove all combining diacritics (if input is in NFD) + .replace(/[\u0300-\u036f]/g, '') + // remove all composed diacritics (if input is in NFC) LATIN-ONLY! + .replace(/[\u00C0-\u212A]/g, function (c) { if (c in PARTIAL_NFD_LOOKUP) { return PARTIAL_NFD_LOOKUP[c]; }