Merge pull request #2954 from keymanapp/lmlayer-default-searchtermtokey-regression

fix(common/lmlayer): use searchTermToKey() on input

common/predictive-text/unit_tests/in_browser/cases/worker-trie-integration.js

@@ -42,6 +42,44 @@ describe('LMLayer using the trie model', function () {
     });
   });
 
+  describe('Regression tests', function () {
+    // Building the model with the default searchTermToKey implementation
+    // should also use the default implementation in the frontend.
+    //
+    // I've compiled a model with the defaults for the word 'naïve', and its
+    // search key is 'naive' accordingly (default searchTermToKey transforms
+    // to NFD and removes common diacritics).
+    //
+    // The lookup should also do the same to the input!
+    //
+    // https://community.software.sil.org/t/search-term-to-key-in-lexical-model-not-working-both-ways-by-default/3133
+    it('should use the default searchTermToKey()', function () {
+      var lmLayer = new LMLayer(helpers.defaultCapabilities);
+
+      return lmLayer.loadModel(
+        // We need to provide an absolute path since the worker is based within a blob.
+        document.location.protocol + '//' + document.location.host + "/resources/models/naive-trie.js"
+      ).then(function (_actualConfiguration) {
+        return Promise.resolve();
+      }).then(function () {
+        // Pretend we are typing 'na|ï'
+        // Notice that we are typing the diacritic!
+        return lmLayer.predict(type('ï'), atEndOfBuffer('na'));
+      }).then(function (rawSuggestions) {
+        // Discard the keep suggestion
+        var suggestions = rawSuggestions.filter(function skimKeepSuggestions(s) {
+          return s.tag !== 'keep'
+        })
+        assert.isAtLeast(suggestions.length, 1)
+
+        // We SHOULD get 'naïve' suggested
+        var topSuggestion = suggestions[0];
+        assert.equal(topSuggestion.displayAs, "naïve")
+        assert.include(topSuggestion.transform.insert, "ïve")
+      });
+    });
+  });
+
   /**
    * Type some text (as a transform).
    * @param  {string} text to type
@@ -53,7 +91,7 @@ describe('LMLayer using the trie model', function () {
 
   /**
    * Create a context at the end of the buffer with the given text.
-   * @param {string} buffer 
+   * @param {string} buffer
    * @return {Context}
    */
   function atEndOfBuffer(buffer) {

common/predictive-text/unit_tests/in_browser/resources/models/naive-trie.js

@@ -0,0 +1,5 @@
+(function() {
+'use strict';
+LMLayerWorker.loadModel(new models.TrieModel({"totalWeight":1,"root":{"type":"leaf","weight":1,"entries":[{"key":"naive","weight":1,"content":"naïve"}]}}, {
+}));
+})();

common/predictive-text/worker/models/trie-model.ts

@@ -429,7 +429,10 @@
    */
   function defaultWordform2Key(wordform: string): SearchKey {
     return wordform
-      .replace(/[\u0100-\u2200]/g, function (c) {
+      // remove all combining diacritics (if input is in NFD)
+      .replace(/[\u0300-\u036f]/g, '')
+      // remove all composed diacritics (if input is in NFC) LATIN-ONLY!
+      .replace(/[\u00C0-\u212A]/g, function (c) {
         if (c in PARTIAL_NFD_LOOKUP) {
           return PARTIAL_NFD_LOOKUP[c];
         }