From c33524e078c0f626d4b28b9941abac2ee2fdedbb Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Mon, 6 Apr 2020 10:24:33 -0600 Subject: [PATCH] fix(common/lmlayer): correct Unicode range in default key fn --- common/predictive-text/worker/models/trie-model.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/common/predictive-text/worker/models/trie-model.ts b/common/predictive-text/worker/models/trie-model.ts index 99b8da4590..05ce5e6273 100644 --- a/common/predictive-text/worker/models/trie-model.ts +++ b/common/predictive-text/worker/models/trie-model.ts @@ -429,7 +429,10 @@ */ function defaultWordform2Key(wordform: string): SearchKey { return wordform - .replace(/[\u0100-\u2200]/g, function (c) { + // remove all combining diacritics (if input is in NFD) + .replace(/[\u0300-\u036f]/g, '') + // remove all composed diacritics (if input is in NFC) LATIN-ONLY! + .replace(/[\u00C0-\u212A]/g, function (c) { if (c in PARTIAL_NFD_LOOKUP) { return PARTIAL_NFD_LOOKUP[c]; }