erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-09-19 06:07:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

feat(core): marker normalization 🙀

Browse source 
- go back to NFD for the context, for now
- anticipating when the privatecontext is NFD but the public context is NFC
- also update the test cases

For: #9468
This commit is contained in:
Steven R. Loomis 2023-11-14 16:36:15 -06:00
parent bf6690948b
commit 2f843d98f3
3 changed files with 15 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
core/src/ldml/ldml_processor.cpp
View file

@ -271,10 +271,10 @@ ldml_processor::process_key_string(km_core_state *state, const std::u16string &k
// so that we don't have to reconvert it inside the transform code.
std::u32string key_str32 = kmx::u16string_to_u32string(key_str);
assert(ldml::normalize_nfd_markers(key_str32)); // TODO-LDML: else fail?
// extract context string, in NFC
std::u32string old_ctxtstr_nfc;
(void)context_to_string(state, old_ctxtstr_nfc, false);
assert(ldml::normalize_nfc_markers(old_ctxtstr_nfc)); // TODO-LDML: else fail?
// extract context string, in NFD
std::u32string old_ctxtstr_nfd;
(void)context_to_string(state, old_ctxtstr_nfd, false);
assert(ldml::normalize_nfd_markers(old_ctxtstr_nfd)); // TODO-LDML: else fail?
// context string in NFD
std::u32string ctxtstr;
@ -304,18 +304,18 @@ ldml_processor::process_key_string(km_core_state *state, const std::u16string &k
// Ok. We've done all the happy manipulations.
/** NFC and no markers */
/** NFD and no markers */
std::u32string ctxtstr_cleanedup = ldml::remove_markers(ctxtstr);
assert(ldml::normalize_nfc_markers(ctxtstr_cleanedup));
assert(ldml::normalize_nfd_markers(ctxtstr_cleanedup));
// find common prefix.
// For example, if the context previously had "aaBBBBB" and it is changing to "aaCCC" then we will have:
// - old_ctxtstr_changed = "BBBBB"
// - new_ctxtstr_changed = "CCC"
// So the BBBBB needs to be removed and then CCC added.
auto ctxt_prefix = mismatch(old_ctxtstr_nfc.begin(), old_ctxtstr_nfc.end(), ctxtstr_cleanedup.begin(), ctxtstr_cleanedup.end());
auto ctxt_prefix = mismatch(old_ctxtstr_nfd.begin(), old_ctxtstr_nfd.end(), ctxtstr_cleanedup.begin(), ctxtstr_cleanedup.end());
/** The part of the old string to be removed */
std::u32string old_ctxtstr_changed(ctxt_prefix.first,old_ctxtstr_nfc.end());
std::u32string old_ctxtstr_changed(ctxt_prefix.first,old_ctxtstr_nfd.end());
/** The new context to be added */
std::u32string new_ctxtstr_changed(ctxt_prefix.second,ctxtstr_cleanedup.end());

4
core/tests/unit/ldml/ldml.cpp
View file

@ -28,6 +28,9 @@
#include "ldml_test_source.hpp"
#include "debuglog.h"
#include "ldml/ldml_transforms.hpp"
namespace {
bool g_beep_found = false;
@ -279,6 +282,7 @@ run_test(const km::core::path &source, const km::core::path &compiled, km::tests
verify_context(text_store, test_state, test_context);
} else if (action.type == km::tests::LDML_ACTION_CHECK_EXPECTED) {
assert(km::core::ldml::normalize_nfd(action.string)); // TODO-LDML: should be NFC
std::cout << "- check expected" << std::endl;
std::cout << "expected : " << string_to_hex(action.string) << " [" << action.string << "]" << std::endl;
std::cout << "text store: " << string_to_hex(text_store) << " [" << text_store << "]" << std::endl;

6
core/tests/unit/ldml/ldml_test_source.cpp
View file

@ -472,7 +472,7 @@ LdmlJsonTestSource::next_action(ldml_action &fillin) {
if (type == "check") {
fillin.type = LDML_ACTION_CHECK_EXPECTED;
fillin.string = LdmlTestSource::parse_u8_source_string(result.get<std::string>());
assert(km::core::ldml::normalize_nfc(fillin.string));
assert(km::core::ldml::normalize_nfd(fillin.string)); // TODO-LDML: should be NFC
return;
} else if (type == "keystroke") {
fillin.type = LDML_ACTION_KEY_EVENT;
@ -483,7 +483,7 @@ LdmlJsonTestSource::next_action(ldml_action &fillin) {
} else if (type == "emit") {
fillin.type = LDML_ACTION_EMIT_STRING;
fillin.string = LdmlTestSource::parse_u8_source_string(to.get<std::string>());
assert(km::core::ldml::normalize_nfc(fillin.string));
assert(km::core::ldml::normalize_nfd(fillin.string)); // TODO-LDML: should be NFC
return;
} else if (type == "backspace") {
// backspace is handled as a key event
@ -507,7 +507,7 @@ int LdmlJsonTestSource::load(const nlohmann::json &data) {
this->data = data; // TODO-LDML
auto startContext = data["/startContext/to"_json_pointer];
context = LdmlTestSource::parse_u8_source_string(startContext);
assert(km::core::ldml::normalize_nfc(context));
assert(km::core::ldml::normalize_nfd(context)); // TODO-LDML: should be NFC
return 0;
}