From 85a5afafc90a3ce94282aed25119df1ed4f5cf89 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Tue, 7 Nov 2023 17:39:57 -0600 Subject: [PATCH 1/5] =?UTF-8?q?feat(core):=20ldml=20tertiary=20reorder=20?= =?UTF-8?q?=F0=9F=99=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - failing test cases, hooray! In XML and C++ For: #9707 --- .../ldml/keyboards/k_201_reorder_esk-test.xml | 24 +++ .../unit/ldml/keyboards/k_201_reorder_esk.xml | 58 +++++++ core/tests/unit/ldml/keyboards/meson.build | 1 + core/tests/unit/ldml/test_transforms.cpp | 143 +++++++++++++++++- 4 files changed, 223 insertions(+), 3 deletions(-) create mode 100644 core/tests/unit/ldml/keyboards/k_201_reorder_esk-test.xml create mode 100644 core/tests/unit/ldml/keyboards/k_201_reorder_esk.xml diff --git a/core/tests/unit/ldml/keyboards/k_201_reorder_esk-test.xml b/core/tests/unit/ldml/keyboards/k_201_reorder_esk-test.xml new file mode 100644 index 0000000000..e97ebd35b6 --- /dev/null +++ b/core/tests/unit/ldml/keyboards/k_201_reorder_esk-test.xml @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/core/tests/unit/ldml/keyboards/k_201_reorder_esk.xml b/core/tests/unit/ldml/keyboards/k_201_reorder_esk.xml new file mode 100644 index 0000000000..0b49446ab1 --- /dev/null +++ b/core/tests/unit/ldml/keyboards/k_201_reorder_esk.xml @@ -0,0 +1,58 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/core/tests/unit/ldml/keyboards/meson.build b/core/tests/unit/ldml/keyboards/meson.build index f4b7f23a29..c703d542ab 100644 --- a/core/tests/unit/ldml/keyboards/meson.build +++ b/core/tests/unit/ldml/keyboards/meson.build @@ -39,6 +39,7 @@ tests_with_testdata = [ 'k_008_transform_norm', 'k_020_fr', # TODO-LDML: move to cldr above (fix vkey) 'k_200_reorder_nod_Lana', + 'k_201_reorder_esk', 'k_210_marker', ] diff --git a/core/tests/unit/ldml/test_transforms.cpp b/core/tests/unit/ldml/test_transforms.cpp index 6960be0ff3..b9316961a1 100644 --- a/core/tests/unit/ldml/test_transforms.cpp +++ b/core/tests/unit/ldml/test_transforms.cpp @@ -15,15 +15,22 @@ #define zassert_string_equal(actual, expected) \ { \ if (actual != expected) { \ - std::wcerr << __FILE__ << ":" << __LINE__ << ": " << console_color::fg(console_color::BRIGHT_RED) << "got: " << actual \ - << " expected " << expected << console_color::reset() << std::endl; \ + std::wcerr << __FILE__ << ":" << __LINE__ << ": " << console_color::fg(console_color::BRIGHT_RED) << "got: " << km::core::kmx::Debug_UnicodeString(actual, 0) \ + << " expected " << km::core::kmx::Debug_UnicodeString(expected, 1) << console_color::reset() << std::endl; \ return EXIT_FAILURE; \ } \ } #endif #ifndef zassert_equal -#define zassert_equal(actual, expected) zassert_string_equal(actual, expected) +#define zassert_equal(actual, expected) \ + { \ + if (actual != expected) { \ + std::wcerr << __FILE__ << ":" << __LINE__ << ": " << console_color::fg(console_color::BRIGHT_RED) << "got: " << actual \ + << " expected " << expected << console_color::reset() << std::endl; \ + return EXIT_FAILURE; \ + } \ + } #endif // needed for streaming operators @@ -407,6 +414,132 @@ test_reorder_standalone() { return EXIT_SUCCESS; } + +// this test case is also in XML form under 'k_201_*' +int +test_reorder_esk() { + std::cout << "== " << __FUNCTION__ << std::endl; + + std::cout << __FILE__ << ":" << __LINE__ << " - k_201_reorder_esk (tertiary reordering) " << std::endl; + { + // now setup the rules + // rules are a little bit simplified, having only the vowel 'a' + + std::cout << "now prepare the reorder elements" << std::endl; + transforms tr; + { + reorder_group rg; + + // + { + element_list e; + e.emplace_back(U'a', (0 << LDML_ELEM_FLAGS_ORDER_BITSHIFT) | LDML_ELEM_FLAGS_TERTIARY_BASE); + rg.list.emplace_back(e); + } + + // + { + element_list e; + e.emplace_back(0x0332, (1 << LDML_ELEM_FLAGS_TERTIARY_BITSHIFT)); + rg.list.emplace_back(e); + } + // + { + element_list e; + e.emplace_back(0x305, (2 << LDML_ELEM_FLAGS_TERTIARY_BITSHIFT)); + e.emplace_back(0x302, (2 << LDML_ELEM_FLAGS_TERTIARY_BITSHIFT)); + rg.list.emplace_back(e); + } + // + { + element_list e; + e.emplace_back(U'x', (1 << LDML_ELEM_FLAGS_ORDER_BITSHIFT)); + rg.list.emplace_back(e); + } + // + { + element_list e; + e.emplace_back(U'y', (2 << LDML_ELEM_FLAGS_ORDER_BITSHIFT)); + rg.list.emplace_back(e); + } + // + { + element_list e; + e.emplace_back(U'z', (3 << LDML_ELEM_FLAGS_ORDER_BITSHIFT)); + rg.list.emplace_back(e); + } + + tr.addGroup(rg); + } + + // now actually test it + std::cout << __FILE__ << ":" << __LINE__ << " - cases " << std::endl; + const std::u32string orig_expect[] = { + // 1short + U"ax\u0305", // orig + U"a\u0305x", // expect + + // 2longer + U"az\u0305x\u0332", // orig + U"a\u0332\u0305xz", // expect + }; + // TODO-LDML: move this into test code perhaps + for (size_t r = 0; r < sizeof(orig_expect) / sizeof(orig_expect[0]); r+= 2) { + const auto &orig = orig_expect[r + 0]; + const auto &expect = orig_expect[r + 1]; + std::cout << __FILE__ << ":" << __LINE__ << " - trying str #" << r+1 << "=" << orig << std::endl; + // try apply with string + { + std::cout << "- try apply(text, output)" << std::endl; + std::u32string text = orig; + std::u32string output; + size_t len = tr.apply(text, output); + if (len == 0) { + std::cout << " (did not apply)" << std::endl; + } else { + std::cout << " applied, matchLen= " << len << std::endl; + text.resize(text.size()-len); // shrink + text.append(output); + std::cout << " = " << text << std::endl; + } + zassert_string_equal(text, expect); + } + // try all-at-once + { + std::cout << "- try apply(text)" << std::endl; + std::u32string text = orig; + if (!tr.apply(text)) { + std::cout << " (did not apply)" << std::endl; + } else if (text == orig) { + std::cout << " (suboptimal: apply returned true but made no change)" << std::endl; + } else { + std::cout << " changed to " << text; + } + zassert_string_equal(text, expect); + std::cout << " matched (converting all at once)!" << std::endl; + } + // simulate typing this one char at a time; + { + std::cout << "- try key-at-a-time" << std::endl; + std::u32string text; + for (auto ch = orig.begin(); ch < orig.end(); ch++) { + // append the string + text.append(1, *ch); + std::cout << "-: " << text << std::endl; + if (!tr.apply(text)) { + std::cout << " (did not apply)" << std::endl; + } + } + // now the moment of truth + zassert_string_equal(text, expect); + std::cout << " matched! (converting char at a time)" << std::endl; + std::cout << std::endl; + } + } + } + return EXIT_SUCCESS; +} + int test_map() { std::cout << "== " << __FUNCTION__ << std::endl; @@ -639,6 +772,10 @@ main(int argc, const char *argv[]) { rc = EXIT_FAILURE; } + if (test_reorder_esk() != EXIT_SUCCESS) { + rc = EXIT_FAILURE; + } + if (test_map() != EXIT_SUCCESS) { rc = EXIT_FAILURE; } From f7566c7356cfca49340a2b816b8ab09004d5ffd7 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Tue, 7 Nov 2023 19:19:02 -0600 Subject: [PATCH 2/5] =?UTF-8?q?feat(core):=20ldml=20tertiary=20reorder=20?= =?UTF-8?q?=F0=9F=99=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - implemented tertiary reordering - reinstated bengali reordering! - fixed reordering to be longest-first (by sorting the list before applying) - updated debug logging Fixes: #9707 --- core/src/ldml/ldml_transforms.cpp | 68 +++++++++++++++++-- core/src/ldml/ldml_transforms.hpp | 8 ++- core/tests/unit/ldml/test_transforms.cpp | 3 +- .../ldml-keyboards/techpreview/3.0/bn.xml | 5 +- 4 files changed, 72 insertions(+), 12 deletions(-) diff --git a/core/src/ldml/ldml_transforms.cpp b/core/src/ldml/ldml_transforms.cpp index 5c4ff428cd..c7f061a310 100644 --- a/core/src/ldml/ldml_transforms.cpp +++ b/core/src/ldml/ldml_transforms.cpp @@ -138,7 +138,7 @@ reorder_sort_key::from(const std::u32string &str) { // seconary weight: c (the string index) // tertiary weight: 0 // quaternary weight: c (the index again) - keylist.emplace_back(reorder_sort_key{*s, 0, c, 0, c}); + keylist.emplace_back(reorder_sort_key{*s, 0, c, 0, c, false}); } return keylist; } @@ -146,7 +146,9 @@ reorder_sort_key::from(const std::u32string &str) { void reorder_sort_key::dump() const { // for debugging… - DebugLog("- U+%04X\t(%d, %d, %d, %d)", ch, (int)primary, (int)secondary, (int)tertiary, (int)quaternary); + DebugLog( + "- U+%04X\t(%d, %d, %d, %d) %c", ch, (int)primary, (int)secondary, (int)tertiary, (int)quaternary, + is_tertiary_base ? 'T' : ' '); } size_t @@ -213,9 +215,14 @@ std::deque & element_list::update_sort_key(size_t offset, std::deque &key) const { /** string index */ size_t c = 0; + bool have_last_base = false; + signed char last_base_primary = -1; + signed char last_base_secondary = -1; for (auto e = begin(); e < end(); e++, c++) { + /** position in the key */ + auto n = offset + c; /** update this key */ - auto &k = key.at(offset + c); + auto &k = key.at(n); // we double check that the character matches. otherwise something // has really gone awry, because we shouldn't be here if this element list doesn't apply. if (!e->matches(k.ch)) { @@ -225,9 +232,30 @@ element_list::update_sort_key(size_t offset, std::deque &key) assert(e->matches(k.ch)); // double check that this element matches } // we only update primary and tertiary weights - k.primary = e->get_order(); - // TODO-LDML: need more detailed tertiary work - k.tertiary = e->get_tertiary(); + k.primary = e->get_order(); + k.tertiary = e->get_tertiary(); + k.is_tertiary_base = e->is_tertiary_base(); + + if (k.tertiary != 0) { + // search backwards for a base + auto n2 = n; + // TODO-LDML: odd loop here because n2 is signed. + do { + n2--; + auto &k2 = key.at(n2); + if (k2.is_tertiary_base) { + last_base_primary = k2.primary; + last_base_secondary = k2.secondary; + have_last_base = true; + } + } while (!have_last_base && n2 > 0); + // we may not have found the base. but the common case is that the base is found. + if (have_last_base) { + // copy the primary and secondary from the last_base + k.primary = last_base_primary; + k.secondary = last_base_secondary; + } + } #if KMXPLUS_DEBUG_TRANSFORM DebugTran("Updating at +%d", c); k.dump(); @@ -271,6 +299,29 @@ reorder_entry::match_end(std::u32string &str, size_t offset, size_t len) const { return match_len; } +int +reorder_entry::compare(const reorder_entry &other) const { + if (this == &other) { + return 0; + } else if (elements.size() < other.elements.size()) { + return -1; + } else if (elements.size() > other.elements.size()) { + return 1; + } else { + return 0; // punt + } +} + +bool +reorder_entry::operator<(const reorder_entry &other) const { + return (compare(other) < 0); +} + +bool +reorder_entry::operator>(const reorder_entry &other) const { + return (compare(other) > 0); +} + bool reorder_group::apply(std::u32string &str) const { /** did we apply anything */ @@ -349,7 +400,9 @@ reorder_group::apply(std::u32string &str) const { /** pointer to the beginning of the current run. */ std::deque::iterator run_start = sort_keys.begin(); for(auto e = run_start; e != sort_keys.end(); e++) { - if ((e->primary == 0) && (e != run_start)) { // it's a base + // find the actual beginning base: primary weight = 0 and tertiary = 0. + // (tertiary chars will have primary=0 BUT will have tertiary nonzero.) + if ((e->primary == 0) && (e->tertiary == 0) && (e != run_start)) { auto run_end = e - 1; DebugTran("Sorting subrange quaternary=[%d..]", run_start->quaternary); std::sort(run_start, run_end); // reversed because it's a reverse iterator…? @@ -838,6 +891,7 @@ transforms::load( return nullptr; } } + std::sort(newGroup.list.begin(), newGroup.list.end()); // sort list by size, so that longer matches match last transforms->addGroup(newGroup); } else { // internal error - some other type - should have been caught by validation diff --git a/core/src/ldml/ldml_transforms.hpp b/core/src/ldml/ldml_transforms.hpp index 4a2948613d..582a7ba3a9 100644 --- a/core/src/ldml/ldml_transforms.hpp +++ b/core/src/ldml/ldml_transforms.hpp @@ -156,11 +156,12 @@ public: /** a single char, categorized according to reorder rules*/ struct reorder_sort_key { - km_core_usv ch; // the single char value + km_core_usv ch; // the single char value signed char primary; // primary order value size_t secondary; // index position signed char tertiary; // tertiary value, defaults to 0 size_t quaternary; // index again + bool is_tertiary_base; // remember that this key was a tertiary base /** @returns -1, 0, 1 depending on ordering */ int compare(const reorder_sort_key &other) const; @@ -213,6 +214,11 @@ public: */ size_t match_end(std::u32string &str, size_t offset, size_t len) const; + /** @returns -1, 0, 1 depending on ordering */ + int compare(const reorder_entry &other) const; + bool operator<(const reorder_entry &other) const; + bool operator>(const reorder_entry &other) const; + public: element_list elements; element_list before; diff --git a/core/tests/unit/ldml/test_transforms.cpp b/core/tests/unit/ldml/test_transforms.cpp index b9316961a1..dd27bae143 100644 --- a/core/tests/unit/ldml/test_transforms.cpp +++ b/core/tests/unit/ldml/test_transforms.cpp @@ -447,7 +447,8 @@ test_reorder_esk() { { element_list e; e.emplace_back(0x305, (2 << LDML_ELEM_FLAGS_TERTIARY_BITSHIFT)); - e.emplace_back(0x302, (2 << LDML_ELEM_FLAGS_TERTIARY_BITSHIFT)); + // (should be a unicodeset, but for simplicity we're dropping the U+302) + // e.emplace_back(0x302, (2 << LDML_ELEM_FLAGS_TERTIARY_BITSHIFT)); rg.list.emplace_back(e); } // diff --git a/resources/standards-data/ldml-keyboards/techpreview/3.0/bn.xml b/resources/standards-data/ldml-keyboards/techpreview/3.0/bn.xml index fc04ec3cbc..167416e944 100644 --- a/resources/standards-data/ldml-keyboards/techpreview/3.0/bn.xml +++ b/resources/standards-data/ldml-keyboards/techpreview/3.0/bn.xml @@ -155,8 +155,7 @@ - - + From 30d951ef67b66a335ac5807591002fb56238ae21 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 16 Nov 2023 13:43:12 -0600 Subject: [PATCH 3/5] =?UTF-8?q?feat(core):=20ldml=20dx:=20launch=20ldml=20?= =?UTF-8?q?unit=20tests=20before=20keyboard=20tests=20=F0=9F=99=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For: #9707 --- core/tests/unit/ldml/meson.build | 41 +++++++++++++++++--------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/core/tests/unit/ldml/meson.build b/core/tests/unit/ldml/meson.build index 2416906473..f85014bebe 100644 --- a/core/tests/unit/ldml/meson.build +++ b/core/tests/unit/ldml/meson.build @@ -61,6 +61,28 @@ ldml = executable('ldml', objects: lib.extract_all_objects(recursive: false), ) + +# Build and run additional test_kmx_plus test + +e = executable('test_kmx_plus', 'test_kmx_plus.cpp', + 'ldml_test_utils.cpp', + cpp_args: defns + warns, + include_directories: [inc, libsrc, '../../../../developer/src/ext/json'], + link_args: links + tests_flags, + dependencies: [icu_uc, icu_i18n], + objects: lib.extract_all_objects(recursive: false)) +test('test_kmx_plus', e, suite: 'ldml') + +# run transforms / ldml utilities unit test + +t = executable('test_transforms', 'test_transforms.cpp', + cpp_args: defns + warns, + include_directories: [inc, libsrc, '../../../../developer/src/ext/json'], + link_args: links + tests_flags, + dependencies: [icu_uc, icu_i18n], + objects: lib.extract_all_objects(recursive: false)) +test('test_transforms', t, suite: 'ldml') + # Run tests on all keyboards (`tests` defined in keyboards/meson.build) foreach kbd : tests @@ -77,22 +99,3 @@ foreach kbd : invalid_tests test(kbd, ldml, args: [kbd_src, kbd_obj], suite: 'ldml-invalid-keyboards') # todo: consider if we should use `should_fail: true`? endforeach - -# Build and run additional test_kmx_plus test - -e = executable('test_kmx_plus', 'test_kmx_plus.cpp', - 'ldml_test_utils.cpp', - cpp_args: defns + warns, - include_directories: [inc, libsrc, '../../../../developer/src/ext/json'], - link_args: links + tests_flags, - dependencies: [icu_uc, icu_i18n], - objects: lib.extract_all_objects(recursive: false)) -test('test_kmx_plus', e, suite: 'ldml') - -t = executable('test_transforms', 'test_transforms.cpp', - cpp_args: defns + warns, - include_directories: [inc, libsrc, '../../../../developer/src/ext/json'], - link_args: links + tests_flags, - dependencies: [icu_uc, icu_i18n], - objects: lib.extract_all_objects(recursive: false)) -test('test_transforms', t, suite: 'ldml') From 3cd6c7630c067a33360834792ac13b3019ef2790 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 16 Nov 2023 13:55:49 -0600 Subject: [PATCH 4/5] =?UTF-8?q?feat(core):=20ldml=20tertiary=20reorder=20?= =?UTF-8?q?=F0=9F=99=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - handle case where there's no tertiary base before the first char - separate some element API tests from the actual tertiary test. Tests were failing because of now-correct implementation. Fixes: #9707 --- core/src/ldml/ldml_transforms.cpp | 2 +- core/tests/unit/ldml/test_transforms.cpp | 26 +++++++++++++++++++----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/core/src/ldml/ldml_transforms.cpp b/core/src/ldml/ldml_transforms.cpp index c7f061a310..270048ac21 100644 --- a/core/src/ldml/ldml_transforms.cpp +++ b/core/src/ldml/ldml_transforms.cpp @@ -236,7 +236,7 @@ element_list::update_sort_key(size_t offset, std::deque &key) k.tertiary = e->get_tertiary(); k.is_tertiary_base = e->is_tertiary_base(); - if (k.tertiary != 0) { + if (k.tertiary != 0 && n > 0) { // search backwards for a base auto n2 = n; // TODO-LDML: odd loop here because n2 is signed. diff --git a/core/tests/unit/ldml/test_transforms.cpp b/core/tests/unit/ldml/test_transforms.cpp index dd27bae143..29357071ac 100644 --- a/core/tests/unit/ldml/test_transforms.cpp +++ b/core/tests/unit/ldml/test_transforms.cpp @@ -158,6 +158,22 @@ int test_reorder_standalone() { std::cout << "== " << __FUNCTION__ << std::endl; + std::cout << __FILE__ << ":" << __LINE__ << " - element API test " << std::endl; + // element API test - not a real element, just here for testing + { + element es(U'a', 0xF4500000 | LDML_ELEM_FLAGS_PREBASE | LDML_ELEM_FLAGS_TERTIARY_BASE); // tertiary -12, primary 80 + std::cout << "es flags" << std::hex << es.get_flags() << std::dec << std::endl; + // verify element metadata + assert_equal(es.is_uset(), false); + assert_equal(es.get_order(), 0x50); + assert_equal(es.get_tertiary(), -12); + assert_equal(es.is_prebase(), true); + assert_equal(es.is_tertiary_base(), true); + // verify element matching + assert_equal(es.matches(U'a'), true); + assert_equal(es.matches(U'b'), false); + } + std::cout << __FILE__ << ":" << __LINE__ << " - nod-Lana " << std::endl; { const std::u32string roasts[] = { @@ -179,17 +195,17 @@ test_reorder_standalone() { assert_equal(toneMarks.contains(0x1A76), true); assert_equal(toneMarks.contains(0x1A60), false); - std::cout << __FILE__ << ":" << __LINE__ << " - element test " << std::endl; + std::cout << __FILE__ << ":" << __LINE__ << " - element API test " << std::endl; // element test { - element es(U'a', 0xF4500000 | LDML_ELEM_FLAGS_PREBASE | LDML_ELEM_FLAGS_TERTIARY_BASE); // tertiary -12, primary 80 + element es(U'a', (80 << LDML_ELEM_FLAGS_ORDER_BITSHIFT) | LDML_ELEM_FLAGS_PREBASE); // tertiary -12, primary 80 std::cout << "es flags" << std::hex << es.get_flags() << std::dec << std::endl; // verify element metadata assert_equal(es.is_uset(), false); assert_equal(es.get_order(), 0x50); - assert_equal(es.get_tertiary(), -12); + assert_equal(es.get_tertiary(), 0); assert_equal(es.is_prebase(), true); - assert_equal(es.is_tertiary_base(), true); + assert_equal(es.is_tertiary_base(), false); // verify element matching assert_equal(es.matches(U'a'), true); assert_equal(es.matches(U'b'), false); @@ -246,7 +262,7 @@ test_reorder_standalone() { // spot check first sortkey assert_equal(keylist.begin()->primary, 80); - assert_equal(keylist.begin()->tertiary, -12); + assert_equal(keylist.begin()->tertiary, 0); assert_equal(keylist.begin()->ch, 0x61); std::cout << __FILE__ << ":" << __LINE__ << " sorted sortkey" << std::endl; From 350eb83ab35413646d60fa1a45acffc0f36e0c0c Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 16 Nov 2023 16:13:48 -0600 Subject: [PATCH 5/5] =?UTF-8?q?feat(core):=20ldml=20typedef=20for=20reorde?= =?UTF-8?q?r=20weights=20=F0=9F=99=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For: #9707 --- core/src/ldml/ldml_transforms.cpp | 36 ++++++++++++------------ core/src/ldml/ldml_transforms.hpp | 17 +++++++---- core/tests/unit/ldml/test_transforms.cpp | 2 +- 3 files changed, 30 insertions(+), 25 deletions(-) diff --git a/core/src/ldml/ldml_transforms.cpp b/core/src/ldml/ldml_transforms.cpp index 270048ac21..973117f36b 100644 --- a/core/src/ldml/ldml_transforms.cpp +++ b/core/src/ldml/ldml_transforms.cpp @@ -45,16 +45,16 @@ element::is_uset() const { return (flags & LDML_ELEM_FLAGS_TYPE) == LDML_ELEM_FLAGS_TYPE_USET; } -signed char +reorder_weight element::get_order() const { unsigned char uorder = ((flags & LDML_ELEM_FLAGS_ORDER_MASK) >> LDML_ELEM_FLAGS_ORDER_BITSHIFT); - return (signed char)uorder; + return (reorder_weight)uorder; // unsigned to signed } -signed char +reorder_weight element::get_tertiary() const { unsigned char uorder = ((flags & LDML_ELEM_FLAGS_TERTIARY_MASK) >> LDML_ELEM_FLAGS_TERTIARY_BITSHIFT); - return (signed char)uorder; + return (reorder_weight)uorder; // unsigned to signed } bool @@ -93,19 +93,19 @@ element::dump() const { int reorder_sort_key::compare(const reorder_sort_key &other) const { - int primaryResult = (int)primary - (int)other.primary; - int secondaryResult = (int)secondary - (int)other.secondary; - int tertiaryResult = (int)tertiary - (int)other.tertiary; - int quaternaryResult = (int)quaternary - (int)other.quaternary; + auto primaryResult = primary - other.primary; + auto secondaryResult = secondary - other.secondary; + auto tertiaryResult = tertiary - other.tertiary; + auto quaternaryResult = quaternary - other.quaternary; if (primaryResult) { - return primaryResult; + return (int)primaryResult; } else if (secondaryResult) { - return secondaryResult; + return (int)secondaryResult; } else if (tertiaryResult) { - return tertiaryResult; + return (int)tertiaryResult; } else if (quaternaryResult) { - return quaternaryResult; + return (int)quaternaryResult; } else { // We don't expect to get here. quaternaryResult is the string index, which // should be unequal. @@ -131,8 +131,8 @@ reorder_sort_key::from(const std::u32string &str) { // construct a 'baseline' sort key, that is, in the absence of // any match rules. std::deque keylist; - auto s = str.begin(); // str iterator - size_t c = 0; // str index + auto s = str.begin(); // str iterator + reorder_weight c = 0; // str index for (auto e = str.begin(); e < str.end(); e++, s++, c++) { // primary weight: 0 // seconary weight: c (the string index) @@ -215,9 +215,9 @@ std::deque & element_list::update_sort_key(size_t offset, std::deque &key) const { /** string index */ size_t c = 0; - bool have_last_base = false; - signed char last_base_primary = -1; - signed char last_base_secondary = -1; + bool have_last_base = false; + reorder_weight last_base_primary = -1; + reorder_weight last_base_secondary = -1; for (auto e = begin(); e < end(); e++, c++) { /** position in the key */ auto n = offset + c; @@ -418,7 +418,7 @@ reorder_group::apply(std::u32string &str) const { // recombine into a string by pulling out the 'ch' value // that's in each sortkey element. std::u32string newSuffix; - size_t q = sort_keys.begin()->quaternary; // start with the first quaternary + signed char q = sort_keys.begin()->quaternary; // start with the first quaternary for (auto e = sort_keys.begin(); e < sort_keys.end(); e++, q++) { if (q != e->quaternary) { // something rearranged in this subrange, because the quaternary values are out of order. diff --git a/core/src/ldml/ldml_transforms.hpp b/core/src/ldml/ldml_transforms.hpp index 582a7ba3a9..dd08289574 100644 --- a/core/src/ldml/ldml_transforms.hpp +++ b/core/src/ldml/ldml_transforms.hpp @@ -47,6 +47,11 @@ inline bool uassert_success(const char *file, int line, const char *function, UE using km::core::kmx::SimpleUSet; +/** a reorder weight, such as primary, secondary, etc. */ +typedef signed char reorder_weight; + + + /** * Type of a group */ @@ -71,9 +76,9 @@ public: /** @returns true if tertiary base bit set */ bool is_tertiary_base() const; /** @returns the primary order */ - signed char get_order() const; + reorder_weight get_order() const; /** @returns the tertiary order */ - signed char get_tertiary() const; + reorder_weight get_tertiary() const; /** @returns raw elem flags */ KMX_DWORD get_flags() const; /** @returns true if matches this character*/ @@ -157,10 +162,10 @@ public: /** a single char, categorized according to reorder rules*/ struct reorder_sort_key { km_core_usv ch; // the single char value - signed char primary; // primary order value - size_t secondary; // index position - signed char tertiary; // tertiary value, defaults to 0 - size_t quaternary; // index again + reorder_weight primary; // primary order value + reorder_weight secondary; // index position + reorder_weight tertiary; // tertiary value, defaults to 0 + reorder_weight quaternary; // index again bool is_tertiary_base; // remember that this key was a tertiary base /** @returns -1, 0, 1 depending on ordering */ diff --git a/core/tests/unit/ldml/test_transforms.cpp b/core/tests/unit/ldml/test_transforms.cpp index 29357071ac..fe47f04395 100644 --- a/core/tests/unit/ldml/test_transforms.cpp +++ b/core/tests/unit/ldml/test_transforms.cpp @@ -251,7 +251,7 @@ test_reorder_standalone() { l.update_sort_key(0, keylist); std::cout << __FILE__ << ":" << __LINE__ << " updated sortkey" << std::endl; assert_equal(keylist.size(), 2); - size_t secondary = 0; + reorder_weight secondary = 0; for (auto i = keylist.begin(); i < keylist.end(); i++) { i->dump(); assert_equal(i->secondary, secondary);