From 52fe9073a047feeca7367a0bb1adf2e362eda1ff Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 5 Jan 2024 18:52:43 -0600 Subject: [PATCH] =?UTF-8?q?chore(core):=20stacked=20markers=20tests=20?= =?UTF-8?q?=F0=9F=99=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - first we break #10320 --- core/src/ldml/ldml_transforms.cpp | 2 +- core/src/ldml/ldml_transforms.hpp | 15 +++++++-- core/tests/unit/ldml/test_transforms.cpp | 39 ++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 4 deletions(-) diff --git a/core/src/ldml/ldml_transforms.cpp b/core/src/ldml/ldml_transforms.cpp index 6f1ddcb0a5..249d19c351 100644 --- a/core/src/ldml/ldml_transforms.cpp +++ b/core/src/ldml/ldml_transforms.cpp @@ -1049,7 +1049,7 @@ bool normalize_nfc(std::u16string &str) { } void -prepend_marker(std::u32string &str, KMX_DWORD marker, marker_encoding encoding) { +prepend_marker(std::u32string &str, marker_num marker, marker_encoding encoding) { if (encoding == plain_sentinel) { km_core_usv markstr[] = {LDML_UC_SENTINEL, LDML_MARKER_CODE, marker}; str.insert(0, markstr, 3); diff --git a/core/src/ldml/ldml_transforms.hpp b/core/src/ldml/ldml_transforms.hpp index 70f8b7b1f7..0256fe6dd3 100644 --- a/core/src/ldml/ldml_transforms.hpp +++ b/core/src/ldml/ldml_transforms.hpp @@ -311,8 +311,14 @@ enum marker_encoding { regex_sentinel, }; -/** map from following-char to marker number. */ -typedef std::map marker_map; +/** a marker ID (1-based) */ +typedef KMX_DWORD marker_num; + +/** list of markers */ +typedef std::deque marker_list; + +/** map from following-char to marker numbers. */ +typedef std::map marker_map; /** Normalize a u32string inplace to NFD. @return false on failure */ bool normalize_nfd(std::u32string &str); @@ -338,17 +344,20 @@ inline bool normalize_nfc_markers(std::u16string &str, marker_encoding encoding /** Normalize a u32string inplace to NFC. @return false on failure */ bool normalize_nfc(std::u32string &str); + /** Normalize a u16string inplace to NFC. @return false on failure */ bool normalize_nfc(std::u16string &str); + /** Remove markers and optionally note their glue characters in the map */ std::u32string remove_markers(const std::u32string &str, marker_map *markers = nullptr, marker_encoding encoding = plain_sentinel); + /** same but with a reference */ inline std::u32string remove_markers(const std::u32string &str, marker_map &markers, marker_encoding encoding = plain_sentinel) { return remove_markers(str, &markers, encoding); } /** prepend the marker string in UC_SENTINEL format to the str */ -void prepend_marker(std::u32string &str, KMX_DWORD marker, marker_encoding encoding = plain_sentinel); +void prepend_marker(std::u32string &str, marker_num marker, marker_encoding encoding = plain_sentinel); /** format 'marker' as 0001...FFFF and put it at the beginning of the string */ void prepend_hex_quad(std::u32string &str, KMX_DWORD marker); diff --git a/core/tests/unit/ldml/test_transforms.cpp b/core/tests/unit/ldml/test_transforms.cpp index fae6eb33db..146a1de9be 100644 --- a/core/tests/unit/ldml/test_transforms.cpp +++ b/core/tests/unit/ldml/test_transforms.cpp @@ -839,6 +839,45 @@ int test_normalize() { assert_equal(map[MARKER_BEFORE_EOT], 0x1L); } + { + // from tests + marker_map map; + std::cout << __FILE__ << ":" << __LINE__ << " - complex test 10 stack o' 2x2" << std::endl; + const std::u32string src = U"9ce\u0300\uFFFF\u0008\u0002\uFFFF\u0008\u0002\u0320"; + const std::u32string expect = U"9ce\uFFFF\u0008\u0002\uFFFF\u0008\u0002\u0320\u0300"; + std::u32string dst = src; + assert(normalize_nfd_markers(dst, map)); + if (dst != expect) { + std::cout << "dst: " << Debug_UnicodeString(dst) << std::endl; + std::cout << "exp: " << Debug_UnicodeString(expect) << std::endl; + } + zassert_string_equal(dst, expect); + // TODO-LDML: map is going to be off + // assert_equal(map.size(), 2); + // assert_equal(map[0x0320], 0x2L); + // assert_equal(map[MARKER_BEFORE_EOT], 0x1L); + } + + + { + // from tests + marker_map map; + std::cout << __FILE__ << ":" << __LINE__ << " - complex test 10 stack o' 2x1x2" << std::endl; + const std::u32string src = U"9ce\u0300\uFFFF\u0008\u0002\uFFFF\u0008\u0001\uFFFF\u0008\u0002\u0320"; + const std::u32string expect = U"9ce\uFFFF\u0008\u0002\uFFFF\u0008\u0001\uFFFF\u0008\u0002\u0320\u0300"; + std::u32string dst = src; + assert(normalize_nfd_markers(dst, map)); + if (dst != expect) { + std::cout << "dst: " << Debug_UnicodeString(dst) << std::endl; + std::cout << "exp: " << Debug_UnicodeString(expect) << std::endl; + } + zassert_string_equal(dst, expect); + // TODO-LDML: map is going to be off + // assert_equal(map.size(), 2); + // assert_equal(map[0x0320], 0x2L); + // assert_equal(map[MARKER_BEFORE_EOT], 0x1L); + } + return EXIT_SUCCESS; }