From 600fdcc2c8cc220833cbdfd95c72bad9dd77d6c4 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Tue, 30 Jan 2024 11:50:18 -0600 Subject: [PATCH] =?UTF-8?q?feat(core):=20=20double=20markers=20once=20agai?= =?UTF-8?q?n=20=F0=9F=99=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - C++ side passes Fixes: #10516 --- core/src/ldml/ldml_markers.cpp | 11 +-- core/src/ldml/ldml_markers.hpp | 8 +++ core/tests/unit/ldml/test_transforms.cpp | 91 ++++++++++++++++-------- 3 files changed, 78 insertions(+), 32 deletions(-) diff --git a/core/src/ldml/ldml_markers.cpp b/core/src/ldml/ldml_markers.cpp index 08981a8ded..a3e31fd0df 100644 --- a/core/src/ldml/ldml_markers.cpp +++ b/core/src/ldml/ldml_markers.cpp @@ -65,7 +65,7 @@ marker_entry::marker_entry(char32_t c, marker_num n) : ch(c), marker(n), process } -static size_t count_markers(const marker_map &map) { +size_t count_markers(const marker_map &map) { size_t m = 0; for (auto i = map.begin(); i < map.end(); i++) { // add all actual markers @@ -436,9 +436,9 @@ add_pending_markers( } else { // 'glue' is the first codepoint of the decomposition. marker_ch = decomposition.char32At(0); - if(decomposition.countChar32() == 1) { + if (decomposition.countChar32() == 1) { decomposition.remove(); // no other entries needed - } + } // else: will add the remainder below } } markers->emplace_back(marker_ch); @@ -449,7 +449,10 @@ add_pending_markers( } // add any further entries due to decomposition if (!decomposition.isEmpty()) { - assert(false); // TODO + // We already added the base char above, add teh rest + for (auto i=1; iemplace_back(decomposition.char32At(i)); + } } // clear the list last_markers.clear(); diff --git a/core/src/ldml/ldml_markers.hpp b/core/src/ldml/ldml_markers.hpp index bf15937d14..1c737387ad 100644 --- a/core/src/ldml/ldml_markers.hpp +++ b/core/src/ldml/ldml_markers.hpp @@ -58,11 +58,19 @@ struct marker_entry { marker_entry(char32_t ch); /** add a 'marker' entry */ marker_entry(char32_t ch, marker_num marker); + + bool operator==(const marker_entry &o) const { + // don't test 'processed' + return (ch == o.ch) && (marker == o.marker) && (end == o.end); + } }; /** map from following-char to marker numbers, in front to back order */ typedef std::deque marker_map; +/** count number of non-end entries */ +size_t count_markers(const marker_map &map); + /** Normalize a u32string inplace to NFD. @return false on failure */ bool normalize_nfd(std::u32string &str); /** Normalize a u16string inplace to NFD. @return false on failure */ diff --git a/core/tests/unit/ldml/test_transforms.cpp b/core/tests/unit/ldml/test_transforms.cpp index d7b015e18f..a161061ae0 100644 --- a/core/tests/unit/ldml/test_transforms.cpp +++ b/core/tests/unit/ldml/test_transforms.cpp @@ -61,17 +61,34 @@ marker_map_to_string(const marker_map &m) { if (i->processed) { s.insert(0, U""); } - prepend_hex_oct(s, i->ch); - s.insert(0, U"=U+"); + if (i->ch == MARKER_BEFORE_EOT) { + s.insert(0, U""); + } else { + prepend_hex_oct(s, i->ch); + s.insert(0, U"=U+"); + } - prepend_hex_quad(s, i->marker); - s.insert(0, U" \\m0x"); + if (i->marker != LDML_MARKER_NO_INDEX) { + prepend_hex_quad(s, i->marker); + s.insert(0, U" \\m0x"); + } } return s; } bool -_assert_marker_map_equal(const char *f, int l, const marker_map a, const marker_map x) { +_assert_marker_map_equal(const char *f, int l, const marker_map orig, const marker_map x) { + marker_map a; + // copy everything but 'end' entries + std::copy_if( + orig.begin(), orig.end(), std::back_inserter(a), [](const marker_entry &m) { return m.marker != LDML_MARKER_NO_INDEX; }); + + // now check equality + if (a == x) { + return true; + } + + // Not equal. Print out why. std::wcerr << f << ":" << l << ": " << console_color::fg(console_color::BRIGHT_RED); std::wcerr << "got: " << marker_map_to_string(a); std::wcerr << " expected: " << marker_map_to_string(x); @@ -640,7 +657,7 @@ test_strutils() { const std::u32string src = U"abc"; const std::u32string dst = remove_markers(src, map); zassert_string_equal(dst, src); // unchanged - assert_equal(map.size(), 0); + assert_equal(count_markers(map), 0); } { marker_map map; @@ -651,7 +668,7 @@ test_strutils() { zassert_string_equal(dst, expect); marker_map expm = {{U'e', 0x1L}}; assert_marker_map_equal(map, expm); // marker 1 @ e - assert_equal(map.size(), 1); + assert_equal(count_markers(map), 1); } { marker_map map; @@ -660,7 +677,7 @@ test_strutils() { const std::u32string dst = remove_markers(src, map); const std::u32string expect = src; zassert_string_equal(dst, expect); - assert_equal(map.size(), 0); + assert_equal(count_markers(map), 0); } { marker_map map; @@ -669,7 +686,7 @@ test_strutils() { const std::u32string dst = remove_markers(src, map); const std::u32string expect = src; // 'q' removed zassert_string_equal(dst, expect); - assert_equal(map.size(), 0); + assert_equal(count_markers(map), 0); } { marker_map map; @@ -678,7 +695,7 @@ test_strutils() { const std::u32string dst = remove_markers(src, map); const std::u32string expect = src; zassert_string_equal(dst, expect); - assert_equal(map.size(), 0); + assert_equal(count_markers(map), 0); } { marker_map map; @@ -687,7 +704,7 @@ test_strutils() { const std::u32string dst = remove_markers(src, map); const std::u32string expect = src; zassert_string_equal(dst, expect); - assert_equal(map.size(), 0); + assert_equal(count_markers(map), 0); } { marker_map map; @@ -698,7 +715,7 @@ test_strutils() { zassert_string_equal(dst, expect); marker_map expm({{MARKER_BEFORE_EOT, 0x1L}}); assert_marker_map_equal(map, expm); - assert_equal(map.size(), 1); + assert_equal(count_markers(map), 1); } { marker_map map; @@ -711,7 +728,7 @@ test_strutils() { zassert_string_equal(dst, expect); marker_map expm({{U'e', 0x1L}, {0x0320, 0x2L}, {0x0300, 0x3L}, {MARKER_BEFORE_EOT, 0x4L}}); assert_marker_map_equal(map, expm); - assert_equal(map.size(), 4); + assert_equal(count_markers(map), 4); } { std::cout << __FILE__ << ":" << __LINE__ << " - prepend hex quad" << std::endl; @@ -756,7 +773,7 @@ test_normalize() { std::u32string dst = src; assert(normalize_nfd_markers_segment(dst, map)); zassert_string_equal(dst, expect); - assert_equal(map.size(), 0); + assert_equal(count_markers(map), 0); } { marker_map map; @@ -766,7 +783,7 @@ test_normalize() { std::u32string dst = src; assert(normalize_nfd_markers_segment(dst, map)); zassert_string_equal(dst, expect); - assert_equal(map.size(), 0); + assert_equal(count_markers(map), 0); } { @@ -781,7 +798,7 @@ test_normalize() { zassert_string_equal(dst, expect); marker_map expm({{U'e', 0x1L}, {0x320, 0x2L}, {0x300, 0x3L}, {MARKER_BEFORE_EOT, 0x4L}}); assert_marker_map_equal(map, expm); - assert_equal(map.size(), 4); + assert_equal(count_markers(map), 4); } { @@ -796,7 +813,7 @@ test_normalize() { zassert_string_equal(dst, expect); marker_map expm({{U'e', 0x1L}, {0x320, 0x2L}, {0x300, 0x3L}, {MARKER_BEFORE_EOT, 0x4L}}); assert_marker_map_equal(map, expm); - assert_equal(map.size(), 4); + assert_equal(count_markers(map), 4); } { marker_map map; @@ -816,7 +833,7 @@ test_normalize() { zassert_string_equal(dst, expect); marker_map expm({{U'e', 0x1L}, {0x300, 0x2L}, {0x320, 0x3L}, {MARKER_BEFORE_EOT, 0x4L}}); assert_marker_map_equal(map, expm); - assert_equal(map.size(), 4); + assert_equal(count_markers(map), 4); } { @@ -834,7 +851,7 @@ test_normalize() { zassert_string_equal(dst, expect); marker_map expm({{0x320, 0x1L}}); assert_marker_map_equal(map, expm); - assert_equal(map.size(), 1); + assert_equal(count_markers(map), 1); } { @@ -852,7 +869,7 @@ test_normalize() { zassert_string_equal(dst, expect); marker_map expm({{0x320, 0x2L}, {MARKER_BEFORE_EOT, 0x1L}}); assert_marker_map_equal(map, expm); - assert_equal(map.size(), 2); + assert_equal(count_markers(map), 2); } { @@ -872,7 +889,7 @@ test_normalize() { zassert_string_equal(dst, expect); marker_map expm({{0x320, 0x2L}, {MARKER_BEFORE_EOT, 0x1L}}); assert_marker_map_equal(map, expm); - assert_equal(map.size(), 2); + assert_equal(count_markers(map), 2); } { // from tests - regex edition @@ -889,7 +906,7 @@ test_normalize() { zassert_string_equal(dst, expect); marker_map expm({{0x320, LDML_MARKER_ANY_INDEX}, {MARKER_BEFORE_EOT, 0x1L}}); assert_marker_map_equal(map, expm); - assert_equal(map.size(), 2); + assert_equal(count_markers(map), 2); } { @@ -907,7 +924,7 @@ test_normalize() { zassert_string_equal(dst, expect); marker_map expm({{0x320, 0x2L}, {0x320, 0x2L}}); assert_marker_map_equal(map, expm); - assert_equal(map.size(), 2); + assert_equal(count_markers(map), 2); } { @@ -1042,7 +1059,7 @@ test_normalize() { const std::u32string expect_rem = U"e\u0300\u0320\u0300"; const std::u32string expect_nfd = U"e\u0320\uffff\u0008\u0001\u0300\u0300"; auto dst_rem = remove_markers(src, &map); -// marker_map expm({{0x0308, 0x1L}}); + marker_map expm({{0x0300, 0x1L}}); zassert_string_equal(dst_rem, expect_rem); std::u32string dst_nfd = src; assert(normalize_nfd_markers(dst_nfd)); @@ -1051,16 +1068,16 @@ test_normalize() { std::cout << "exp: " << Debug_UnicodeString(expect_nfd) << std::endl; } zassert_string_equal(dst_nfd, expect_nfd); - // assert_marker_map_equal(map, expm); + assert_marker_map_equal(map, expm); } { marker_map map; std::cout << __FILE__ << ":" << __LINE__ << " - doubled unchanged marker " << std::endl; const std::u32string src = U"e\u0320\uffff\u0008\u0001\u0300\u0300"; - const std::u32string expect_rem = U"e\u0300\u0320\u0300"; + const std::u32string expect_rem = U"e\u0320\u0300\u0300"; const std::u32string expect_nfd = src; auto dst_rem = remove_markers(src, &map); - // marker_map expm({{0x0308, 0x1L}}); + marker_map expm({{0x300, 0x1L}}); zassert_string_equal(dst_rem, expect_rem); std::u32string dst_nfd = src; assert(normalize_nfd_markers(dst_nfd)); @@ -1069,7 +1086,25 @@ test_normalize() { std::cout << "exp: " << Debug_UnicodeString(expect_nfd) << std::endl; } zassert_string_equal(dst_nfd, expect_nfd); - // assert_marker_map_equal(map, expm); + assert_marker_map_equal(map, expm); + } + { + marker_map map; + std::cout << __FILE__ << ":" << __LINE__ << " - marker-before-double-greek " << std::endl; + const std::u32string src = U"\u03B5\uFFFF\u0008\u0001\u0344\uFFFF\u0008\u0002\u0344\uFFFF\u0008\u0003"; + const std::u32string expect_rem = U"\u03B5\u0344\u0344"; + const std::u32string expect_nfd = U"\u03B5\uFFFF\u0008\u0001\u0308\u0301\uFFFF\u0008\u0002\u0308\u0301\uFFFF\u0008\u0003"; + auto dst_rem = remove_markers(src, &map); + marker_map expm({{0x0308, 0x1L},{0x0308, 0x2L},{MARKER_BEFORE_EOT, 0x3L}}); + zassert_string_equal(dst_rem, expect_rem); + std::u32string dst_nfd = src; + assert(normalize_nfd_markers(dst_nfd)); + if (dst_nfd != expect_nfd) { + std::cout << "dst: " << Debug_UnicodeString(dst_nfd) << std::endl; + std::cout << "exp: " << Debug_UnicodeString(expect_nfd) << std::endl; + } + zassert_string_equal(dst_nfd, expect_nfd); + assert_marker_map_equal(map, expm); } return EXIT_SUCCESS;