erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-08-09 10:25:32 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

feat(core): double markers once again 🙀

Browse source 
- C++ side passes

Fixes: #10516
This commit is contained in:
Steven R. Loomis 2024-01-30 11:50:18 -06:00
parent 2891d19d88
commit 600fdcc2c8
3 changed files with 78 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
core/src/ldml/ldml_markers.cpp
View file

@ -65,7 +65,7 @@ marker_entry::marker_entry(char32_t c, marker_num n) : ch(c), marker(n), process
}
static size_t count_markers(const marker_map &map) {
size_t count_markers(const marker_map &map) {
size_t m = 0;
for (auto i = map.begin(); i < map.end(); i++) {
// add all actual markers
@ -436,9 +436,9 @@ add_pending_markers(
} else {
// 'glue' is the first codepoint of the decomposition.
marker_ch = decomposition.char32At(0);
if(decomposition.countChar32() == 1) {
if (decomposition.countChar32() == 1) {
decomposition.remove(); // no other entries needed
}
} // else: will add the remainder below
}
}
markers->emplace_back(marker_ch);
@ -449,7 +449,10 @@ add_pending_markers(
}
// add any further entries due to decomposition
if (!decomposition.isEmpty()) {
assert(false); // TODO
// We already added the base char above, add teh rest
for (auto i=1; i<decomposition.countChar32(); i++) {
markers->emplace_back(decomposition.char32At(i));
}
}
// clear the list
last_markers.clear();

8
core/src/ldml/ldml_markers.hpp
View file

@ -58,11 +58,19 @@ struct marker_entry {
marker_entry(char32_t ch);
/** add a 'marker' entry */
marker_entry(char32_t ch, marker_num marker);
bool operator==(const marker_entry &o) const {
// don't test 'processed'
return (ch == o.ch) && (marker == o.marker) && (end == o.end);
}
};
/** map from following-char to marker numbers, in front to back order */
typedef std::deque<marker_entry> marker_map;
/** count number of non-end entries */
size_t count_markers(const marker_map &map);
/** Normalize a u32string inplace to NFD. @return false on failure */
bool normalize_nfd(std::u32string &str);
/** Normalize a u16string inplace to NFD. @return false on failure */

91
core/tests/unit/ldml/test_transforms.cpp
View file

@ -61,17 +61,34 @@ marker_map_to_string(const marker_map &m) {
if (i->processed) {
s.insert(0, U"<PROCESSED?>");
}
prepend_hex_oct(s, i->ch);
s.insert(0, U"=U+");
if (i->ch == MARKER_BEFORE_EOT) {
s.insert(0, U"<EOT>");
} else {
prepend_hex_oct(s, i->ch);
s.insert(0, U"=U+");
}
prepend_hex_quad(s, i->marker);
s.insert(0, U" \\m0x");
if (i->marker != LDML_MARKER_NO_INDEX) {
prepend_hex_quad(s, i->marker);
s.insert(0, U" \\m0x");
}
}
return s;
}
bool
_assert_marker_map_equal(const char *f, int l, const marker_map a, const marker_map x) {
_assert_marker_map_equal(const char *f, int l, const marker_map orig, const marker_map x) {
marker_map a;
// copy everything but 'end' entries
std::copy_if(
orig.begin(), orig.end(), std::back_inserter(a), [](const marker_entry &m) { return m.marker != LDML_MARKER_NO_INDEX; });
// now check equality
if (a == x) {
return true;
}
// Not equal. Print out why.
std::wcerr << f << ":" << l << ": " << console_color::fg(console_color::BRIGHT_RED);
std::wcerr << "got: " << marker_map_to_string(a);
std::wcerr << " expected: " << marker_map_to_string(x);
@ -640,7 +657,7 @@ test_strutils() {
const std::u32string src = U"abc";
const std::u32string dst = remove_markers(src, map);
zassert_string_equal(dst, src); // unchanged
assert_equal(map.size(), 0);
assert_equal(count_markers(map), 0);
}
{
marker_map map;
@ -651,7 +668,7 @@ test_strutils() {
zassert_string_equal(dst, expect);
marker_map expm = {{U'e', 0x1L}};
assert_marker_map_equal(map, expm); // marker 1 @ e
assert_equal(map.size(), 1);
assert_equal(count_markers(map), 1);
}
{
marker_map map;
@ -660,7 +677,7 @@ test_strutils() {
const std::u32string dst = remove_markers(src, map);
const std::u32string expect = src;
zassert_string_equal(dst, expect);
assert_equal(map.size(), 0);
assert_equal(count_markers(map), 0);
}
{
marker_map map;
@ -669,7 +686,7 @@ test_strutils() {
const std::u32string dst = remove_markers(src, map);
const std::u32string expect = src; // 'q' removed
zassert_string_equal(dst, expect);
assert_equal(map.size(), 0);
assert_equal(count_markers(map), 0);
}
{
marker_map map;
@ -678,7 +695,7 @@ test_strutils() {
const std::u32string dst = remove_markers(src, map);
const std::u32string expect = src;
zassert_string_equal(dst, expect);
assert_equal(map.size(), 0);
assert_equal(count_markers(map), 0);
}
{
marker_map map;
@ -687,7 +704,7 @@ test_strutils() {
const std::u32string dst = remove_markers(src, map);
const std::u32string expect = src;
zassert_string_equal(dst, expect);
assert_equal(map.size(), 0);
assert_equal(count_markers(map), 0);
}
{
marker_map map;
@ -698,7 +715,7 @@ test_strutils() {
zassert_string_equal(dst, expect);
marker_map expm({{MARKER_BEFORE_EOT, 0x1L}});
assert_marker_map_equal(map, expm);
assert_equal(map.size(), 1);
assert_equal(count_markers(map), 1);
}
{
marker_map map;
@ -711,7 +728,7 @@ test_strutils() {
zassert_string_equal(dst, expect);
marker_map expm({{U'e', 0x1L}, {0x0320, 0x2L}, {0x0300, 0x3L}, {MARKER_BEFORE_EOT, 0x4L}});
assert_marker_map_equal(map, expm);
assert_equal(map.size(), 4);
assert_equal(count_markers(map), 4);
}
{
std::cout << __FILE__ << ":" << __LINE__ << " - prepend hex quad" << std::endl;
@ -756,7 +773,7 @@ test_normalize() {
std::u32string dst = src;
assert(normalize_nfd_markers_segment(dst, map));
zassert_string_equal(dst, expect);
assert_equal(map.size(), 0);
assert_equal(count_markers(map), 0);
}
{
marker_map map;
@ -766,7 +783,7 @@ test_normalize() {
std::u32string dst = src;
assert(normalize_nfd_markers_segment(dst, map));
zassert_string_equal(dst, expect);
assert_equal(map.size(), 0);
assert_equal(count_markers(map), 0);
}
{
@ -781,7 +798,7 @@ test_normalize() {
zassert_string_equal(dst, expect);
marker_map expm({{U'e', 0x1L}, {0x320, 0x2L}, {0x300, 0x3L}, {MARKER_BEFORE_EOT, 0x4L}});
assert_marker_map_equal(map, expm);
assert_equal(map.size(), 4);
assert_equal(count_markers(map), 4);
}
{
@ -796,7 +813,7 @@ test_normalize() {
zassert_string_equal(dst, expect);
marker_map expm({{U'e', 0x1L}, {0x320, 0x2L}, {0x300, 0x3L}, {MARKER_BEFORE_EOT, 0x4L}});
assert_marker_map_equal(map, expm);
assert_equal(map.size(), 4);
assert_equal(count_markers(map), 4);
}
{
marker_map map;
@ -816,7 +833,7 @@ test_normalize() {
zassert_string_equal(dst, expect);
marker_map expm({{U'e', 0x1L}, {0x300, 0x2L}, {0x320, 0x3L}, {MARKER_BEFORE_EOT, 0x4L}});
assert_marker_map_equal(map, expm);
assert_equal(map.size(), 4);
assert_equal(count_markers(map), 4);
}
{
@ -834,7 +851,7 @@ test_normalize() {
zassert_string_equal(dst, expect);
marker_map expm({{0x320, 0x1L}});
assert_marker_map_equal(map, expm);
assert_equal(map.size(), 1);
assert_equal(count_markers(map), 1);
}
{
@ -852,7 +869,7 @@ test_normalize() {
zassert_string_equal(dst, expect);
marker_map expm({{0x320, 0x2L}, {MARKER_BEFORE_EOT, 0x1L}});
assert_marker_map_equal(map, expm);
assert_equal(map.size(), 2);
assert_equal(count_markers(map), 2);
}
{
@ -872,7 +889,7 @@ test_normalize() {
zassert_string_equal(dst, expect);
marker_map expm({{0x320, 0x2L}, {MARKER_BEFORE_EOT, 0x1L}});
assert_marker_map_equal(map, expm);
assert_equal(map.size(), 2);
assert_equal(count_markers(map), 2);
}
{
// from tests - regex edition
@ -889,7 +906,7 @@ test_normalize() {
zassert_string_equal(dst, expect);
marker_map expm({{0x320, LDML_MARKER_ANY_INDEX}, {MARKER_BEFORE_EOT, 0x1L}});
assert_marker_map_equal(map, expm);
assert_equal(map.size(), 2);
assert_equal(count_markers(map), 2);
}
{
@ -907,7 +924,7 @@ test_normalize() {
zassert_string_equal(dst, expect);
marker_map expm({{0x320, 0x2L}, {0x320, 0x2L}});
assert_marker_map_equal(map, expm);
assert_equal(map.size(), 2);
assert_equal(count_markers(map), 2);
}
{
@ -1042,7 +1059,7 @@ test_normalize() {
const std::u32string expect_rem = U"e\u0300\u0320\u0300";
const std::u32string expect_nfd = U"e\u0320\uffff\u0008\u0001\u0300\u0300";
auto dst_rem = remove_markers(src, &map);
// marker_map expm({{0x0308, 0x1L}});
marker_map expm({{0x0300, 0x1L}});
zassert_string_equal(dst_rem, expect_rem);
std::u32string dst_nfd = src;
assert(normalize_nfd_markers(dst_nfd));
@ -1051,16 +1068,16 @@ test_normalize() {
std::cout << "exp: " << Debug_UnicodeString(expect_nfd) << std::endl;
}
zassert_string_equal(dst_nfd, expect_nfd);
// assert_marker_map_equal(map, expm);
assert_marker_map_equal(map, expm);
}
{
marker_map map;
std::cout << __FILE__ << ":" << __LINE__ << " - doubled unchanged marker " << std::endl;
const std::u32string src = U"e\u0320\uffff\u0008\u0001\u0300\u0300";
const std::u32string expect_rem = U"e\u0300\u0320\u0300";
const std::u32string expect_rem = U"e\u0320\u0300\u0300";
const std::u32string expect_nfd = src;
auto dst_rem = remove_markers(src, &map);
// marker_map expm({{0x0308, 0x1L}});
marker_map expm({{0x300, 0x1L}});
zassert_string_equal(dst_rem, expect_rem);
std::u32string dst_nfd = src;
assert(normalize_nfd_markers(dst_nfd));
@ -1069,7 +1086,25 @@ test_normalize() {
std::cout << "exp: " << Debug_UnicodeString(expect_nfd) << std::endl;
}
zassert_string_equal(dst_nfd, expect_nfd);
// assert_marker_map_equal(map, expm);
assert_marker_map_equal(map, expm);
}
{
marker_map map;
std::cout << __FILE__ << ":" << __LINE__ << " - marker-before-double-greek " << std::endl;
const std::u32string src = U"\u03B5\uFFFF\u0008\u0001\u0344\uFFFF\u0008\u0002\u0344\uFFFF\u0008\u0003";
const std::u32string expect_rem = U"\u03B5\u0344\u0344";
const std::u32string expect_nfd = U"\u03B5\uFFFF\u0008\u0001\u0308\u0301\uFFFF\u0008\u0002\u0308\u0301\uFFFF\u0008\u0003";
auto dst_rem = remove_markers(src, &map);
marker_map expm({{0x0308, 0x1L},{0x0308, 0x2L},{MARKER_BEFORE_EOT, 0x3L}});
zassert_string_equal(dst_rem, expect_rem);
std::u32string dst_nfd = src;
assert(normalize_nfd_markers(dst_nfd));
if (dst_nfd != expect_nfd) {
std::cout << "dst: " << Debug_UnicodeString(dst_nfd) << std::endl;
std::cout << "exp: " << Debug_UnicodeString(expect_nfd) << std::endl;
}
zassert_string_equal(dst_nfd, expect_nfd);
assert_marker_map_equal(map, expm);
}
return EXIT_SUCCESS;