From 410a4824bc83d669e415c370556742cb62279793 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 6 May 2026 14:01:18 +0200 Subject: [PATCH 1/8] chore(core): use ASSERT_FALSE and ASSERT_TRUE where possible Relates-to: #15913 Test-bot: skip --- core/tests/unit/api/kmx_xstring.tests.cpp | 30 +++++++++++------------ core/tests/unit/ldml/kmx_plus.tests.cpp | 16 ++++++------ core/tests/unit/ldml/transforms.tests.cpp | 26 ++++++++++---------- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/core/tests/unit/api/kmx_xstring.tests.cpp b/core/tests/unit/api/kmx_xstring.tests.cpp index c7fdbe7797..bf0a296179 100644 --- a/core/tests/unit/api/kmx_xstring.tests.cpp +++ b/core/tests/unit/api/kmx_xstring.tests.cpp @@ -1352,23 +1352,23 @@ TEST(XStringTests, TestU32stringToU16string) { TEST(XStringTests, TestIsValid) { // valid - ASSERT_EQ(Uni_IsValid(0x0000), true); - ASSERT_EQ(Uni_IsValid(0x0127), true); - ASSERT_EQ(Uni_IsValid(U'🙀'), true); + ASSERT_TRUE(Uni_IsValid(0x0000)); + ASSERT_TRUE(Uni_IsValid(0x0127)); + ASSERT_TRUE(Uni_IsValid(U'🙀')); // invalid - ASSERT_EQ(Uni_IsValid(0xDECAFBAD), false); // out of range - ASSERT_EQ(Uni_IsValid(0x566D4128), false); - ASSERT_EQ(Uni_IsValid(0xFFFF), false); // nonchar - ASSERT_EQ(Uni_IsValid(0xFFFE), false); // nonchar - ASSERT_EQ(Uni_IsValid(0x10FFFF), false); // nonchar - ASSERT_EQ(Uni_IsValid(0x10FFFE), false); // nonchar - ASSERT_EQ(Uni_IsValid(0x01FFFF), false); // nonchar - ASSERT_EQ(Uni_IsValid(0x01FFFE), false); // nonchar - ASSERT_EQ(Uni_IsValid(0x02FFFF), false); // nonchar - ASSERT_EQ(Uni_IsValid(0x02FFFE), false); // nonchar - ASSERT_EQ(Uni_IsValid(0xFDD1), false); // nonchar - ASSERT_EQ(Uni_IsValid(0xFDD0), false); // nonchar + ASSERT_FALSE(Uni_IsValid(0xDECAFBAD)); // out of range + ASSERT_FALSE(Uni_IsValid(0x566D4128)); + ASSERT_FALSE(Uni_IsValid(0xFFFF)); // nonchar + ASSERT_FALSE(Uni_IsValid(0xFFFE)); // nonchar + ASSERT_FALSE(Uni_IsValid(0x10FFFF)); // nonchar + ASSERT_FALSE(Uni_IsValid(0x10FFFE)); // nonchar + ASSERT_FALSE(Uni_IsValid(0x01FFFF)); // nonchar + ASSERT_FALSE(Uni_IsValid(0x01FFFE)); // nonchar + ASSERT_FALSE(Uni_IsValid(0x02FFFF)); // nonchar + ASSERT_FALSE(Uni_IsValid(0x02FFFE)); // nonchar + ASSERT_FALSE(Uni_IsValid(0xFDD1)); // nonchar + ASSERT_FALSE(Uni_IsValid(0xFDD0)); // nonchar // positive range test diff --git a/core/tests/unit/ldml/kmx_plus.tests.cpp b/core/tests/unit/ldml/kmx_plus.tests.cpp index 42ce3a7b49..a07c104cc3 100644 --- a/core/tests/unit/ldml/kmx_plus.tests.cpp +++ b/core/tests/unit/ldml/kmx_plus.tests.cpp @@ -104,13 +104,13 @@ TEST(KMXPlusTest, VkeysHandledCorrectly) { bool found = false; ASSERT_EQ(vk.lookup(km::tests::get_vk( "K_F"), 0, found), u""); - ASSERT_EQ(found, true); // K_F found, but empty string (gap) + ASSERT_TRUE(found); // K_F found, but empty string (gap) ASSERT_EQ(vk.lookup(km::tests::get_vk( "K_ENTER"), 0, found), u""); - ASSERT_EQ(found, false); // K_ENTER not found, empty string + ASSERT_FALSE(found); // K_ENTER not found, empty string ASSERT_EQ(vk.lookup(km::tests::get_vk( "K_A"), 0, found), u"K_A-0"); - ASSERT_EQ(found, true); // expect + ASSERT_TRUE(found); // expect ASSERT_EQ(vk.lookup(km::tests::get_vk( "K_A"), LCTRLFLAG, found), u"K_A-LCTRLFLAG"); ASSERT_EQ(vk.lookup(km::tests::get_vk( @@ -173,13 +173,13 @@ TEST(KMXPlusTest, UsetHandledCorrectly) { }; SimpleUSet u0(&r[0], 2); - ASSERT_EQ(u0.contains(0x62), true); // b - ASSERT_EQ(u0.contains(0x41), false); // A - ASSERT_EQ(u0.contains(0x127), true); // ħ + ASSERT_TRUE(u0.contains(0x62)); // b + ASSERT_FALSE(u0.contains(0x41)); // A + ASSERT_TRUE(u0.contains(0x127)); // ħ SimpleUSet uempty; - ASSERT_EQ(uempty.contains(0x62), false); - ASSERT_EQ(uempty.contains(0x127), false); + ASSERT_FALSE(uempty.contains(0x62)); + ASSERT_FALSE(uempty.contains(0x127)); } /** tests of the COMP_KMXPLUS_STRS::valid_string() */ diff --git a/core/tests/unit/ldml/transforms.tests.cpp b/core/tests/unit/ldml/transforms.tests.cpp index 58da00f5a9..a51070e743 100644 --- a/core/tests/unit/ldml/transforms.tests.cpp +++ b/core/tests/unit/ldml/transforms.tests.cpp @@ -200,8 +200,8 @@ TEST(TransformsTests, TestReordersWithStandaloneNodLanaExample) { const COMP_KMXPLUS_USET_USET &toneMarksUset = usets[0]; const SimpleUSet toneMarks(&ranges[toneMarksUset.range], toneMarksUset.count); // validate that the range [1A75, 1A79] matches - ASSERT_EQ(toneMarks.contains(0x1A76), true); - ASSERT_EQ(toneMarks.contains(0x1A60), false); + ASSERT_TRUE(toneMarks.contains(0x1A76)); + ASSERT_FALSE(toneMarks.contains(0x1A60)); std::cout << __FILE__ << ":" << __LINE__ << " - element API test " << std::endl; // element test @@ -209,28 +209,28 @@ TEST(TransformsTests, TestReordersWithStandaloneNodLanaExample) { element es(U'a', (80 << LDML_ELEM_FLAGS_ORDER_BITSHIFT) | LDML_ELEM_FLAGS_PREBASE); // tertiary -12, primary 80 std::cout << "es flags" << std::hex << es.get_flags() << std::dec << std::endl; // verify element metadata - ASSERT_EQ(es.is_uset(), false); + ASSERT_FALSE(es.is_uset()); ASSERT_EQ(es.get_order(), 0x50); ASSERT_EQ(es.get_tertiary(), 0); - ASSERT_EQ(es.is_prebase(), true); - ASSERT_EQ(es.is_tertiary_base(), false); + ASSERT_TRUE(es.is_prebase()); + ASSERT_FALSE(es.is_tertiary_base()); // verify element matching - ASSERT_EQ(es.matches(U'a'), true); - ASSERT_EQ(es.matches(U'b'), false); + ASSERT_TRUE(es.matches(U'a')); + ASSERT_FALSE(es.matches(U'b')); element eu(toneMarks, 0x37F40000); // element metadata std::cout << "eu flags" << std::hex << eu.get_flags() << std::dec << std::endl; - ASSERT_EQ(eu.is_uset(), true); + ASSERT_TRUE(eu.is_uset()); std::cout << "order" << (int)eu.get_order() << std::endl; ASSERT_EQ(eu.get_order(), -12); ASSERT_EQ(eu.get_tertiary(), 55); - ASSERT_EQ(eu.is_prebase(), false); - ASSERT_EQ(eu.is_tertiary_base(), false); + ASSERT_FALSE(eu.is_prebase()); + ASSERT_FALSE(eu.is_tertiary_base()); // element matching - ASSERT_EQ(eu.matches(U'a'), false); - ASSERT_EQ(eu.matches(U'\u1A76'), true); - ASSERT_EQ(eu.matches(U'\u1A75'), true); + ASSERT_FALSE(eu.matches(U'a')); + ASSERT_TRUE(eu.matches(U'\u1A76')); + ASSERT_TRUE(eu.matches(U'\u1A75')); element_list l; // '[tones]a' l.emplace_back(es); From 5da3b4effcab3f4ee6bbfdc199afb4a7a39fc370 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 6 May 2026 14:19:05 +0200 Subject: [PATCH 2/8] chore(core): cleanup transforms.tests.cpp 1. Rename various shorthand variables 2. Split simple tests into separate Google Test cases (have not tackled the more complex tests which have loops and/or setup) 3. Remove redundant logging from simple tests Relates-to: #15913 --- core/tests/unit/ldml/transforms.tests.cpp | 1209 ++++++++++----------- 1 file changed, 601 insertions(+), 608 deletions(-) diff --git a/core/tests/unit/ldml/transforms.tests.cpp b/core/tests/unit/ldml/transforms.tests.cpp index a51070e743..4dbdcb73e8 100644 --- a/core/tests/unit/ldml/transforms.tests.cpp +++ b/core/tests/unit/ldml/transforms.tests.cpp @@ -69,55 +69,55 @@ assert_marker_map_equal(const marker_map actual, const marker_map expected) { TEST(TransformsTests, TestTransformsSimple) { // start with one - transform_entry te(std::u32string(U"e\\^"), std::u32string(U"E")); // keep it simple + transform_entry trans_entry(std::u32string(U"e\\^"), std::u32string(U"E")); // keep it simple // OK now make a group do it - transforms tr(false); - transform_group st; + transforms trans(false); + transform_group trans_group; - st.push_back(te); + trans_group.push_back(trans_entry); - tr.addGroup(st); + trans.addGroup(trans_group); // see if we can match the same { std::u32string src(U"barQ^"); - bool res = tr.apply(src); + bool res = trans.apply(src); EXPECT_FALSE(res); EXPECT_EQ(src, std::u32string(U"barQ^")); // no change } { std::u32string src(U"fooe^"); - bool res = tr.apply(src); + bool res = trans.apply(src); EXPECT_TRUE(res); EXPECT_EQ(src, std::u32string(U"fooE")); } } TEST(TransformsTests, TestTransformsMoreComplex) { - transforms tr(false); + transforms trans(false); // setup { - transform_group st; - st.emplace_back(std::u32string(U"za"), std::u32string(U"c")); - st.emplace_back(std::u32string(U"a"), std::u32string(U"bb")); - tr.addGroup(st); + transform_group trans_group; + trans_group.emplace_back(std::u32string(U"za"), std::u32string(U"c")); + trans_group.emplace_back(std::u32string(U"a"), std::u32string(U"bb")); + trans.addGroup(trans_group); } { - transform_group st; - st.emplace_back(std::u32string(U"bb"), std::u32string(U"ccc")); - tr.addGroup(st); + transform_group trans_group; + trans_group.emplace_back(std::u32string(U"bb"), std::u32string(U"ccc")); + trans.addGroup(trans_group); } { - transform_group st; - st.emplace_back(std::u32string(U"cc"), std::u32string(U"d")); - tr.addGroup(st); + transform_group trans_group; + trans_group.emplace_back(std::u32string(U"cc"), std::u32string(U"d")); + trans.addGroup(trans_group); } { - transform_group st; - st.emplace_back(std::u32string(U"tcd"), std::u32string(U"e")); - tr.addGroup(st); + transform_group trans_group; + trans_group.emplace_back(std::u32string(U"tcd"), std::u32string(U"e")); + trans.addGroup(trans_group); } // now test @@ -125,7 +125,7 @@ TEST(TransformsTests, TestTransformsMoreComplex) { // see if we can match the same { std::u32string src(U"ta"); - bool res = tr.apply(src); + bool res = trans.apply(src); // pipe (|) symbol shows where the 'output' is delineated // t|a --> t|bb --> t|ccc --> t|cd --> |e EXPECT_EQ(src, std::u32string(U"e")); @@ -133,7 +133,7 @@ TEST(TransformsTests, TestTransformsMoreComplex) { } { std::u32string src(U"qza"); - bool res = tr.apply(src); + bool res = trans.apply(src); // pipe (|) symbol shows where the 'output' is delineated // q|za -> q|c EXPECT_EQ(src, std::u32string(U"qc")); @@ -141,28 +141,28 @@ TEST(TransformsTests, TestTransformsMoreComplex) { } { std::u32string src(U"qa"); - bool res = tr.apply(src); + bool res = trans.apply(src); EXPECT_EQ(src, std::u32string(U"qcd")); EXPECT_TRUE(res); } { std::u32string src(U"tb"); - bool res = tr.apply(src); + bool res = trans.apply(src); EXPECT_EQ(src, std::u32string(U"tb")); EXPECT_FALSE(res); } } TEST(TransformsTests, TestHindiExample) { - transforms tr(false); + transforms trans(false); { - transform_group st; - st.emplace_back(std::u32string(U"िह"), std::u32string(U"हि")); - tr.addGroup(st); + transform_group trans_group; + trans_group.emplace_back(std::u32string(U"िह"), std::u32string(U"हि")); + trans.addGroup(trans_group); } { std::u32string src(U"िह"); - bool res = tr.apply(src); + bool res = trans.apply(src); EXPECT_EQ(src, std::u32string(U"हि")); EXPECT_TRUE(res); } @@ -170,17 +170,17 @@ TEST(TransformsTests, TestHindiExample) { TEST(TransformsTests, TestReorderStandaloneElementApi) { // element API test - not a real element, just here for testing - element es(U'a', 0xF4500000 | LDML_ELEM_FLAGS_PREBASE | LDML_ELEM_FLAGS_TERTIARY_BASE); // tertiary -12, primary 80 - std::cout << "es flags" << std::hex << es.get_flags() << std::dec << std::endl; + element elem_char(U'a', 0xF4500000 | LDML_ELEM_FLAGS_PREBASE | LDML_ELEM_FLAGS_TERTIARY_BASE); // tertiary -12, primary 80 + std::cout << "elem_char flags" << std::hex << elem_char.get_flags() << std::dec << std::endl; // verify element metadata - ASSERT_FALSE(es.is_uset()); - ASSERT_EQ(es.get_order(), 0x50); - ASSERT_EQ(es.get_tertiary(), -12); - ASSERT_TRUE(es.is_prebase()); - ASSERT_TRUE(es.is_tertiary_base()); + ASSERT_FALSE(elem_char.is_uset()); + ASSERT_EQ(elem_char.get_order(), 0x50); + ASSERT_EQ(elem_char.get_tertiary(), -12); + ASSERT_TRUE(elem_char.is_prebase()); + ASSERT_TRUE(elem_char.is_tertiary_base()); // verify element matching - ASSERT_TRUE(es.matches(U'a')); - ASSERT_FALSE(es.matches(U'b')); + ASSERT_TRUE(elem_char.matches(U'a')); + ASSERT_FALSE(elem_char.matches(U'b')); } TEST(TransformsTests, TestReordersWithStandaloneNodLanaExample) { @@ -203,38 +203,37 @@ TEST(TransformsTests, TestReordersWithStandaloneNodLanaExample) { ASSERT_TRUE(toneMarks.contains(0x1A76)); ASSERT_FALSE(toneMarks.contains(0x1A60)); - std::cout << __FILE__ << ":" << __LINE__ << " - element API test " << std::endl; // element test { - element es(U'a', (80 << LDML_ELEM_FLAGS_ORDER_BITSHIFT) | LDML_ELEM_FLAGS_PREBASE); // tertiary -12, primary 80 - std::cout << "es flags" << std::hex << es.get_flags() << std::dec << std::endl; + element elem_char(U'a', (80 << LDML_ELEM_FLAGS_ORDER_BITSHIFT) | LDML_ELEM_FLAGS_PREBASE); // tertiary -12, primary 80 + std::cout << "elem_char flags" << std::hex << elem_char.get_flags() << std::dec << std::endl; // verify element metadata - ASSERT_FALSE(es.is_uset()); - ASSERT_EQ(es.get_order(), 0x50); - ASSERT_EQ(es.get_tertiary(), 0); - ASSERT_TRUE(es.is_prebase()); - ASSERT_FALSE(es.is_tertiary_base()); + ASSERT_FALSE(elem_char.is_uset()); + ASSERT_EQ(elem_char.get_order(), 0x50); + ASSERT_EQ(elem_char.get_tertiary(), 0); + ASSERT_TRUE(elem_char.is_prebase()); + ASSERT_FALSE(elem_char.is_tertiary_base()); // verify element matching - ASSERT_TRUE(es.matches(U'a')); - ASSERT_FALSE(es.matches(U'b')); + ASSERT_TRUE(elem_char.matches(U'a')); + ASSERT_FALSE(elem_char.matches(U'b')); - element eu(toneMarks, 0x37F40000); + element elem_uset(toneMarks, 0x37F40000); // element metadata - std::cout << "eu flags" << std::hex << eu.get_flags() << std::dec << std::endl; - ASSERT_TRUE(eu.is_uset()); - std::cout << "order" << (int)eu.get_order() << std::endl; - ASSERT_EQ(eu.get_order(), -12); - ASSERT_EQ(eu.get_tertiary(), 55); - ASSERT_FALSE(eu.is_prebase()); - ASSERT_FALSE(eu.is_tertiary_base()); + std::cout << "elem_uset flags" << std::hex << elem_uset.get_flags() << std::dec << std::endl; + ASSERT_TRUE(elem_uset.is_uset()); + std::cout << "order" << (int)elem_uset.get_order() << std::endl; + ASSERT_EQ(elem_uset.get_order(), -12); + ASSERT_EQ(elem_uset.get_tertiary(), 55); + ASSERT_FALSE(elem_uset.is_prebase()); + ASSERT_FALSE(elem_uset.is_tertiary_base()); // element matching - ASSERT_FALSE(eu.matches(U'a')); - ASSERT_TRUE(eu.matches(U'\u1A76')); - ASSERT_TRUE(eu.matches(U'\u1A75')); + ASSERT_FALSE(elem_uset.matches(U'a')); + ASSERT_TRUE(elem_uset.matches(U'\u1A76')); + ASSERT_TRUE(elem_uset.matches(U'\u1A75')); element_list l; // '[tones]a' - l.emplace_back(es); - l.emplace_back(eu); + l.emplace_back(elem_char); + l.emplace_back(elem_uset); std::cout << __FILE__ << ":" << __LINE__ << " - list test " << std::endl; ASSERT_EQ(l.match_end(U"asdfasdf"), 0); // no match @@ -314,7 +313,7 @@ TEST(TransformsTests, TestReordersWithStandaloneNodLanaExample) { ASSERT_EQ(sorted, expect); } std::cout << "now prepare the reorder elements" << std::endl; - transforms tr(false); + transforms trans(false); { reorder_group rg; @@ -359,7 +358,7 @@ TEST(TransformsTests, TestReordersWithStandaloneNodLanaExample) { e5before.emplace_back(U'\u1A6B', 0); rg.list.emplace_back(e5, e5before); - tr.addGroup(rg); + trans.addGroup(rg); } // now actually test it @@ -373,7 +372,7 @@ TEST(TransformsTests, TestReordersWithStandaloneNodLanaExample) { std::cout << "- try apply(text, output)" << std::endl; std::u32string text = roast; std::u32string output; - size_t len = tr.apply(text, output); + size_t len = trans.apply(text, output); if (len == 0) { std::cout << " (did not apply)" << std::endl; } else { @@ -388,7 +387,7 @@ TEST(TransformsTests, TestReordersWithStandaloneNodLanaExample) { { std::cout << "- try apply(text)" << std::endl; std::u32string text = roast; - if (!tr.apply(text)) { + if (!trans.apply(text)) { std::cout << " (did not apply)" << std::endl; } else if (text == roast) { std::cout << " (suboptimal: apply returned true but made no change)" << std::endl; @@ -406,7 +405,7 @@ TEST(TransformsTests, TestReordersWithStandaloneNodLanaExample) { // append the string text.append(1, *ch); std::cout << "-: " << text << std::endl; - if (!tr.apply(text)) { + if (!trans.apply(text)) { std::cout << " (did not apply)" << std::endl; } } @@ -422,14 +421,14 @@ TEST(TransformsTests, TestReordersWithStandaloneNodLanaExample) { const std::u32string expect = U"\u1A21\u1A60\u1A45"; // this string shouldn't mutate at all. { std::u32string text = expect; - tr.apply(text); + trans.apply(text); ASSERT_EQ(text, expect); } { // try submatch std::u32string text = expect; std::u32string output; - size_t len = tr.apply(text, output); + size_t len = trans.apply(text, output); ASSERT_EQ(output, U""); ASSERT_EQ(len, 0); } @@ -442,7 +441,7 @@ TEST(TransformsTests, TestReorderEsk) { // rules are a little bit simplified, having only the vowel 'a' std::cout << "now prepare the reorder elements" << std::endl; - transforms tr(false); + transforms trans(false); { reorder_group rg; @@ -486,7 +485,7 @@ TEST(TransformsTests, TestReorderEsk) { rg.list.emplace_back(e); } - tr.addGroup(rg); + trans.addGroup(rg); } // now actually test it @@ -510,7 +509,7 @@ TEST(TransformsTests, TestReorderEsk) { std::cout << "- try apply(text, output)" << std::endl; std::u32string text = orig; std::u32string output; - size_t len = tr.apply(text, output); + size_t len = trans.apply(text, output); if (len == 0) { std::cout << " (did not apply)" << std::endl; } else { @@ -525,7 +524,7 @@ TEST(TransformsTests, TestReorderEsk) { { std::cout << "- try apply(text)" << std::endl; std::u32string text = orig; - if (!tr.apply(text)) { + if (!trans.apply(text)) { std::cout << " (did not apply)" << std::endl; } else if (text == orig) { std::cout << " (suboptimal: apply returned true but made no change)" << std::endl; @@ -543,7 +542,7 @@ TEST(TransformsTests, TestReorderEsk) { // append the string text.append(1, *ch); std::cout << "-: " << text << std::endl; - if (!tr.apply(text)) { + if (!trans.apply(text)) { std::cout << " (did not apply)" << std::endl; } } @@ -569,114 +568,114 @@ TEST(TransformsTests, TestMap) { ASSERT_EQ(km::core::util::km_regex::findIndex(U"Nowhere", list), -1); } -TEST(TransformsTests, TestStrutils) { - { - std::cout << __FILE__ << ":" << __LINE__ << " - basic test0" << std::endl; - const std::u32string src = U"abc"; - const std::u32string dst = remove_markers(src); - ASSERT_EQ(dst, src); // unchanged - } - { - marker_map map; - std::cout << __FILE__ << ":" << __LINE__ << " - basic test" << std::endl; - const std::u32string src = U"abc"; - const std::u32string dst = remove_markers(src, map); - ASSERT_EQ(dst, src); // unchanged - ASSERT_EQ(count_markers(map), 0); - } - { - marker_map map; - std::cout << __FILE__ << ":" << __LINE__ << " - marker test" << std::endl; - const std::u32string src = U"6\U0000ffff\U00000008\U00000001e"; - const std::u32string dst = remove_markers(src, map); - const std::u32string expect = U"6e"; - ASSERT_EQ(dst, expect); - marker_map expm = {{U'e', 0x1L}}; - assert_marker_map_equal(map, expm); // marker 1 @ e - ASSERT_EQ(count_markers(map), 1); - } - { - marker_map map; - std::cout << __FILE__ << ":" << __LINE__ << " - bad0" << std::endl; - const std::u32string src = U"6\U0000ffff\U00000008"; // missing trailing marker # - const std::u32string dst = remove_markers(src, map); - const std::u32string expect = src; - ASSERT_EQ(dst, expect); - ASSERT_EQ(count_markers(map), 0); - } - { - marker_map map; - std::cout << __FILE__ << ":" << __LINE__ << " - bad1" << std::endl; - const std::u32string src = U"6\U0000ffffq"; // missing sentinel subtype - const std::u32string dst = remove_markers(src, map); - const std::u32string expect = src; // 'q' removed - ASSERT_EQ(dst, expect); - ASSERT_EQ(count_markers(map), 0); - } - { - marker_map map; - std::cout << __FILE__ << ":" << __LINE__ << " - bad1b" << std::endl; - const std::u32string src = U"6\U0000ffff"; // missing code - const std::u32string dst = remove_markers(src, map); - const std::u32string expect = src; - ASSERT_EQ(dst, expect); - ASSERT_EQ(count_markers(map), 0); - } - { - marker_map map; - std::cout << __FILE__ << ":" << __LINE__ << " - bad1c" << std::endl; - const std::u32string src = U"6\U0000ffffzz"; // missing code - const std::u32string dst = remove_markers(src, map); - const std::u32string expect = src; - ASSERT_EQ(dst, expect); - ASSERT_EQ(count_markers(map), 0); - } - { - marker_map map; - std::cout << __FILE__ << ":" << __LINE__ << " - marker end test" << std::endl; - const std::u32string src = U"6\U0000ffff\U00000008\U00000001"; - const std::u32string dst = remove_markers(src, map); - const std::u32string expect = U"6"; - ASSERT_EQ(dst, expect); - marker_map expm({{MARKER_BEFORE_EOT, 0x1L}}); - assert_marker_map_equal(map, expm); - ASSERT_EQ(count_markers(map), 1); - } - { - marker_map map; - std::cout << __FILE__ << ":" << __LINE__ << " - complex test" << std::endl; - const std::u32string src = - U"6\U0000ffff\U00000008\U00000001e\U0000ffff\U00000008\U00000002\U00000320\U0000ffff\U00000008\U00000003\U00000300" - U"\U0000ffff\U00000008\U00000004"; - const std::u32string dst = remove_markers(src, map); - const std::u32string expect = U"6e\U00000320\U00000300"; - ASSERT_EQ(dst, expect); - marker_map expm({{U'e', 0x1L}, {0x0320, 0x2L}, {0x0300, 0x3L}, {MARKER_BEFORE_EOT, 0x4L}}); - assert_marker_map_equal(map, expm); - ASSERT_EQ(count_markers(map), 4); - } +// Strutils + +TEST(TransformsTests, TestStrutilsBasicTest0) { + const std::u32string src = U"abc"; + const std::u32string dst = remove_markers(src); + ASSERT_EQ(dst, src); // unchanged } -TEST(TransformsTests, TestStrutilsPrependHexQuad) { - { - std::u32string dst; - prepend_hex_quad(dst, 0x0001); - ASSERT_EQ(dst, U"0001"); - } - { - std::u32string dst; - prepend_hex_quad(dst, 0xCAFE); - ASSERT_EQ(dst, U"CAFE"); - } - { - std::u32string dst; - prepend_hex_quad(dst, 0xFFFF); - ASSERT_EQ(dst, U"FFFF"); - } +TEST(TransformsTests, TestStrutilsBasicTest) { + marker_map map; + const std::u32string src = U"abc"; + const std::u32string dst = remove_markers(src, map); + ASSERT_EQ(dst, src); // unchanged + ASSERT_EQ(count_markers(map), 0); +} + +TEST(TransformsTests, TestStrutilsMarkerTest) { + marker_map map; + const std::u32string src = U"6\U0000ffff\U00000008\U00000001e"; + const std::u32string dst = remove_markers(src, map); + const std::u32string expect = U"6e"; + ASSERT_EQ(dst, expect); + marker_map expm = {{U'e', 0x1L}}; + assert_marker_map_equal(map, expm); // marker 1 @ e + ASSERT_EQ(count_markers(map), 1); +} + +TEST(TransformsTests, TestStrutilsBad0) { + marker_map map; + const std::u32string src = U"6\U0000ffff\U00000008"; // missing trailing marker # + const std::u32string dst = remove_markers(src, map); + const std::u32string expect = src; + ASSERT_EQ(dst, expect); + ASSERT_EQ(count_markers(map), 0); +} + +TEST(TransformsTests, TestStrutilsBad1) { + marker_map map; + const std::u32string src = U"6\U0000ffffq"; // missing sentinel subtype + const std::u32string dst = remove_markers(src, map); + const std::u32string expect = src; // 'q' removed + ASSERT_EQ(dst, expect); + ASSERT_EQ(count_markers(map), 0); +} + +TEST(TransformsTests, TestStrutilsBad1B) { + marker_map map; + const std::u32string src = U"6\U0000ffff"; // missing code + const std::u32string dst = remove_markers(src, map); + const std::u32string expect = src; + ASSERT_EQ(dst, expect); + ASSERT_EQ(count_markers(map), 0); +} + +TEST(TransformsTests, TestStrutilsBad1C) { + marker_map map; + const std::u32string src = U"6\U0000ffffzz"; // missing code + const std::u32string dst = remove_markers(src, map); + const std::u32string expect = src; + ASSERT_EQ(dst, expect); + ASSERT_EQ(count_markers(map), 0); +} + +TEST(TransformsTests, TestStrutilsMarkerEnd) { + marker_map map; + const std::u32string src = U"6\U0000ffff\U00000008\U00000001"; + const std::u32string dst = remove_markers(src, map); + const std::u32string expect = U"6"; + ASSERT_EQ(dst, expect); + marker_map expm({{MARKER_BEFORE_EOT, 0x1L}}); + assert_marker_map_equal(map, expm); + ASSERT_EQ(count_markers(map), 1); +} + +TEST(TransformsTests, TestStrutilsComplex) { + marker_map map; + const std::u32string src = + U"6\U0000ffff\U00000008\U00000001e\U0000ffff\U00000008\U00000002\U00000320\U0000ffff\U00000008\U00000003\U00000300" + U"\U0000ffff\U00000008\U00000004"; + const std::u32string dst = remove_markers(src, map); + const std::u32string expect = U"6e\U00000320\U00000300"; + ASSERT_EQ(dst, expect); + marker_map expm({{U'e', 0x1L}, {0x0320, 0x2L}, {0x0300, 0x3L}, {MARKER_BEFORE_EOT, 0x4L}}); + assert_marker_map_equal(map, expm); + ASSERT_EQ(count_markers(map), 4); +} + +// PrependHex + +TEST(TransformsTests, TestStrutilsPrependHexQuad1) { + std::u32string dst; + prepend_hex_quad(dst, 0x0001); + ASSERT_EQ(dst, U"0001"); +} + +TEST(TransformsTests, TestStrutilsPrependHexQuadCAFE) { + std::u32string dst; + prepend_hex_quad(dst, 0xCAFE); + ASSERT_EQ(dst, U"CAFE"); +} + +TEST(TransformsTests, TestStrutilsPrependHexQuadFFFF) { + std::u32string dst; + prepend_hex_quad(dst, 0xFFFF); + ASSERT_EQ(dst, U"FFFF"); } TEST(TransformsTests, TestStrutilsParseHexQuad) { - std::cout << __FILE__ << ":" << __LINE__ << " - parse hex quad" << std::endl; ASSERT_EQ(parse_hex_quad(U"0001"), 0x0001); ASSERT_EQ(parse_hex_quad(U"CAFE"), 0xCAFE); ASSERT_EQ(parse_hex_quad(U"D00d"), 0xD00D); @@ -684,462 +683,456 @@ TEST(TransformsTests, TestStrutilsParseHexQuad) { ASSERT_EQ(parse_hex_quad(U"zzzz"), 0); // err } -TEST(TransformsTests, TestNormalize) { - { - marker_map map; - std::cout << __FILE__ << ":" << __LINE__ << " - noop test" << std::endl; - const std::u32string src = U"6e\U00000320\U00000300"; // already NFD - const std::u32string expect = src; - std::u32string dst = src; - ASSERT_TRUE(normalize_nfd_markers_segment(dst, map)); - ASSERT_EQ(dst, expect); - ASSERT_EQ(count_markers(map), 0); - } - { - marker_map map; - std::cout << __FILE__ << ":" << __LINE__ << " - medium test" << std::endl; - const std::u32string src = U"6e\U00000300\U00000320"; // swapped - const std::u32string expect = U"6e\U00000320\U00000300"; // correct NFD - std::u32string dst = src; - ASSERT_TRUE(normalize_nfd_markers_segment(dst, map)); - ASSERT_EQ(dst, expect); - ASSERT_EQ(count_markers(map), 0); - } +TEST(TransformsTests, TestNormalizeNoop) { + marker_map map; + const std::u32string src = U"6e\U00000320\U00000300"; // already NFD + const std::u32string expect = src; + std::u32string dst = src; + ASSERT_TRUE(normalize_nfd_markers_segment(dst, map)); + ASSERT_EQ(dst, expect); + ASSERT_EQ(count_markers(map), 0); +} - { - marker_map map; - std::cout << __FILE__ << ":" << __LINE__ << " - noop test w markers" << std::endl; - const std::u32string src = - U"6\U0000ffff\U00000008\U00000001e\U0000ffff\U00000008\U00000002\U00000320\U0000ffff\U00000008\U00000003\U00000300" - U"\U0000ffff\U00000008\U00000004"; - const std::u32string expect = src; - std::u32string dst = src; - ASSERT_TRUE(normalize_nfd_markers_segment(dst, map)); - ASSERT_EQ(dst, expect); - marker_map expm({{U'e', 0x1L}, {0x320, 0x2L}, {0x300, 0x3L}, {MARKER_BEFORE_EOT, 0x4L}}); - assert_marker_map_equal(map, expm); - ASSERT_EQ(count_markers(map), 4); - } +TEST(TransformsTests, TestNormalizeMedium) { + marker_map map; + const std::u32string src = U"6e\U00000300\U00000320"; // swapped + const std::u32string expect = U"6e\U00000320\U00000300"; // correct NFD + std::u32string dst = src; + ASSERT_TRUE(normalize_nfd_markers_segment(dst, map)); + ASSERT_EQ(dst, expect); + ASSERT_EQ(count_markers(map), 0); +} - { - marker_map map; - std::cout << __FILE__ << ":" << __LINE__ << " - complex test" << std::endl; - const std::u32string src = // already in order: 320+300 - U"6\U0000ffff\U00000008\U00000001e\U0000ffff\U00000008\U00000002\U00000320\U0000ffff\U00000008\U00000003\U00000300" - U"\U0000ffff\U00000008\U00000004"; - const std::u32string expect = src; - std::u32string dst = src; - ASSERT_TRUE(normalize_nfd_markers_segment(dst, map)); - ASSERT_EQ(dst, expect); - marker_map expm({{U'e', 0x1L}, {0x320, 0x2L}, {0x300, 0x3L}, {MARKER_BEFORE_EOT, 0x4L}}); - assert_marker_map_equal(map, expm); - ASSERT_EQ(count_markers(map), 4); - } - { - marker_map map; - std::cout << __FILE__ << ":" << __LINE__ << " - complex test2" << std::endl; - const std::u32string src = // out of order, 300-320 - U"6\U0000ffff\U00000008\U00000001e\U0000ffff\U00000008\U00000002\U00000300\U0000ffff\U00000008\U00000003\U00000320" - U"\U0000ffff\U00000008\U00000004"; - const std::u32string expect = - U"6\U0000ffff\U00000008\U00000001e\U0000ffff\U00000008\U00000003\U00000320\U0000ffff\U00000008\U00000002\U00000300" - U"\U0000ffff\U00000008\U00000004"; - std::u32string dst = src; - ASSERT_TRUE(normalize_nfd_markers_segment(dst, map)); - if (dst != expect) { - std::cout << "dst: " << Debug_UnicodeString(dst) << std::endl; - std::cout << "exp: " << Debug_UnicodeString(expect) << std::endl; - } - ASSERT_EQ(dst, expect); - marker_map expm({{U'e', 0x1L}, {0x300, 0x2L}, {0x320, 0x3L}, {MARKER_BEFORE_EOT, 0x4L}}); - assert_marker_map_equal(map, expm); - ASSERT_EQ(count_markers(map), 4); - } +TEST(TransformsTests, TestNormalizeNoopWithMarkers) { + marker_map map; + const std::u32string src = + U"6\U0000ffff\U00000008\U00000001e\U0000ffff\U00000008\U00000002\U00000320\U0000ffff\U00000008\U00000003\U00000300" + U"\U0000ffff\U00000008\U00000004"; + const std::u32string expect = src; + std::u32string dst = src; + ASSERT_TRUE(normalize_nfd_markers_segment(dst, map)); + ASSERT_EQ(dst, expect); + marker_map expm({{U'e', 0x1L}, {0x320, 0x2L}, {0x300, 0x3L}, {MARKER_BEFORE_EOT, 0x4L}}); + assert_marker_map_equal(map, expm); + ASSERT_EQ(count_markers(map), 4); +} - { - // u"4è\U0000ffff\u0008\U00000001̠" - marker_map map; - std::cout << __FILE__ << ":" << __LINE__ << " - complex test 4a" << std::endl; - const std::u32string src = U"4e\u0300\uFFFF\u0008\u0001\u0320"; - const std::u32string expect = U"4e\uFFFF\u0008\u0001\u0320\u0300"; - std::u32string dst = src; - ASSERT_TRUE(normalize_nfd_markers_segment(dst, map)); - if (dst != expect) { - std::cout << "dst: " << Debug_UnicodeString(dst) << std::endl; - std::cout << "exp: " << Debug_UnicodeString(expect) << std::endl; - } - ASSERT_EQ(dst, expect); - marker_map expm({{0x320, 0x1L}}); - assert_marker_map_equal(map, expm); - ASSERT_EQ(count_markers(map), 1); - } +TEST(TransformsTests, TestNormalizeComplex) { + marker_map map; + const std::u32string src = // already in order: 320+300 + U"6\U0000ffff\U00000008\U00000001e\U0000ffff\U00000008\U00000002\U00000320\U0000ffff\U00000008\U00000003\U00000300" + U"\U0000ffff\U00000008\U00000004"; + const std::u32string expect = src; + std::u32string dst = src; + ASSERT_TRUE(normalize_nfd_markers_segment(dst, map)); + ASSERT_EQ(dst, expect); + marker_map expm({{U'e', 0x1L}, {0x320, 0x2L}, {0x300, 0x3L}, {MARKER_BEFORE_EOT, 0x4L}}); + assert_marker_map_equal(map, expm); + ASSERT_EQ(count_markers(map), 4); +} - { - // from tests - marker_map map; - std::cout << __FILE__ << ":" << __LINE__ << " - complex test 9c" << std::endl; - const std::u32string src = U"9ce\u0300\uFFFF\u0008\u0002\u0320\uFFFF\u0008\u0001"; - const std::u32string expect = U"9ce\uFFFF\u0008\u0002\u0320\u0300\uFFFF\u0008\u0001"; - std::u32string dst = src; - ASSERT_TRUE(normalize_nfd_markers_segment(dst, map)); - if (dst != expect) { - std::cout << "dst: " << Debug_UnicodeString(dst) << std::endl; - std::cout << "exp: " << Debug_UnicodeString(expect) << std::endl; - } - ASSERT_EQ(dst, expect); - marker_map expm({{0x320, 0x2L}, {MARKER_BEFORE_EOT, 0x1L}}); - assert_marker_map_equal(map, expm); - ASSERT_EQ(count_markers(map), 2); +TEST(TransformsTests, TestNormalizeComplex2) { + marker_map map; + const std::u32string src = // out of order, 300-320 + U"6\U0000ffff\U00000008\U00000001e\U0000ffff\U00000008\U00000002\U00000300\U0000ffff\U00000008\U00000003\U00000320" + U"\U0000ffff\U00000008\U00000004"; + const std::u32string expect = + U"6\U0000ffff\U00000008\U00000001e\U0000ffff\U00000008\U00000003\U00000320\U0000ffff\U00000008\U00000002\U00000300" + U"\U0000ffff\U00000008\U00000004"; + std::u32string dst = src; + ASSERT_TRUE(normalize_nfd_markers_segment(dst, map)); + if (dst != expect) { + std::cout << "dst: " << Debug_UnicodeString(dst) << std::endl; + std::cout << "exp: " << Debug_UnicodeString(expect) << std::endl; } + ASSERT_EQ(dst, expect); + marker_map expm({{U'e', 0x1L}, {0x300, 0x2L}, {0x320, 0x3L}, {MARKER_BEFORE_EOT, 0x4L}}); + assert_marker_map_equal(map, expm); + ASSERT_EQ(count_markers(map), 4); +} - { - // from tests - regex edition - marker_map map; - std::cout << __FILE__ << ":" << __LINE__ << " - complex test 9c+regex" << std::endl; - const std::u32string src = U"9ce\u0300\\uffff\\u0008\\u0002\u0320\\uffff\\u0008\\u0001"; - const std::u32string expect = U"9ce\\uffff\\u0008\\u0002\u0320\u0300\\uffff\\u0008\\u0001"; - std::u32string dst = src; - ASSERT_TRUE(normalize_nfd_markers_segment(dst, map, regex_sentinel)); - if (dst != expect) { - std::cout << "dst: " << Debug_UnicodeString(dst) << std::endl; - std::cout << " " << dst << std::endl; - std::cout << "exp: " << Debug_UnicodeString(expect) << std::endl; - std::cout << " " << expect << std::endl; - } - ASSERT_EQ(dst, expect); - marker_map expm({{0x320, 0x2L}, {MARKER_BEFORE_EOT, 0x1L}}); - assert_marker_map_equal(map, expm); - ASSERT_EQ(count_markers(map), 2); - } - { - // from tests - regex edition - marker_map map; - std::cout << __FILE__ << ":" << __LINE__ << " - complex test \\m{.}" << std::endl; - const std::u32string src = U"9ce\u0300\\uffff\\u0008[\\u0001-\\ud7fe]\u0320\\uffff\\u0008\\u0001"; - const std::u32string expect = U"9ce\\uffff\\u0008[\\u0001-\\ud7fe]\u0320\u0300\\uffff\\u0008\\u0001"; - std::u32string dst = src; - ASSERT_TRUE(normalize_nfd_markers_segment(dst, map, regex_sentinel)); - if (dst != expect) { - std::cout << "dst: " << Debug_UnicodeString(dst) << std::endl; - std::cout << "exp: " << Debug_UnicodeString(expect) << std::endl; - } - ASSERT_EQ(dst, expect); - marker_map expm({{0x320, LDML_MARKER_ANY_INDEX}, {MARKER_BEFORE_EOT, 0x1L}}); - assert_marker_map_equal(map, expm); - ASSERT_EQ(count_markers(map), 2); +TEST(TransformsTests, TestNormalizeComplex4A) { + // u"4è\U0000ffff\u0008\U00000001̠" + marker_map map; + const std::u32string src = U"4e\u0300\uFFFF\u0008\u0001\u0320"; + const std::u32string expect = U"4e\uFFFF\u0008\u0001\u0320\u0300"; + std::u32string dst = src; + ASSERT_TRUE(normalize_nfd_markers_segment(dst, map)); + if (dst != expect) { + std::cout << "dst: " << Debug_UnicodeString(dst) << std::endl; + std::cout << "exp: " << Debug_UnicodeString(expect) << std::endl; } + ASSERT_EQ(dst, expect); + marker_map expm({{0x320, 0x1L}}); + assert_marker_map_equal(map, expm); + ASSERT_EQ(count_markers(map), 1); +} - { - // from tests - marker_map map; - std::cout << __FILE__ << ":" << __LINE__ << " - complex test 10 stack o' 2x2" << std::endl; - const std::u32string src = U"9ce\u0300\uFFFF\u0008\u0002\uFFFF\u0008\u0002\u0320"; - const std::u32string expect = U"9ce\uFFFF\u0008\u0002\uFFFF\u0008\u0002\u0320\u0300"; - std::u32string dst = src; - ASSERT_TRUE(normalize_nfd_markers_segment(dst, map)); - if (dst != expect) { - std::cout << "dst: " << Debug_UnicodeString(dst) << std::endl; - std::cout << "exp: " << Debug_UnicodeString(expect) << std::endl; - } - ASSERT_EQ(dst, expect); - marker_map expm({{0x320, 0x2L}, {0x320, 0x2L}}); - assert_marker_map_equal(map, expm); - ASSERT_EQ(count_markers(map), 2); +TEST(TransformsTests, TestNormalizeComplex9C) { + // from tests + marker_map map; + const std::u32string src = U"9ce\u0300\uFFFF\u0008\u0002\u0320\uFFFF\u0008\u0001"; + const std::u32string expect = U"9ce\uFFFF\u0008\u0002\u0320\u0300\uFFFF\u0008\u0001"; + std::u32string dst = src; + ASSERT_TRUE(normalize_nfd_markers_segment(dst, map)); + if (dst != expect) { + std::cout << "dst: " << Debug_UnicodeString(dst) << std::endl; + std::cout << "exp: " << Debug_UnicodeString(expect) << std::endl; } + ASSERT_EQ(dst, expect); + marker_map expm({{0x320, 0x2L}, {MARKER_BEFORE_EOT, 0x1L}}); + assert_marker_map_equal(map, expm); + ASSERT_EQ(count_markers(map), 2); +} - { - // from tests - marker_map map; - std::cout << __FILE__ << ":" << __LINE__ << " - complex test 10 stack o' 2x1x2" << std::endl; - const std::u32string src = U"9ce\u0300\uFFFF\u0008\u0002\uFFFF\u0008\u0001\uFFFF\u0008\u0003\u0320"; - const std::u32string expect = U"9ce\uFFFF\u0008\u0002\uFFFF\u0008\u0001\uFFFF\u0008\u0003\u0320\u0300"; - std::u32string dst = src; - ASSERT_TRUE(normalize_nfd_markers_segment(dst, map)); - if (dst != expect) { - std::cout << "dst: " << Debug_UnicodeString(dst) << std::endl; - std::cout << "exp: " << Debug_UnicodeString(expect) << std::endl; - } - ASSERT_EQ(dst, expect); - marker_map expm({{0x320, 0x2L}, {0x320, 0x1L}, {0x320, 0x3L}}); - assert_marker_map_equal(map, expm); +TEST(TransformsTests, TestNormalizeComplex9CRegex) { + // from tests - regex edition + marker_map map; + const std::u32string src = U"9ce\u0300\\uffff\\u0008\\u0002\u0320\\uffff\\u0008\\u0001"; + const std::u32string expect = U"9ce\\uffff\\u0008\\u0002\u0320\u0300\\uffff\\u0008\\u0001"; + std::u32string dst = src; + ASSERT_TRUE(normalize_nfd_markers_segment(dst, map, regex_sentinel)); + if (dst != expect) { + std::cout << "dst: " << Debug_UnicodeString(dst) << std::endl; + std::cout << " " << dst << std::endl; + std::cout << "exp: " << Debug_UnicodeString(expect) << std::endl; + std::cout << " " << expect << std::endl; } + ASSERT_EQ(dst, expect); + marker_map expm({{0x320, 0x2L}, {MARKER_BEFORE_EOT, 0x1L}}); + assert_marker_map_equal(map, expm); + ASSERT_EQ(count_markers(map), 2); +} - { - marker_map map; - std::cout << __FILE__ << ":" << __LINE__ << " - dup-char test" << std::endl; - const std::u32string src = U"a\uFFFF\u0008\u0001\u0300e\uFFFF\u0008\u0002\u0300"; - const std::u32string dst = remove_markers(src, map); - const std::u32string expect = U"a\u0300e\u0300"; // U+0300 twice! This should be removed in 2 segments - ASSERT_EQ(dst, expect); - marker_map expm({{0x300, 0x1L}, {0x300, 0x2L}}); - assert_marker_map_equal(map, expm); +TEST(TransformsTests, TestNormalizeComplexMarkerAny) { + // from tests - regex edition + marker_map map; + const std::u32string src = U"9ce\u0300\\uffff\\u0008[\\u0001-\\ud7fe]\u0320\\uffff\\u0008\\u0001"; + const std::u32string expect = U"9ce\\uffff\\u0008[\\u0001-\\ud7fe]\u0320\u0300\\uffff\\u0008\\u0001"; + std::u32string dst = src; + ASSERT_TRUE(normalize_nfd_markers_segment(dst, map, regex_sentinel)); + if (dst != expect) { + std::cout << "dst: " << Debug_UnicodeString(dst) << std::endl; + std::cout << "exp: " << Debug_UnicodeString(expect) << std::endl; } - { - marker_map map; - std::cout << __FILE__ << ":" << __LINE__ << " - support 2-segment markers " << std::endl; - // e\m{1}`\m{2}_E\m{3}`\m{4}_ - const std::u32string src = - U"e\uFFFF\u0008\u0001\u0300\uFFFF\u0008\u0002\u0320E\uFFFF\u0008\u0003\u0300\uFFFF\u0008\u0004\u0320"; - // e\m{2}_\m{1}`E\m{4}_\m{3}` - const std::u32string expect_rem = - U"e\u0300\u0320E\u0300\u0320"; - const std::u32string expect_nfd = - U"e\uFFFF\u0008\u0002\u0320\uFFFF\u0008\u0001\u0300E\uFFFF\u0008\u0004\u0320\uFFFF\u0008\u0003\u0300"; - auto dst_rem = remove_markers(src, &map); // note: this is bigger than a single segment. so it is a degenerate test case. - marker_map expm({{0x300, 0x1L}, {0x320, 0x2L}, {0x300, 0x3L}, {0x320, 0x4L}}); - assert_marker_map_equal(map, expm); - ASSERT_EQ(dst_rem, expect_rem); - std::u32string dst_nfd = src; - ASSERT_TRUE(normalize_nfd_markers(dst_nfd)); - if (dst_nfd != expect_nfd) { - std::cout << "dst: " << Debug_UnicodeString(dst_nfd) << std::endl; - std::cout << "exp: " << Debug_UnicodeString(expect_nfd) << std::endl; - } - ASSERT_EQ(dst_nfd, expect_nfd); + ASSERT_EQ(dst, expect); + marker_map expm({{0x320, LDML_MARKER_ANY_INDEX}, {MARKER_BEFORE_EOT, 0x1L}}); + assert_marker_map_equal(map, expm); + ASSERT_EQ(count_markers(map), 2); +} + +TEST(TransformsTests, TestNormalizeComplex10StackOf2x2) { + // from tests + marker_map map; + const std::u32string src = U"9ce\u0300\uFFFF\u0008\u0002\uFFFF\u0008\u0002\u0320"; + const std::u32string expect = U"9ce\uFFFF\u0008\u0002\uFFFF\u0008\u0002\u0320\u0300"; + std::u32string dst = src; + ASSERT_TRUE(normalize_nfd_markers_segment(dst, map)); + if (dst != expect) { + std::cout << "dst: " << Debug_UnicodeString(dst) << std::endl; + std::cout << "exp: " << Debug_UnicodeString(expect) << std::endl; + } + ASSERT_EQ(dst, expect); + marker_map expm({{0x320, 0x2L}, {0x320, 0x2L}}); + assert_marker_map_equal(map, expm); + ASSERT_EQ(count_markers(map), 2); +} + +TEST(TransformsTests, TestNormalizeComplex10StackOf2x1x2) { + // from tests + marker_map map; + const std::u32string src = U"9ce\u0300\uFFFF\u0008\u0002\uFFFF\u0008\u0001\uFFFF\u0008\u0003\u0320"; + const std::u32string expect = U"9ce\uFFFF\u0008\u0002\uFFFF\u0008\u0001\uFFFF\u0008\u0003\u0320\u0300"; + std::u32string dst = src; + ASSERT_TRUE(normalize_nfd_markers_segment(dst, map)); + if (dst != expect) { + std::cout << "dst: " << Debug_UnicodeString(dst) << std::endl; + std::cout << "exp: " << Debug_UnicodeString(expect) << std::endl; + } + ASSERT_EQ(dst, expect); + marker_map expm({{0x320, 0x2L}, {0x320, 0x1L}, {0x320, 0x3L}}); + assert_marker_map_equal(map, expm); +} + +TEST(TransformsTests, TestNormalizeDupChar) { + marker_map map; + const std::u32string src = U"a\uFFFF\u0008\u0001\u0300e\uFFFF\u0008\u0002\u0300"; + const std::u32string dst = remove_markers(src, map); + const std::u32string expect = U"a\u0300e\u0300"; // U+0300 twice! This should be removed in 2 segments + ASSERT_EQ(dst, expect); + marker_map expm({{0x300, 0x1L}, {0x300, 0x2L}}); + assert_marker_map_equal(map, expm); +} + +TEST(TransformsTests, TestNormalizeTwoSegmentMarkers) { + marker_map map; + // e\m{1}`\m{2}_E\m{3}`\m{4}_ + const std::u32string src = + U"e\uFFFF\u0008\u0001\u0300\uFFFF\u0008\u0002\u0320E\uFFFF\u0008\u0003\u0300\uFFFF\u0008\u0004\u0320"; + // e\m{2}_\m{1}`E\m{4}_\m{3}` + const std::u32string expect_rem = + U"e\u0300\u0320E\u0300\u0320"; + const std::u32string expect_nfd = + U"e\uFFFF\u0008\u0002\u0320\uFFFF\u0008\u0001\u0300E\uFFFF\u0008\u0004\u0320\uFFFF\u0008\u0003\u0300"; + auto dst_rem = remove_markers(src, &map); // note: this is bigger than a single segment. so it is a degenerate test case. + marker_map expm({{0x300, 0x1L}, {0x320, 0x2L}, {0x300, 0x3L}, {0x320, 0x4L}}); + assert_marker_map_equal(map, expm); + ASSERT_EQ(dst_rem, expect_rem); + std::u32string dst_nfd = src; + ASSERT_TRUE(normalize_nfd_markers(dst_nfd)); + if (dst_nfd != expect_nfd) { + std::cout << "dst: " << Debug_UnicodeString(dst_nfd) << std::endl; + std::cout << "exp: " << Debug_UnicodeString(expect_nfd) << std::endl; } - { - marker_map map; - std::cout << __FILE__ << ":" << __LINE__ << " - marker-before-NFC " << std::endl; - // KA \m O -> KA \m E AA - const std::u32string src = U"\u0995\uFFFF\u0008\u0001\u09CB"; - const std::u32string expect_rem = U"\u0995\u09CB"; - const std::u32string expect_nfd = U"\u0995\uFFFF\u0008\u0001\u09C7\u09BE"; - auto dst_rem = remove_markers(src, &map); - marker_map expm({{0x09C7, 0x1L}}); - ASSERT_EQ(dst_rem, expect_rem); - std::u32string dst_nfd = src; - ASSERT_TRUE(normalize_nfd_markers(dst_nfd)); - if (dst_nfd != expect_nfd) { - std::cout << "dst: " << Debug_UnicodeString(dst_nfd) << std::endl; - std::cout << "exp: " << Debug_UnicodeString(expect_nfd) << std::endl; - } ASSERT_EQ(dst_nfd, expect_nfd); - assert_marker_map_equal(map, expm); - } - { - marker_map map; - std::cout << __FILE__ << ":" << __LINE__ << " - marker-before-NFC " << std::endl; - const std::u32string src = U"\u0995\u09BE\uFFFF\u0008\u0001\u09C7"; - const std::u32string expect_rem = U"\u0995\u09BE\u09C7"; - const std::u32string expect_nfd = src; // does not get reordered - auto dst_rem = remove_markers(src, &map); - marker_map expm({{0x09C7, 0x1L}}); - ASSERT_EQ(dst_rem, expect_rem); - std::u32string dst_nfd = src; - ASSERT_TRUE(normalize_nfd_markers(dst_nfd)); - if (dst_nfd != expect_nfd) { - std::cout << "dst: " << Debug_UnicodeString(dst_nfd) << std::endl; - std::cout << "exp: " << Debug_UnicodeString(expect_nfd) << std::endl; - } - ASSERT_EQ(dst_nfd, expect_nfd); - assert_marker_map_equal(map, expm); - } - { - marker_map map; - std::cout << __FILE__ << ":" << __LINE__ << " - marker-before-NFC " << std::endl; - const std::u32string src = U"\u0995\u09BE\uFFFF\u0008\u0001\u09C7"; - const std::u32string expect_rem = U"\u0995\u09BE\u09C7"; - const std::u32string expect_nfd = src; // does not get reordered - auto dst_rem = remove_markers(src, &map); - marker_map expm({{0x09C7, 0x1L}}); - ASSERT_EQ(dst_rem, expect_rem); - std::u32string dst_nfd = src; - ASSERT_TRUE(normalize_nfd_markers(dst_nfd)); - if (dst_nfd != expect_nfd) { - std::cout << "dst: " << Debug_UnicodeString(dst_nfd) << std::endl; - std::cout << "exp: " << Debug_UnicodeString(expect_nfd) << std::endl; - } - ASSERT_EQ(dst_nfd, expect_nfd); - assert_marker_map_equal(map, expm); - } - { - marker_map map; - std::cout << __FILE__ << ":" << __LINE__ << " - marker-before-greek " << std::endl; - const std::u32string src = U"\u03B5\uFFFF\u0008\u0001\u0344"; - const std::u32string expect_rem = U"\u03B5\u0344"; - const std::u32string expect_nfd = U"\u03B5\uFFFF\u0008\u0001\u0308\u0301"; - auto dst_rem = remove_markers(src, &map); - marker_map expm({{0x0308, 0x1L}}); - ASSERT_EQ(dst_rem, expect_rem); - std::u32string dst_nfd = src; - ASSERT_TRUE(normalize_nfd_markers(dst_nfd)); - if (dst_nfd != expect_nfd) { - std::cout << "dst: " << Debug_UnicodeString(dst_nfd) << std::endl; - std::cout << "exp: " << Debug_UnicodeString(expect_nfd) << std::endl; - } - ASSERT_EQ(dst_nfd, expect_nfd); - assert_marker_map_equal(map, expm); - } +} - // doubled markers tests - { - marker_map map; - std::cout << __FILE__ << ":" << __LINE__ << " - doubled marker1 " << std::endl; - const std::u32string src = U"e\uffff\u0008\u0001\u0300\u0320\u0300"; - const std::u32string expect_rem = U"e\u0300\u0320\u0300"; - const std::u32string expect_nfd = U"e\u0320\uffff\u0008\u0001\u0300\u0300"; - auto dst_rem = remove_markers(src, &map); - marker_map expm({{0x0300, 0x1L}}); - ASSERT_EQ(dst_rem, expect_rem); - std::u32string dst_nfd = src; - ASSERT_TRUE(normalize_nfd_markers(dst_nfd)); - if (dst_nfd != expect_nfd) { - std::cout << "dst: " << Debug_UnicodeString(dst_nfd) << std::endl; - std::cout << "exp: " << Debug_UnicodeString(expect_nfd) << std::endl; - } - ASSERT_EQ(dst_nfd, expect_nfd); - assert_marker_map_equal(map, expm); - } - { - marker_map map; - std::cout << __FILE__ << ":" << __LINE__ << " - doubled unchanged marker " << std::endl; - const std::u32string src = U"e\u0320\uffff\u0008\u0001\u0300\u0300"; - const std::u32string expect_rem = U"e\u0320\u0300\u0300"; - const std::u32string expect_nfd = src; - auto dst_rem = remove_markers(src, &map); - marker_map expm({{0x300, 0x1L}}); - ASSERT_EQ(dst_rem, expect_rem); - std::u32string dst_nfd = src; - ASSERT_TRUE(normalize_nfd_markers(dst_nfd)); - if (dst_nfd != expect_nfd) { - std::cout << "dst: " << Debug_UnicodeString(dst_nfd) << std::endl; - std::cout << "exp: " << Debug_UnicodeString(expect_nfd) << std::endl; - } - ASSERT_EQ(dst_nfd, expect_nfd); - assert_marker_map_equal(map, expm); - } - { - marker_map map; - std::cout << __FILE__ << ":" << __LINE__ << " - marker-before-double-greek " << std::endl; - const std::u32string src = U"\u03B5\uFFFF\u0008\u0001\u0344\uFFFF\u0008\u0002\u0344\uFFFF\u0008\u0003"; - const std::u32string expect_rem = U"\u03B5\u0344\u0344"; - const std::u32string expect_nfd = U"\u03B5\uFFFF\u0008\u0001\u0308\u0301\uFFFF\u0008\u0002\u0308\u0301\uFFFF\u0008\u0003"; - auto dst_rem = remove_markers(src, &map); - marker_map expm({{0x0308, 0x1L},{0x0308, 0x2L},{MARKER_BEFORE_EOT, 0x3L}}); - ASSERT_EQ(dst_rem, expect_rem); - std::u32string dst_nfd = src; - ASSERT_TRUE(normalize_nfd_markers(dst_nfd)); - if (dst_nfd != expect_nfd) { - std::cout << "dst: " << Debug_UnicodeString(dst_nfd) << std::endl; - std::cout << "exp: " << Debug_UnicodeString(expect_nfd) << std::endl; - } - ASSERT_EQ(dst_nfd, expect_nfd); - assert_marker_map_equal(map, expm); +TEST(TransformsTests, TestNormalizeMarkerBeforeNFC) { + marker_map map; + // KA \m O -> KA \m E AA + const std::u32string src = U"\u0995\uFFFF\u0008\u0001\u09CB"; + const std::u32string expect_rem = U"\u0995\u09CB"; + const std::u32string expect_nfd = U"\u0995\uFFFF\u0008\u0001\u09C7\u09BE"; + auto dst_rem = remove_markers(src, &map); + marker_map expm({{0x09C7, 0x1L}}); + ASSERT_EQ(dst_rem, expect_rem); + std::u32string dst_nfd = src; + ASSERT_TRUE(normalize_nfd_markers(dst_nfd)); + if (dst_nfd != expect_nfd) { + std::cout << "dst: " << Debug_UnicodeString(dst_nfd) << std::endl; + std::cout << "exp: " << Debug_UnicodeString(expect_nfd) << std::endl; } + ASSERT_EQ(dst_nfd, expect_nfd); + assert_marker_map_equal(map, expm); +} -// macro for moving tests from test-pattern-parser.ts -#define TEST_NFD_PLAIN(x, y) \ - { \ - marker_map map; \ - std::cout << __FILE__ << ":" << __LINE__ << ": nfd test " << std::endl; \ - const std::u32string src = x; \ - const std::u32string expect_nfd = y; \ - std::u32string dst_nfd = src; \ - ASSERT_TRUE(normalize_nfd_markers(dst_nfd)); \ - if (dst_nfd != expect_nfd) { \ - std::cout << "dst: " << Debug_UnicodeString(dst_nfd) << std::endl; \ - std::cout << "exp: " << Debug_UnicodeString(expect_nfd) << std::endl; \ - } \ - ASSERT_EQ(dst_nfd, expect_nfd); \ +TEST(TransformsTests, TestNormalizeMarkerBeforeNFC2) { + marker_map map; + const std::u32string src = U"\u0995\u09BE\uFFFF\u0008\u0001\u09C7"; + const std::u32string expect_rem = U"\u0995\u09BE\u09C7"; + const std::u32string expect_nfd = src; // does not get reordered + auto dst_rem = remove_markers(src, &map); + marker_map expm({{0x09C7, 0x1L}}); + ASSERT_EQ(dst_rem, expect_rem); + std::u32string dst_nfd = src; + ASSERT_TRUE(normalize_nfd_markers(dst_nfd)); + if (dst_nfd != expect_nfd) { + std::cout << "dst: " << Debug_UnicodeString(dst_nfd) << std::endl; + std::cout << "exp: " << Debug_UnicodeString(expect_nfd) << std::endl; } + ASSERT_EQ(dst_nfd, expect_nfd); + assert_marker_map_equal(map, expm); +} +TEST(TransformsTests, TestNormalizeMarkerBeforeNFC3) { + marker_map map; + const std::u32string src = U"\u0995\u09BE\uFFFF\u0008\u0001\u09C7"; + const std::u32string expect_rem = U"\u0995\u09BE\u09C7"; + const std::u32string expect_nfd = src; // does not get reordered + auto dst_rem = remove_markers(src, &map); + marker_map expm({{0x09C7, 0x1L}}); + ASSERT_EQ(dst_rem, expect_rem); + std::u32string dst_nfd = src; + ASSERT_TRUE(normalize_nfd_markers(dst_nfd)); + if (dst_nfd != expect_nfd) { + std::cout << "dst: " << Debug_UnicodeString(dst_nfd) << std::endl; + std::cout << "exp: " << Debug_UnicodeString(expect_nfd) << std::endl; + } + ASSERT_EQ(dst_nfd, expect_nfd); + assert_marker_map_equal(map, expm); +} + +TEST(TransformsTests, TestNormalizeMarkerBeforeGreek) { + marker_map map; + const std::u32string src = U"\u03B5\uFFFF\u0008\u0001\u0344"; + const std::u32string expect_rem = U"\u03B5\u0344"; + const std::u32string expect_nfd = U"\u03B5\uFFFF\u0008\u0001\u0308\u0301"; + auto dst_rem = remove_markers(src, &map); + marker_map expm({{0x0308, 0x1L}}); + ASSERT_EQ(dst_rem, expect_rem); + std::u32string dst_nfd = src; + ASSERT_TRUE(normalize_nfd_markers(dst_nfd)); + if (dst_nfd != expect_nfd) { + std::cout << "dst: " << Debug_UnicodeString(dst_nfd) << std::endl; + std::cout << "exp: " << Debug_UnicodeString(expect_nfd) << std::endl; + } + ASSERT_EQ(dst_nfd, expect_nfd); + assert_marker_map_equal(map, expm); +} + +// doubled markers tests + +TEST(TransformsTests, TestNormalizeDoubledMarker1) { + marker_map map; + const std::u32string src = U"e\uffff\u0008\u0001\u0300\u0320\u0300"; + const std::u32string expect_rem = U"e\u0300\u0320\u0300"; + const std::u32string expect_nfd = U"e\u0320\uffff\u0008\u0001\u0300\u0300"; + auto dst_rem = remove_markers(src, &map); + marker_map expm({{0x0300, 0x1L}}); + ASSERT_EQ(dst_rem, expect_rem); + std::u32string dst_nfd = src; + ASSERT_TRUE(normalize_nfd_markers(dst_nfd)); + if (dst_nfd != expect_nfd) { + std::cout << "dst: " << Debug_UnicodeString(dst_nfd) << std::endl; + std::cout << "exp: " << Debug_UnicodeString(expect_nfd) << std::endl; + } + ASSERT_EQ(dst_nfd, expect_nfd); + assert_marker_map_equal(map, expm); +} + +TEST(TransformsTests, TestNormalizeDoubledUnchangedMarker) { + marker_map map; + const std::u32string src = U"e\u0320\uffff\u0008\u0001\u0300\u0300"; + const std::u32string expect_rem = U"e\u0320\u0300\u0300"; + const std::u32string expect_nfd = src; + auto dst_rem = remove_markers(src, &map); + marker_map expm({{0x300, 0x1L}}); + ASSERT_EQ(dst_rem, expect_rem); + std::u32string dst_nfd = src; + ASSERT_TRUE(normalize_nfd_markers(dst_nfd)); + if (dst_nfd != expect_nfd) { + std::cout << "dst: " << Debug_UnicodeString(dst_nfd) << std::endl; + std::cout << "exp: " << Debug_UnicodeString(expect_nfd) << std::endl; + } + ASSERT_EQ(dst_nfd, expect_nfd); + assert_marker_map_equal(map, expm); +} + +TEST(TransformsTests, TestNormalizeMarkerBeforeDoubleGreek) { + marker_map map; + const std::u32string src = U"\u03B5\uFFFF\u0008\u0001\u0344\uFFFF\u0008\u0002\u0344\uFFFF\u0008\u0003"; + const std::u32string expect_rem = U"\u03B5\u0344\u0344"; + const std::u32string expect_nfd = U"\u03B5\uFFFF\u0008\u0001\u0308\u0301\uFFFF\u0008\u0002\u0308\u0301\uFFFF\u0008\u0003"; + auto dst_rem = remove_markers(src, &map); + marker_map expm({{0x0308, 0x1L},{0x0308, 0x2L},{MARKER_BEFORE_EOT, 0x3L}}); + ASSERT_EQ(dst_rem, expect_rem); + std::u32string dst_nfd = src; + ASSERT_TRUE(normalize_nfd_markers(dst_nfd)); + if (dst_nfd != expect_nfd) { + std::cout << "dst: " << Debug_UnicodeString(dst_nfd) << std::endl; + std::cout << "exp: " << Debug_UnicodeString(expect_nfd) << std::endl; + } + ASSERT_EQ(dst_nfd, expect_nfd); + assert_marker_map_equal(map, expm); +} + +void TEST_NFD_PLAIN(std::u32string x, std::u32string y) { + marker_map map; + const std::u32string src = x; + const std::u32string expect_nfd = y; + std::u32string dst_nfd = src; + ASSERT_TRUE(normalize_nfd_markers(dst_nfd)); + if (dst_nfd != expect_nfd) { + std::cout << "dst: " << Debug_UnicodeString(dst_nfd) << std::endl; + std::cout << "exp: " << Debug_UnicodeString(expect_nfd) << std::endl; + } + ASSERT_EQ(dst_nfd, expect_nfd); +} + +TEST(TransformsTests, TestNormalizeNFDDoubleMarker) { // double marker - in front of second, no change - TEST_NFD_PLAIN(U"e\u0320\u0300\uffff\u0008\u0001\u0300", U"e\u0320\u0300\uffff\u0008\u0001\u0300") + ASSERT_NO_FATAL_FAILURE(TEST_NFD_PLAIN(U"e\u0320\u0300\uffff\u0008\u0001\u0300", U"e\u0320\u0300\uffff\u0008\u0001\u0300")); +} + +TEST(TransformsTests, TestNormalizeNFDDoubleMarkerWithSegmentReordering) { // double marker - in front of second, with segment reordering - TEST_NFD_PLAIN(U"e\u0300\u0320\uffff\u0008\u0001\u0300", U"e\u0320\u0300\uffff\u0008\u0001\u0300") + ASSERT_NO_FATAL_FAILURE(TEST_NFD_PLAIN(U"e\u0300\u0320\uffff\u0008\u0001\u0300", U"e\u0320\u0300\uffff\u0008\u0001\u0300")); +} +TEST(TransformsTests, TestNormalizeNFDDoubleMarkerAlternatePattern) { // double marker - alternate pattern with reordering needed - TEST_NFD_PLAIN(U"e\u0300\uffff\u0008\u0001\u0300\u0320", U"e\u0320\u0300\uffff\u0008\u0001\u0300") + ASSERT_NO_FATAL_FAILURE(TEST_NFD_PLAIN(U"e\u0300\uffff\u0008\u0001\u0300\u0320", U"e\u0320\u0300\uffff\u0008\u0001\u0300")); +} + +TEST(TransformsTests, TestNormalizeNFDTripleDiacriticAndMarker) { // triple diacritic + marker - reordering needed - TEST_NFD_PLAIN(U"e\u0300\uffff\u0008\u0001\u0300\u0320\u0300", U"e\u0320\u0300\uffff\u0008\u0001\u0300\u0300") + ASSERT_NO_FATAL_FAILURE(TEST_NFD_PLAIN(U"e\u0300\uffff\u0008\u0001\u0300\u0320\u0300", U"e\u0320\u0300\uffff\u0008\u0001\u0300\u0300")); } -/** test for the util_regex.hpp functions */ -TEST(TransformsTests, TestUtilRegex) { - { - std::cout << __FILE__ << ":" << __LINE__ << " * util_regex.hpp null tests" << std::endl; - km::core::util::km_regex r; - ASSERT_TRUE(!r.valid()); // not valid because of an empty string - } - { - std::cout << __FILE__ << ":" << __LINE__ << " * util_regex.hpp simple tests" << std::endl; - km::core::util::km_regex r(U"ion"); - ASSERT_TRUE(r.valid()); - const std::u32string to(U"ivity"); - const std::deque fromList; - const std::deque toList; - std::u32string output; - auto apply0 = r.apply(U"not present", output, to, fromList, toList); - ASSERT_EQ(apply0, 0); // not found +/* tests for the util_regex.hpp functions */ - const std::u32string input(U"action"); - auto apply1 = r.apply(input, output, to, fromList, toList); - ASSERT_EQ(apply1, 3); // matched last 3 codepoints - std::u32string expect(U"ivity"); - ASSERT_EQ(output, expect); - } - { - std::cout << __FILE__ << ":" << __LINE__ << " * util_regex.hpp wide tests" << std::endl; - km::core::util::km_regex r(U"e𐒻"); - ASSERT_TRUE(r.valid()); - const std::u32string to(U"𐓏"); - const std::deque fromList; - const std::deque toList; - std::u32string output; - const std::u32string input(U":e𐒻"); - auto apply1 = r.apply(input, output, to, fromList, toList); - ASSERT_EQ(apply1, 2); // matched last 2 codepoints - std::u32string expect(U"𐓏"); - ASSERT_EQ(output, expect); - } - { - std::cout << __FILE__ << ":" << __LINE__ << " * util_regex.hpp simple map tests" << std::endl; - km::core::util::km_regex r(U"(A|B|C)"); - ASSERT_TRUE(r.valid()); - const std::u32string to(U"$[1:alpha2]"); // ignored - std::deque fromList; - fromList.emplace_back(U"A"); - fromList.emplace_back(U"B"); - fromList.emplace_back(U"C"); - std::deque toList; - toList.emplace_back(U"N"); - toList.emplace_back(U"O"); - toList.emplace_back(U"P"); - std::u32string output; - auto apply0 = r.apply(U"not present", output, to, fromList, toList); - ASSERT_EQ(apply0, 0); // not found - - const std::u32string input(U"WHOA"); - auto apply1 = r.apply(input, output, to, fromList, toList); - ASSERT_EQ(apply1, 1); // matched last 1 codepoint - std::u32string expect(U"N"); - ASSERT_EQ(output, expect); - } - { - std::cout << __FILE__ << ":" << __LINE__ << " * util_regex.hpp wide map tests" << std::endl; - km::core::util::km_regex r(U"(𐒷|𐒻|𐓏𐓏|x)"); - ASSERT_TRUE(r.valid()); - const std::u32string to(U"$[1:alpha2]"); // ignored - std::deque fromList; - fromList.emplace_back(U"𐒷"); - fromList.emplace_back(U"𐒻"); - fromList.emplace_back(U"𐓏𐓏"); - fromList.emplace_back(U"x"); - std::deque toList; - toList.emplace_back(U"x"); - toList.emplace_back(U"𐒷"); - toList.emplace_back(U"𐒻"); - toList.emplace_back(U"𐓏"); - std::u32string output; - auto apply0 = r.apply(U"not present", output, to, fromList, toList); - ASSERT_EQ(apply0, 0); // not found - - ASSERT_EQ(r.apply(U"WHO𐓏𐒷", output, to, fromList, toList), 1); - ASSERT_EQ(output, U"x"); - - ASSERT_EQ(r.apply(U"WHO𐓏x", output, to, fromList, toList), 1); - ASSERT_EQ(output, U"𐓏"); - - ASSERT_EQ(r.apply(U"WHO𐓏𐓏", output, to, fromList, toList), 2); // 2 codepoints - ASSERT_EQ(output, U"𐒻"); - } +TEST(TransformsTests, TestUtilRegexNull) { + km::core::util::km_regex r; + ASSERT_TRUE(!r.valid()); // not valid because of an empty string +} + +TEST(TransformsTests, TestUtilRegexSimple) { + km::core::util::km_regex r(U"ion"); + ASSERT_TRUE(r.valid()); + const std::u32string to(U"ivity"); + const std::deque fromList; + const std::deque toList; + std::u32string output; + auto apply0 = r.apply(U"not present", output, to, fromList, toList); + ASSERT_EQ(apply0, 0); // not found + + const std::u32string input(U"action"); + auto apply1 = r.apply(input, output, to, fromList, toList); + ASSERT_EQ(apply1, 3); // matched last 3 codepoints + std::u32string expect(U"ivity"); + ASSERT_EQ(output, expect); +} + +TEST(TransformsTests, TestUtilRegexWide) { + km::core::util::km_regex r(U"e𐒻"); + ASSERT_TRUE(r.valid()); + const std::u32string to(U"𐓏"); + const std::deque fromList; + const std::deque toList; + std::u32string output; + const std::u32string input(U":e𐒻"); + auto apply1 = r.apply(input, output, to, fromList, toList); + ASSERT_EQ(apply1, 2); // matched last 2 codepoints + std::u32string expect(U"𐓏"); + ASSERT_EQ(output, expect); +} + +TEST(TransformsTests, TestUtilRegexSimpleMap) { + km::core::util::km_regex r(U"(A|B|C)"); + ASSERT_TRUE(r.valid()); + const std::u32string to(U"$[1:alpha2]"); // ignored + std::deque fromList; + fromList.emplace_back(U"A"); + fromList.emplace_back(U"B"); + fromList.emplace_back(U"C"); + std::deque toList; + toList.emplace_back(U"N"); + toList.emplace_back(U"O"); + toList.emplace_back(U"P"); + std::u32string output; + auto apply0 = r.apply(U"not present", output, to, fromList, toList); + ASSERT_EQ(apply0, 0); // not found + + const std::u32string input(U"WHOA"); + auto apply1 = r.apply(input, output, to, fromList, toList); + ASSERT_EQ(apply1, 1); // matched last 1 codepoint + std::u32string expect(U"N"); + ASSERT_EQ(output, expect); +} + +TEST(TransformsTests, TestUtilRegexWideMap) { + km::core::util::km_regex r(U"(𐒷|𐒻|𐓏𐓏|x)"); + ASSERT_TRUE(r.valid()); + const std::u32string to(U"$[1:alpha2]"); // ignored + std::deque fromList; + fromList.emplace_back(U"𐒷"); + fromList.emplace_back(U"𐒻"); + fromList.emplace_back(U"𐓏𐓏"); + fromList.emplace_back(U"x"); + std::deque toList; + toList.emplace_back(U"x"); + toList.emplace_back(U"𐒷"); + toList.emplace_back(U"𐒻"); + toList.emplace_back(U"𐓏"); + std::u32string output; + auto apply0 = r.apply(U"not present", output, to, fromList, toList); + ASSERT_EQ(apply0, 0); // not found + + ASSERT_EQ(r.apply(U"WHO𐓏𐒷", output, to, fromList, toList), 1); + ASSERT_EQ(output, U"x"); + + ASSERT_EQ(r.apply(U"WHO𐓏x", output, to, fromList, toList), 1); + ASSERT_EQ(output, U"𐓏"); + + ASSERT_EQ(r.apply(U"WHO𐓏𐓏", output, to, fromList, toList), 2); // 2 codepoints + ASSERT_EQ(output, U"𐒻"); } From 86571de22f1f2ddd9c6454c0ab376f3d4268b9bc Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 6 May 2026 14:27:51 +0200 Subject: [PATCH 3/8] chore(core): cleanup compare_debug_items unit test helper Relates-to: #15913 --- core/tests/unit/helpers/debug_items.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/core/tests/unit/helpers/debug_items.cpp b/core/tests/unit/helpers/debug_items.cpp index 2d8452a9e2..a7fb7a74d5 100644 --- a/core/tests/unit/helpers/debug_items.cpp +++ b/core/tests/unit/helpers/debug_items.cpp @@ -77,7 +77,6 @@ void debug_items_equal( } } -// TODO-WEB-CORE: avoid cout; instead use google test reporting once all tests are google test void compare_debug_items( km_core_state const * state, std::initializer_list const & expected @@ -86,21 +85,21 @@ void compare_debug_items( auto act = km_core_state_debug_items(state, &n); for (auto &rhs: expected) { - if ((int)--n < 0) { - std::cout << "expected longer than actual" << std::endl; + if(n == 0) { print_debug_item("next expected item:", rhs); - FAIL(); + ASSERT_NE(n, 0) << "expected longer than actual"; } + n--; + bool result; ASSERT_NO_FATAL_FAILURE(debug_items_equal(*act++, rhs, result)); ASSERT_TRUE(result); } if(n != 0) { - std::cout << "actual longer than expected" << std::endl; print_debug_item("next actual item:", *act); - FAIL(); + ASSERT_EQ(n, 0) << "actual longer than expected"; } } From f7e210beb045c2b670cf1af5feaa79cb10d9e779 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 6 May 2026 14:47:15 +0200 Subject: [PATCH 4/8] chore(core): cleanup state_context_api.tests.cpp * Rename citems and eliminate shadowed class member * Cleanup is_identical_context and related helper functions and make consistent name and signature, to assert_identical_context. * Use clearer variable names for expected and actual values. Relates-to: #15913 --- .../unit/api/state_context_api.tests.cpp | 219 +++++++++--------- 1 file changed, 111 insertions(+), 108 deletions(-) diff --git a/core/tests/unit/api/state_context_api.tests.cpp b/core/tests/unit/api/state_context_api.tests.cpp index a3d41d83c9..946fc1700f 100644 --- a/core/tests/unit/api/state_context_api.tests.cpp +++ b/core/tests/unit/api/state_context_api.tests.cpp @@ -22,7 +22,6 @@ class StateContextApiTests : public testing::Test { protected: km_core_keyboard *test_kb = nullptr; km_core_state *test_state = nullptr; - km_core_context_item *citems = nullptr; void Initialize(const char *keyboard, const km_core_cu *context, bool setup_app_context = true) { km::core::path path = km::core::path(test_dir / ".." / "kmx" / keyboard); @@ -35,18 +34,20 @@ protected: ASSERT_STATUS_OK(km_core_keyboard_load_from_blob(path.stem().c_str(), blob.data(), blob.size(), &test_kb)); } ASSERT_STATUS_OK(km_core_state_create(test_kb, test_empty_env_opts, &test_state)); - ASSERT_STATUS_OK(context_items_from_utf16(context, &citems)); - ASSERT_STATUS_OK(km_core_context_set(km_core_state_context(test_state), citems)); + + km_core_context_item *context_items = nullptr; + ASSERT_STATUS_OK(context_items_from_utf16(context, &context_items)); + ASSERT_STATUS_OK(km_core_context_set(km_core_state_context(test_state), context_items)); if (setup_app_context) { - ASSERT_STATUS_OK(km_core_context_set(km_core_state_app_context(test_state), citems)); + ASSERT_STATUS_OK(km_core_context_set(km_core_state_app_context(test_state), context_items)); + } + if (context_items) { + km_core_context_items_dispose(context_items); + context_items = nullptr; } } void TearDown() override { - if (citems) { - km_core_context_items_dispose(citems); - citems = nullptr; - } if (test_state) { km_core_state_dispose(test_state); test_state = nullptr; @@ -56,57 +57,59 @@ protected: test_kb = nullptr; } } - - void is_identical_context(km_core_cu const *cached_context) { - size_t buf_size; - ASSERT_STATUS_OK(km_core_context_get(km_core_state_context(test_state), &citems)); - ASSERT_STATUS_OK(context_items_to_utf16(citems, nullptr, &buf_size)); - km_core_cu *new_cached_context = new km_core_cu[buf_size]; - ASSERT_STATUS_OK(context_items_to_utf16(citems, new_cached_context, &buf_size)); - ASSERT_EQ(std::u16string(cached_context), new_cached_context); - delete[] new_cached_context; - } - - void is_different_context(km_core_cu const *cached_context) { - size_t buf_size; - ASSERT_STATUS_OK(km_core_context_get(km_core_state_context(test_state), &citems)); - ASSERT_STATUS_OK(context_items_to_utf16(citems, nullptr, &buf_size)); - km_core_cu *new_cached_context = new km_core_cu[buf_size]; - ASSERT_STATUS_OK(context_items_to_utf16(citems, new_cached_context, &buf_size)); - ASSERT_NE(std::u16string(cached_context), new_cached_context); - delete[] new_cached_context; - } }; - -void assert_identical_context_with_markers(const km_core_context *context, const km_core_context_item *citems) { - km_core_context_item *citems_new; - ASSERT_STATUS_OK(km_core_context_get(context, &citems_new)); - for (int i = 0; citems[i].type || citems_new[i].type; i++) { - ASSERT_EQ(citems_new[i].type, citems[i].type) << "Unexpected type:"; - if (citems[i].type == KM_CORE_CT_CHAR) { - ASSERT_EQ(citems_new[i].character, citems[i].character) << "Unexpected character:"; - } else { - ASSERT_EQ(citems_new[i].marker, citems[i].marker) << "Unexpected marker:"; - } - } - km_core_context_items_dispose(citems_new); +void assert_identical_context(const km_core_context *actual_context, km_core_cu const *expected_context) { + km_core_context_item *actual_context_items; + size_t buf_size; + ASSERT_STATUS_OK(km_core_context_get(actual_context, &actual_context_items)); + ASSERT_STATUS_OK(context_items_to_utf16(actual_context_items, nullptr, &buf_size)); + km_core_cu *actual_context_cu = new km_core_cu[buf_size]; + ASSERT_STATUS_OK(context_items_to_utf16(actual_context_items, actual_context_cu, &buf_size)); + ASSERT_EQ(actual_context_cu, std::u16string(expected_context)); + delete[] actual_context_cu; } -// citems contains markers, but we will skip over them -void assert_identical_context_without_markers(const km_core_context *context, const km_core_context_item *citems) { - km_core_context_item *citems_new; - ASSERT_STATUS_OK(km_core_context_get(context, &citems_new)); - for (int i = 0, i_new = 0; citems[i].type || citems_new[i_new].type; i++) { - if (citems[i].type == KM_CORE_CT_CHAR) { - ASSERT_EQ(citems_new[i_new].type, citems[i].type) << "Unexpected type:"; - ASSERT_EQ(citems_new[i_new].character, citems[i].character) << "Unexpected character:"; - i_new++; - } else if(citems[i].type == KM_CORE_CT_END) { - ASSERT_EQ(citems_new[i_new].type, citems[i].type) << "Unexpected type:"; +void assert_different_context(const km_core_context *actual_context, km_core_cu const *expected_context) { + km_core_context_item *actual_context_items; + size_t buf_size; + ASSERT_STATUS_OK(km_core_context_get(actual_context, &actual_context_items)); + ASSERT_STATUS_OK(context_items_to_utf16(actual_context_items, nullptr, &buf_size)); + km_core_cu *actual_context_cu = new km_core_cu[buf_size]; + ASSERT_STATUS_OK(context_items_to_utf16(actual_context_items, actual_context_cu, &buf_size)); + ASSERT_NE(actual_context_cu, std::u16string(expected_context)); + delete[] actual_context_cu; +} + + +void assert_identical_context_with_markers(const km_core_context *actual_context, const km_core_context_item *expected_context_items) { + km_core_context_item *actual_context_items; + ASSERT_STATUS_OK(km_core_context_get(actual_context, &actual_context_items)); + for (int i = 0; expected_context_items[i].type || actual_context_items[i].type; i++) { + ASSERT_EQ(actual_context_items[i].type, expected_context_items[i].type) << "Unexpected type:"; + if (expected_context_items[i].type == KM_CORE_CT_CHAR) { + ASSERT_EQ(actual_context_items[i].character, expected_context_items[i].character) << "Unexpected character:"; + } else { + ASSERT_EQ(actual_context_items[i].marker, expected_context_items[i].marker) << "Unexpected marker:"; } } - km_core_context_items_dispose(citems_new); + km_core_context_items_dispose(actual_context_items); +} + +// expected_context_items contains markers, but we will skip over them +void assert_identical_context_without_markers(const km_core_context *actual_context, const km_core_context_item *expected_context_items) { + km_core_context_item *actual_context_items; + ASSERT_STATUS_OK(km_core_context_get(actual_context, &actual_context_items)); + for (int i = 0, i_new = 0; expected_context_items[i].type || actual_context_items[i_new].type; i++) { + if (expected_context_items[i].type == KM_CORE_CT_CHAR) { + ASSERT_EQ(actual_context_items[i_new].type, expected_context_items[i].type) << "Unexpected type:"; + ASSERT_EQ(actual_context_items[i_new].character, expected_context_items[i].character) << "Unexpected character:"; + i_new++; + } else if(expected_context_items[i].type == KM_CORE_CT_END) { + ASSERT_EQ(actual_context_items[i_new].type, expected_context_items[i].type) << "Unexpected type:"; + } + } + km_core_context_items_dispose(actual_context_items); } // Scenarios from #10100: @@ -117,7 +120,7 @@ TEST_F(StateContextApiTests, TestContextSetIfNeededIdenticalContext) { km_core_cu const *new_app_context = u"This is a test"; ASSERT_NO_FATAL_FAILURE(Initialize("k_0000___null_keyboard.kmx", cached_context, false)); ASSERT_EQ(km_core_state_context_set_if_needed(test_state, new_app_context), KM_CORE_CONTEXT_STATUS_UNCHANGED); - ASSERT_NO_FATAL_FAILURE(is_identical_context(cached_context)); + ASSERT_NO_FATAL_FAILURE(assert_identical_context(km_core_state_context(test_state), cached_context)); } // 1a. cached context has markers and is identical to app context @@ -126,17 +129,17 @@ TEST_F(StateContextApiTests, TestContextSetIfNeededIdenticalContextAndMarkers) { km_core_cu const *new_app_context = u"123"; ASSERT_NO_FATAL_FAILURE(Initialize("k_0000___null_keyboard.kmx", cached_context)); - km_core_context_item const citems[] = { + km_core_context_item const expected_context_items[] = { {KM_CORE_CT_MARKER, {0}, {5}}, {KM_CORE_CT_CHAR, {0}, {'1'}}, {KM_CORE_CT_MARKER, {0}, {1}}, {KM_CORE_CT_CHAR, {0}, {'2'}}, {KM_CORE_CT_MARKER, {0}, {2}}, {KM_CORE_CT_CHAR, {0}, {'3'}}, {KM_CORE_CT_MARKER, {0}, {3}}, {KM_CORE_CT_MARKER, {0}, {4}}, KM_CORE_CONTEXT_ITEM_END}; - ASSERT_STATUS_OK(km_core_context_set(km_core_state_context(test_state), citems)); + ASSERT_STATUS_OK(km_core_context_set(km_core_state_context(test_state), expected_context_items)); ASSERT_EQ(km_core_state_context_set_if_needed(test_state, new_app_context), KM_CORE_CONTEXT_STATUS_UNCHANGED); - ASSERT_NO_FATAL_FAILURE(assert_identical_context_with_markers(km_core_state_context(test_state), citems)); - ASSERT_NO_FATAL_FAILURE(assert_identical_context_without_markers(km_core_state_app_context(test_state), citems)); + ASSERT_NO_FATAL_FAILURE(assert_identical_context_with_markers(km_core_state_context(test_state), expected_context_items)); + ASSERT_NO_FATAL_FAILURE(assert_identical_context_without_markers(km_core_state_app_context(test_state), expected_context_items)); } // 2. cached context same length as app context but content is different @@ -145,8 +148,8 @@ TEST_F(StateContextApiTests, TestContextSetIfNeededDifferentContext) { km_core_cu const *new_app_context = u"This is a test"; ASSERT_NO_FATAL_FAILURE(Initialize("k_0000___null_keyboard.kmx", cached_context)); ASSERT_EQ(km_core_state_context_set_if_needed(test_state, new_app_context), KM_CORE_CONTEXT_STATUS_UPDATED); - ASSERT_NO_FATAL_FAILURE(is_different_context(cached_context)); - ASSERT_NO_FATAL_FAILURE(is_identical_context(new_app_context)); + ASSERT_NO_FATAL_FAILURE(assert_different_context(km_core_state_context(test_state), cached_context)); + ASSERT_NO_FATAL_FAILURE(assert_identical_context(km_core_state_context(test_state), new_app_context)); } // 3. cached context is shorter than app context, but content is same as far as it goes @@ -155,8 +158,8 @@ TEST_F(StateContextApiTests, TestContextSetIfNeededAppContextIsLonger) { km_core_cu const *new_app_context = u"Longer This is a test"; ASSERT_NO_FATAL_FAILURE(Initialize("k_0000___null_keyboard.kmx", cached_context)); ASSERT_EQ(km_core_state_context_set_if_needed(test_state, new_app_context), KM_CORE_CONTEXT_STATUS_UPDATED); - ASSERT_NO_FATAL_FAILURE(is_different_context(cached_context)); - ASSERT_NO_FATAL_FAILURE(is_identical_context(new_app_context)); + ASSERT_NO_FATAL_FAILURE(assert_different_context(km_core_state_context(test_state), cached_context)); + ASSERT_NO_FATAL_FAILURE(assert_identical_context(km_core_state_context(test_state), new_app_context)); } // 3a. cached context has markers and is shorter than app context, @@ -166,22 +169,22 @@ TEST_F(StateContextApiTests, TestContextSetIfNeededCachedContextShorterAndMarker km_core_cu const *new_app_context = u"0123"; ASSERT_NO_FATAL_FAILURE(Initialize("k_0000___null_keyboard.kmx", cached_context)); - km_core_context_item const citems[] = { + km_core_context_item const initial_context_items[] = { {KM_CORE_CT_MARKER, {0}, {5}}, {KM_CORE_CT_CHAR, {0}, {'1'}}, {KM_CORE_CT_MARKER, {0}, {1}}, {KM_CORE_CT_CHAR, {0}, {'2'}}, {KM_CORE_CT_MARKER, {0}, {2}}, {KM_CORE_CT_CHAR, {0}, {'3'}}, {KM_CORE_CT_MARKER, {0}, {3}}, {KM_CORE_CT_MARKER, {0}, {4}}, KM_CORE_CONTEXT_ITEM_END}; - ASSERT_STATUS_OK(km_core_context_set(km_core_state_context(test_state), citems)); + ASSERT_STATUS_OK(km_core_context_set(km_core_state_context(test_state), initial_context_items)); ASSERT_EQ(km_core_state_context_set_if_needed(test_state, new_app_context), KM_CORE_CONTEXT_STATUS_UPDATED); - km_core_context_item const expected_citems[] = { + km_core_context_item const expected_context_items[] = { {KM_CORE_CT_CHAR, {0}, {'0'}}, {KM_CORE_CT_MARKER, {0}, {5}}, {KM_CORE_CT_CHAR, {0}, {'1'}}, {KM_CORE_CT_MARKER, {0}, {1}}, {KM_CORE_CT_CHAR, {0}, {'2'}}, {KM_CORE_CT_MARKER, {0}, {2}}, {KM_CORE_CT_CHAR, {0}, {'3'}}, {KM_CORE_CT_MARKER, {0}, {3}}, {KM_CORE_CT_MARKER, {0}, {4}}, KM_CORE_CONTEXT_ITEM_END}; - ASSERT_NO_FATAL_FAILURE(assert_identical_context_with_markers(km_core_state_context(test_state), expected_citems)); - ASSERT_NO_FATAL_FAILURE(assert_identical_context_without_markers(km_core_state_app_context(test_state), expected_citems)); - ASSERT_NO_FATAL_FAILURE(is_identical_context(new_app_context)); + ASSERT_NO_FATAL_FAILURE(assert_identical_context_with_markers(km_core_state_context(test_state), expected_context_items)); + ASSERT_NO_FATAL_FAILURE(assert_identical_context_without_markers(km_core_state_app_context(test_state), expected_context_items)); + ASSERT_NO_FATAL_FAILURE(assert_identical_context(km_core_state_context(test_state), new_app_context)); } TEST_F(StateContextApiTests, TestContextSetIfNeededCachedContextShorterAndMarkersNfu) { @@ -189,22 +192,22 @@ TEST_F(StateContextApiTests, TestContextSetIfNeededCachedContextShorterAndMarker km_core_cu const *new_app_context = u"abcệ"; ASSERT_NO_FATAL_FAILURE(Initialize("/a/dummy/keyboard.mock", cached_context)); - km_core_context_item const citems[] = { + km_core_context_item const initial_context_items[] = { {KM_CORE_CT_MARKER, {0}, {1}}, {KM_CORE_CT_CHAR, {0}, {'b'}}, {KM_CORE_CT_MARKER, {0}, {2}}, {KM_CORE_CT_CHAR, {0}, {'c'}}, {KM_CORE_CT_MARKER, {0}, {3}}, {KM_CORE_CT_MARKER, {0}, {4}}, {KM_CORE_CT_CHAR, {0}, {'e'}}, {KM_CORE_CT_CHAR, {0}, {u'\u0323'}}, {KM_CORE_CT_CHAR, {0}, {u'\u0302'}}, KM_CORE_CONTEXT_ITEM_END}; - ASSERT_STATUS_OK(km_core_context_set(km_core_state_context(test_state), citems)); + ASSERT_STATUS_OK(km_core_context_set(km_core_state_context(test_state), initial_context_items)); ASSERT_EQ(km_core_state_context_set_if_needed(test_state, new_app_context), KM_CORE_CONTEXT_STATUS_UPDATED); - km_core_context_item const expected_citems[] = { + km_core_context_item const expected_context_items[] = { {KM_CORE_CT_CHAR, {0}, {'a'}}, {KM_CORE_CT_MARKER, {0}, {1}}, {KM_CORE_CT_CHAR, {0}, {'b'}}, {KM_CORE_CT_MARKER, {0}, {2}}, {KM_CORE_CT_CHAR, {0}, {'c'}}, {KM_CORE_CT_MARKER, {0}, {3}}, {KM_CORE_CT_MARKER, {0}, {4}}, {KM_CORE_CT_CHAR, {0}, {'e'}}, {KM_CORE_CT_CHAR, {0}, {u'\u0323'}}, {KM_CORE_CT_CHAR, {0}, {u'\u0302'}}, KM_CORE_CONTEXT_ITEM_END}; - ASSERT_NO_FATAL_FAILURE(assert_identical_context_with_markers(km_core_state_context(test_state), expected_citems)); + ASSERT_NO_FATAL_FAILURE(assert_identical_context_with_markers(km_core_state_context(test_state), expected_context_items)); km_core_context_item const expected_app_citems[] = { {KM_CORE_CT_CHAR, {0}, {'a'}}, {KM_CORE_CT_CHAR, {0}, {'b'}}, @@ -220,8 +223,8 @@ TEST_F(StateContextApiTests, TestContextSetIfNeededCachedContextCleared) { ASSERT_NO_FATAL_FAILURE(Initialize("k_0000___null_keyboard.kmx", cached_context)); km_core_state_context_clear(test_state); ASSERT_EQ(km_core_state_context_set_if_needed(test_state, new_app_context), KM_CORE_CONTEXT_STATUS_UPDATED); - ASSERT_NO_FATAL_FAILURE(is_different_context(cached_context)); - ASSERT_NO_FATAL_FAILURE(is_identical_context(new_app_context)); + ASSERT_NO_FATAL_FAILURE(assert_different_context(km_core_state_context(test_state), cached_context)); + ASSERT_NO_FATAL_FAILURE(assert_identical_context(km_core_state_context(test_state), new_app_context)); } // 5. cached context is longer than app context, but content is same as far as it goes @@ -230,8 +233,8 @@ TEST_F(StateContextApiTests, TestContextSetIfNeededAppContextIsShorter) { km_core_cu const *new_app_context = u"is a test"; ASSERT_NO_FATAL_FAILURE(Initialize("k_0000___null_keyboard.kmx", cached_context)); ASSERT_EQ(km_core_state_context_set_if_needed(test_state, new_app_context), KM_CORE_CONTEXT_STATUS_UPDATED); - ASSERT_NO_FATAL_FAILURE(is_different_context(cached_context)); - ASSERT_NO_FATAL_FAILURE(is_identical_context(new_app_context)); + ASSERT_NO_FATAL_FAILURE(assert_different_context(km_core_state_context(test_state), cached_context)); + ASSERT_NO_FATAL_FAILURE(assert_identical_context(km_core_state_context(test_state), new_app_context)); } // 5a. cached context has markers and is longer than app context, but @@ -241,21 +244,21 @@ TEST_F(StateContextApiTests, TestContextSetIfNeededCachedContextLongerAndMarkers km_core_cu const *new_app_context = u"123"; ASSERT_NO_FATAL_FAILURE(Initialize("k_0000___null_keyboard.kmx", cached_context)); - km_core_context_item const citems[] = { + km_core_context_item const initial_context_items[] = { {KM_CORE_CT_CHAR, {0}, {'0'}}, {KM_CORE_CT_MARKER, {0}, {5}}, {KM_CORE_CT_CHAR, {0}, {'1'}}, {KM_CORE_CT_MARKER, {0}, {1}}, {KM_CORE_CT_CHAR, {0}, {'2'}}, {KM_CORE_CT_MARKER, {0}, {2}}, {KM_CORE_CT_CHAR, {0}, {'3'}}, {KM_CORE_CT_MARKER, {0}, {3}}, {KM_CORE_CT_MARKER, {0}, {4}}, KM_CORE_CONTEXT_ITEM_END}; - ASSERT_STATUS_OK(km_core_context_set(km_core_state_context(test_state), citems)); + ASSERT_STATUS_OK(km_core_context_set(km_core_state_context(test_state), initial_context_items)); ASSERT_EQ(km_core_state_context_set_if_needed(test_state, new_app_context), KM_CORE_CONTEXT_STATUS_UPDATED); - km_core_context_item const expected_citems[] = { + km_core_context_item const expected_context_items[] = { {KM_CORE_CT_MARKER, {0}, {5}}, {KM_CORE_CT_CHAR, {0}, {'1'}}, {KM_CORE_CT_MARKER, {0}, {1}}, {KM_CORE_CT_CHAR, {0}, {'2'}}, {KM_CORE_CT_MARKER, {0}, {2}}, {KM_CORE_CT_CHAR, {0}, {'3'}}, {KM_CORE_CT_MARKER, {0}, {3}}, {KM_CORE_CT_MARKER, {0}, {4}}, KM_CORE_CONTEXT_ITEM_END}; - ASSERT_NO_FATAL_FAILURE(assert_identical_context_with_markers(km_core_state_context(test_state), expected_citems)); - ASSERT_NO_FATAL_FAILURE(assert_identical_context_without_markers(km_core_state_app_context(test_state), expected_citems)); + ASSERT_NO_FATAL_FAILURE(assert_identical_context_with_markers(km_core_state_context(test_state), expected_context_items)); + ASSERT_NO_FATAL_FAILURE(assert_identical_context_without_markers(km_core_state_app_context(test_state), expected_context_items)); } TEST_F(StateContextApiTests, TestContextSetIfNeededCachedContextLongerAndMarkersNfu) { @@ -263,22 +266,22 @@ TEST_F(StateContextApiTests, TestContextSetIfNeededCachedContextLongerAndMarkers km_core_cu const *new_app_context = u"bcệ"; ASSERT_NO_FATAL_FAILURE(Initialize("/a/dummy/keyboard.mock", cached_context)); - km_core_context_item const citems[] = { + km_core_context_item const initial_context_items[] = { {KM_CORE_CT_CHAR, {0}, {'a'}}, {KM_CORE_CT_MARKER, {0}, {1}}, {KM_CORE_CT_CHAR, {0}, {'b'}}, {KM_CORE_CT_MARKER, {0}, {2}}, {KM_CORE_CT_CHAR, {0}, {'c'}}, {KM_CORE_CT_MARKER, {0}, {3}}, {KM_CORE_CT_MARKER, {0}, {4}}, {KM_CORE_CT_CHAR, {0}, {'e'}}, {KM_CORE_CT_CHAR, {0}, {u'\u0323'}}, {KM_CORE_CT_CHAR, {0}, {u'\u0302'}}, KM_CORE_CONTEXT_ITEM_END}; - ASSERT_STATUS_OK(km_core_context_set(km_core_state_context(test_state), citems)); + ASSERT_STATUS_OK(km_core_context_set(km_core_state_context(test_state), initial_context_items)); ASSERT_EQ(km_core_state_context_set_if_needed(test_state, new_app_context), KM_CORE_CONTEXT_STATUS_UPDATED); - km_core_context_item const expected_citems[] = { + km_core_context_item const expected_context_items[] = { {KM_CORE_CT_MARKER, {0}, {1}}, {KM_CORE_CT_CHAR, {0}, {'b'}}, {KM_CORE_CT_MARKER, {0}, {2}}, {KM_CORE_CT_CHAR, {0}, {'c'}}, {KM_CORE_CT_MARKER, {0}, {3}}, {KM_CORE_CT_MARKER, {0}, {4}}, {KM_CORE_CT_CHAR, {0}, {'e'}}, {KM_CORE_CT_CHAR, {0}, {u'\u0323'}}, {KM_CORE_CT_CHAR, {0}, {u'\u0302'}}, KM_CORE_CONTEXT_ITEM_END}; - ASSERT_NO_FATAL_FAILURE(assert_identical_context_with_markers(km_core_state_context(test_state), expected_citems)); + ASSERT_NO_FATAL_FAILURE(assert_identical_context_with_markers(km_core_state_context(test_state), expected_context_items)); km_core_context_item const expected_app_citems[] = { {KM_CORE_CT_CHAR, {0}, {'b'}}, @@ -294,8 +297,8 @@ TEST_F(StateContextApiTests, TestContextSetIfNeededApplicationContextEmpty) { km_core_cu const *new_app_context = u""; ASSERT_NO_FATAL_FAILURE(Initialize("k_0000___null_keyboard.kmx", cached_context)); ASSERT_EQ(km_core_state_context_set_if_needed(test_state, new_app_context), KM_CORE_CONTEXT_STATUS_UPDATED); - ASSERT_NO_FATAL_FAILURE(is_different_context(cached_context)); - ASSERT_NO_FATAL_FAILURE(is_identical_context(new_app_context)); + ASSERT_NO_FATAL_FAILURE(assert_different_context(km_core_state_context(test_state), cached_context)); + ASSERT_NO_FATAL_FAILURE(assert_identical_context(km_core_state_context(test_state), new_app_context)); } // 7. surrogate pairs in context @@ -304,7 +307,7 @@ TEST_F(StateContextApiTests, TestContextSetIfNeededSurrogatePairsUnchanged) { km_core_cu const *new_app_context = u"a\U00010100"; ASSERT_NO_FATAL_FAILURE(Initialize("k_0000___null_keyboard.kmx", cached_context)); ASSERT_EQ(km_core_state_context_set_if_needed(test_state, new_app_context), KM_CORE_CONTEXT_STATUS_UNCHANGED); - ASSERT_NO_FATAL_FAILURE(is_identical_context(new_app_context)); + ASSERT_NO_FATAL_FAILURE(assert_identical_context(km_core_state_context(test_state), new_app_context)); } TEST_F(StateContextApiTests, TestContextSetIfNeededSurrogatePairsAppContextLonger) { @@ -312,7 +315,7 @@ TEST_F(StateContextApiTests, TestContextSetIfNeededSurrogatePairsAppContextLonge km_core_cu const *new_app_context = u"xa\U00010100"; ASSERT_NO_FATAL_FAILURE(Initialize("k_0000___null_keyboard.kmx", cached_context)); ASSERT_EQ(km_core_state_context_set_if_needed(test_state, new_app_context), KM_CORE_CONTEXT_STATUS_UPDATED); - ASSERT_NO_FATAL_FAILURE(is_identical_context(new_app_context)); + ASSERT_NO_FATAL_FAILURE(assert_identical_context(km_core_state_context(test_state), new_app_context)); } TEST_F(StateContextApiTests, TestContextSetIfNeededSurrogatePairsCachedContextLonger) { @@ -320,7 +323,7 @@ TEST_F(StateContextApiTests, TestContextSetIfNeededSurrogatePairsCachedContextLo km_core_cu const *new_app_context = u"a\U00010100"; ASSERT_NO_FATAL_FAILURE(Initialize("k_0000___null_keyboard.kmx", cached_context)); ASSERT_EQ(km_core_state_context_set_if_needed(test_state, new_app_context), KM_CORE_CONTEXT_STATUS_UPDATED); - ASSERT_NO_FATAL_FAILURE(is_identical_context(new_app_context)); + ASSERT_NO_FATAL_FAILURE(assert_identical_context(km_core_state_context(test_state), new_app_context)); } TEST_F(StateContextApiTests, TestContextSetIfNeededSurrogatePairsUnchangedAndMarkers) { @@ -328,16 +331,16 @@ TEST_F(StateContextApiTests, TestContextSetIfNeededSurrogatePairsUnchangedAndMar km_core_cu const *new_app_context = u"a\U00010100"; ASSERT_NO_FATAL_FAILURE(Initialize("k_0000___null_keyboard.kmx", cached_context)); - km_core_context_item const citems[] = { + km_core_context_item const expected_context_items[] = { {KM_CORE_CT_CHAR, {0}, {'a'}}, {KM_CORE_CT_MARKER, {0}, {5}}, {KM_CORE_CT_CHAR, {0}, {0x10100}}, KM_CORE_CONTEXT_ITEM_END}; - ASSERT_STATUS_OK(km_core_context_set(km_core_state_context(test_state), citems)); + ASSERT_STATUS_OK(km_core_context_set(km_core_state_context(test_state), expected_context_items)); ASSERT_EQ(km_core_state_context_set_if_needed(test_state, new_app_context), KM_CORE_CONTEXT_STATUS_UNCHANGED); - ASSERT_NO_FATAL_FAILURE(assert_identical_context_with_markers(km_core_state_context(test_state), citems)); - ASSERT_NO_FATAL_FAILURE(assert_identical_context_without_markers(km_core_state_app_context(test_state), citems)); + ASSERT_NO_FATAL_FAILURE(assert_identical_context_with_markers(km_core_state_context(test_state), expected_context_items)); + ASSERT_NO_FATAL_FAILURE(assert_identical_context_without_markers(km_core_state_app_context(test_state), expected_context_items)); } TEST_F(StateContextApiTests, TestContextSetIfNeededSurrogatePairsAppContextLongerAndMarkers) { @@ -345,23 +348,23 @@ TEST_F(StateContextApiTests, TestContextSetIfNeededSurrogatePairsAppContextLonge km_core_cu const *new_app_context = u"\U00010200a\U00010100"; ASSERT_NO_FATAL_FAILURE(Initialize("k_0000___null_keyboard.kmx", cached_context)); - km_core_context_item const citems[] = { + km_core_context_item const initial_context_items[] = { {KM_CORE_CT_CHAR, {0}, {'a'}}, {KM_CORE_CT_MARKER, {0}, {5}}, {KM_CORE_CT_CHAR, {0}, {0x10100}}, KM_CORE_CONTEXT_ITEM_END}; - ASSERT_STATUS_OK(km_core_context_set(km_core_state_context(test_state), citems)); + ASSERT_STATUS_OK(km_core_context_set(km_core_state_context(test_state), initial_context_items)); ASSERT_EQ(km_core_state_context_set_if_needed(test_state, new_app_context), KM_CORE_CONTEXT_STATUS_UPDATED); - km_core_context_item const expected_citems[] = { + km_core_context_item const expected_context_items[] = { {KM_CORE_CT_CHAR, {0}, {0x10200}}, {KM_CORE_CT_CHAR, {0}, {'a'}}, {KM_CORE_CT_MARKER, {0}, {5}}, {KM_CORE_CT_CHAR, {0}, {0x10100}}, KM_CORE_CONTEXT_ITEM_END}; - ASSERT_NO_FATAL_FAILURE(assert_identical_context_with_markers(km_core_state_context(test_state), expected_citems)); - ASSERT_NO_FATAL_FAILURE(assert_identical_context_without_markers(km_core_state_app_context(test_state), expected_citems)); + ASSERT_NO_FATAL_FAILURE(assert_identical_context_with_markers(km_core_state_context(test_state), expected_context_items)); + ASSERT_NO_FATAL_FAILURE(assert_identical_context_without_markers(km_core_state_app_context(test_state), expected_context_items)); } TEST_F(StateContextApiTests, TestContextSetIfNeededSurrogatePairsCachedContextLongerAndMarkers) { @@ -369,23 +372,23 @@ TEST_F(StateContextApiTests, TestContextSetIfNeededSurrogatePairsCachedContextLo km_core_cu const *new_app_context = u"a\U00010100"; ASSERT_NO_FATAL_FAILURE(Initialize("k_0000___null_keyboard.kmx", cached_context)); - km_core_context_item const citems[] = { + km_core_context_item const initial_context_items[] = { {KM_CORE_CT_CHAR, {0}, {0x10200}}, {KM_CORE_CT_CHAR, {0}, {'a'}}, {KM_CORE_CT_MARKER, {0}, {5}}, {KM_CORE_CT_CHAR, {0}, {0x10100}}, KM_CORE_CONTEXT_ITEM_END}; - ASSERT_STATUS_OK(km_core_context_set(km_core_state_context(test_state), citems)); + ASSERT_STATUS_OK(km_core_context_set(km_core_state_context(test_state), initial_context_items)); ASSERT_EQ(km_core_state_context_set_if_needed(test_state, new_app_context), KM_CORE_CONTEXT_STATUS_UPDATED); - km_core_context_item const expected_citems[] = { + km_core_context_item const expected_context_items[] = { {KM_CORE_CT_CHAR, {0}, {'a'}}, {KM_CORE_CT_MARKER, {0}, {5}}, {KM_CORE_CT_CHAR, {0}, {0x10100}}, KM_CORE_CONTEXT_ITEM_END}; - ASSERT_NO_FATAL_FAILURE(assert_identical_context_with_markers(km_core_state_context(test_state), expected_citems)); - ASSERT_NO_FATAL_FAILURE(assert_identical_context_without_markers(km_core_state_app_context(test_state), expected_citems)); + ASSERT_NO_FATAL_FAILURE(assert_identical_context_with_markers(km_core_state_context(test_state), expected_context_items)); + ASSERT_NO_FATAL_FAILURE(assert_identical_context_without_markers(km_core_state_app_context(test_state), expected_context_items)); } @@ -393,8 +396,8 @@ TEST_F(StateContextApiTests, TestContextClear) { km_core_cu const *cached_context = u"This is a test"; ASSERT_NO_FATAL_FAILURE(Initialize("k_0000___null_keyboard.kmx", cached_context)); ASSERT_STATUS_OK(km_core_state_context_clear(test_state)); - ASSERT_NO_FATAL_FAILURE(is_different_context(cached_context)); - ASSERT_NO_FATAL_FAILURE(is_identical_context(u"")); + ASSERT_NO_FATAL_FAILURE(assert_different_context(km_core_state_context(test_state), cached_context)); + ASSERT_NO_FATAL_FAILURE(assert_identical_context(km_core_state_context(test_state), u"")); } //------------------------------------------------------------------------------------- @@ -412,7 +415,7 @@ TEST_F(StateContextApiTests, TestContextDebugVarious) { km_core_cu const *cached_context = u"123\U0001F923"; ASSERT_NO_FATAL_FAILURE(Initialize("k_0000___null_keyboard.kmx", cached_context)); - km_core_context_item const citems[] = { + km_core_context_item const expected_context_items[] = { { KM_CORE_CT_MARKER, {0}, { 5 } }, { KM_CORE_CT_CHAR, {0}, { '1' } }, { KM_CORE_CT_MARKER, {0}, { 1 } }, @@ -425,7 +428,7 @@ TEST_F(StateContextApiTests, TestContextDebugVarious) { KM_CORE_CONTEXT_ITEM_END }; - ASSERT_STATUS_OK(km_core_context_set(km_core_state_context(test_state), citems)); + ASSERT_STATUS_OK(km_core_context_set(km_core_state_context(test_state), expected_context_items)); auto str = km_core_state_context_debug(test_state, KM_CORE_DEBUG_CONTEXT_CACHED); // std::cout << str << std::endl; From 40d36f8e2c6ceb1085198eea51d2073100d0ec23 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 6 May 2026 15:09:25 +0200 Subject: [PATCH 5/8] chore(core): add cross-reference between ldml_test_source and kmx_test_source Relates-to: #15913 --- core/tests/kmx_test_source/kmx_test_source.cpp | 3 +++ core/tests/unit/ldml/ldml_test_source.cpp | 3 +++ 2 files changed, 6 insertions(+) diff --git a/core/tests/kmx_test_source/kmx_test_source.cpp b/core/tests/kmx_test_source/kmx_test_source.cpp index c22eb92718..d264b4c7ab 100644 --- a/core/tests/kmx_test_source/kmx_test_source.cpp +++ b/core/tests/kmx_test_source/kmx_test_source.cpp @@ -3,6 +3,9 @@ * * Keyman Core - Shared test library to load .kmn test rules for automated * testing of .kmn keyboards + * + * Note that this file is superficially similar to ldml_test_source.cpp, but the + * parsing and use diverges far enough that they cannot be easily merged. */ #include diff --git a/core/tests/unit/ldml/ldml_test_source.cpp b/core/tests/unit/ldml/ldml_test_source.cpp index 407f3b8062..b7553f8511 100644 --- a/core/tests/unit/ldml/ldml_test_source.cpp +++ b/core/tests/unit/ldml/ldml_test_source.cpp @@ -2,6 +2,9 @@ * Keyman is copyright (C) SIL International. MIT License. * * Keyman Core - Helper to load LDML keyboard test definitions + * + * Note that this file is superficially similar to kmx_test_source.cpp, but the + * parsing and use diverges far enough that they cannot be easily merged. */ #include From a3de441eda4f2a7743ede46867166a1eee6de90a Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 6 May 2026 15:11:59 +0200 Subject: [PATCH 6/8] chore(core): add test for NFD context deleting one combining mark Relates-to: #15913 --- core/tests/unit/api/actions_normalize.tests.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/core/tests/unit/api/actions_normalize.tests.cpp b/core/tests/unit/api/actions_normalize.tests.cpp index eb0f084726..22d6644e61 100644 --- a/core/tests/unit/api/actions_normalize.tests.cpp +++ b/core/tests/unit/api/actions_normalize.tests.cpp @@ -276,6 +276,17 @@ const TestData values[] = { /* app_context: */ u"abce" }, + { + "OneBackspaceToDeleteLastNfdCharacterWithNfcAppContext15487", + /* app context pre transform: */ u"abce\u0302", // NFD + /* cached context post transform: */ u"abce", + /* cached context post transform: */ nullptr, + /* action del, output: */ 1, U"", // NFD input; delete 1: \u0302 + // ---- results ---- + /* action del, output: */ 1, U"", // NFC output; delete 1: e + /* app_context: */ u"abce" + }, + { "OneBackspaceForNfdConvertsIntoOneCharInNfcAndRecombine", /* app context pre transform: */ u"abcê", From 82f4c954a5eb8f548df38cc58938cd7ee340f188 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Thu, 7 May 2026 12:31:19 +0200 Subject: [PATCH 7/8] chore(core): merge actions test data for unit tests Merge actions test data struct and some of the data for the actions API unit tests. actions_update_app_context_nfu.tests.cpp data has not been merged because the expected results are different, as it does not do normalization, unlike actions_get_api and actions_normalize. Merging of that data could be done, but would require adding extra fields to each test result, which would make the data considerably harder to read! Thus I have left it alone for now, just merged the struct, which is still a win. Fixes: #15913 --- core/tests/unit/api/actions_get_api.tests.cpp | 252 +------------ .../unit/api/actions_normalize.tests.cpp | 339 +----------------- core/tests/unit/api/actions_test_data.cpp | 306 ++++++++++++++++ core/tests/unit/api/actions_test_data.h | 64 ++++ .../actions_update_app_context_nfu.tests.cpp | 151 ++++---- core/tests/unit/api/meson.build | 6 +- 6 files changed, 466 insertions(+), 652 deletions(-) create mode 100644 core/tests/unit/api/actions_test_data.cpp create mode 100644 core/tests/unit/api/actions_test_data.h diff --git a/core/tests/unit/api/actions_get_api.tests.cpp b/core/tests/unit/api/actions_get_api.tests.cpp index 10a1a979a8..8d975a94f1 100644 --- a/core/tests/unit/api/actions_get_api.tests.cpp +++ b/core/tests/unit/api/actions_get_api.tests.cpp @@ -15,73 +15,26 @@ #include "context.hpp" #include "../helpers/core_test_helpers.h" +#include "./actions_test_data.h" -struct TestData { - const char* test_name; - - /** - * the app context stored in the state, _before_ transform is applied -- NFU - */ - const km_core_cu *initial_app_context; - - /** - * cached context _after_ actions have been applied -- guaranteed NFD - * (essentially, this is initial_cached_context - - * actions_code_points_to_delete + actions_output) - no markers supported - */ - const km_core_cu *final_cached_context; - - /** - * number of NFD code points that the keyboard processor has asked to remove - * in its actions - */ - int actions_code_points_to_delete; - - /** - * NFD string that the keyboard processor has asked to insert in its actions - */ - const std::u32string actions_output; - - /** - * expected: NFU code points to ask app to remove - */ - const unsigned int expected_delete; - - /** - * expected: adjusted NFC output to insert into the app - */ - const std::u32string expected_output; - - /** - * expected: NFU adjusted final app context, which will be NFC from the - * boundary of the transform, but will not have been modified prior to that. - * Should match char-for-char what the app ends up with in its text buffer. - */ - const km_core_cu *expected_final_app_context; - - /** - * expected: the characters deleted from the context - */ - const std::u32string expected_deleted_context; -}; - -std::string GenerateTestName(const testing::TestParamInfo& info) { - return info.param.test_name; -} - -class GetActionApiTest : public testing::TestWithParam { +class GetActionApiTest : public testing::TestWithParam { protected: km_core_keyboard * test_kb = nullptr; km_core_state * test_state = nullptr; km_core_actions * test_actions = nullptr; - void Initialize(TestData const& data) { + void Initialize(ActionsTestData const& data) { + if(data.final_cached_context_string == nullptr) { + // We skip the shared tests that don't have a input context string + return; + } + km::core::path path = km::core::path::join(test_dir, "..", "ldml", "fixtures", "keyboards", "17.0", "k_001_tiny.kmx"); auto blob = km::tests::load_kmx_file(path.native().c_str()); ASSERT_STATUS_OK(km_core_keyboard_load_from_blob(path.stem().c_str(), blob.data(), blob.size(), &test_kb)); ASSERT_STATUS_OK(km_core_state_create(test_kb, test_empty_env_opts, &test_state)); - ASSERT_STATUS_OK(set_context_from_string(km_core_state_context(test_state), data.final_cached_context)); + ASSERT_STATUS_OK(set_context_from_string(km_core_state_context(test_state), data.final_cached_context_string)); ASSERT_STATUS_OK(set_context_from_string(km_core_state_app_context(test_state), data.initial_app_context)); test_actions = new km_core_actions; @@ -132,6 +85,11 @@ TEST_P(GetActionApiTest, TestActionsApi) { auto data = GetParam(); ASSERT_NO_FATAL_FAILURE(Initialize(data)); + if(data.final_cached_context_string == nullptr) { + // We skip the shared tests that don't have a input context string + return; + } + // setup(initial_app_context, final_cached_context, actions_code_points_to_delete, actions_output); auto actual_actions = km_core_state_get_actions(test_state); @@ -162,184 +120,6 @@ TEST_P(GetActionApiTest, TestActionsApi) { delete [] actual_final_app_context; } -const TestData values[] = { - // Null boundary tests +// Note that .final_cached_context_items is not used in these tests - { - "Noop", - /* app context pre transform: */ u"", - /* cached context post transform: */ u"", - /* action del, output: */ 0, U"", - // ---- results ---- - /* action del, output: */ 0, U"", - /* app_context: */ u"", - /* expected del */ U"" - }, - - { - "NoOutput", - /* app context pre transform: */ u"abc", - /* cached context post transform: */ u"abc", - /* action del, output: */ 0, U"", - // ---- results ---- - /* action del, output: */ 0, U"", - /* app_context: */ u"abc", - /* expected del */ U"" - }, - - { - "NoContext", - /* app context pre transform: */ u"", - /* cached context post transform: */ u"def", - /* action del, output: */ 0, U"def", - // ---- results ---- - /* action del, output: */ 0, U"def", - /* app_context: */ u"def", - /* expected del */ U"" - }, - - // Simple tests -- no deletions involved - - { - "NoNormalization", - /* app context pre transform: */ u"abc", - /* cached context post transform: */ u"abcdef", - /* action del, output: */ 0, U"def", - // ---- results ---- - /* action del, output: */ 0, U"def", - /* app_context: */ u"abcdef", - /* expected del */ U"" - }, - - { - "OutputToNfcBasic", - /* app context pre transform: */ u"abc", - /* cached context post transform: */ u"abcde\u0300f", - /* action del, output: */ 0, U"de\u0300f", - // ---- results ---- - /* action del, output: */ 0, U"dèf", - /* app_context: */ u"abcdèf", - /* expected del */ U"" - }, - - { - "OutputToNfcHefty", - /* app context pre transform: */ u"abc", - /* cached context post transform: */ u"abcA\u0300" u"e\u0316\u0301" u"\u0073\u0323\u0307" u"\u0041\u030a" u"\U000114B9\U000114B0", - /* action del, output: */ 0, U"A\u0300" U"e\u0316\u0301" U"\u0073\u0323\u0307" U"\u0041\u030a" U"\U000114B9\U000114B0", - // ---- results ---- - /* action del, output: */ 0, U"À" U"é̖" U"\u1e69" U"\u00c5" U"\U000114BC", - /* app_context: */ u"abcÀé̖\u1e69\u00c5\U000114BC", - /* expected del */ U"" - }, - - // Interaction with input context when not on normalization boundary - - { - "BacktrackOneCharacterToCombineAsNfc", - /* app context pre transform: */ u"XYZA", - /* cached context post transform: */ u"XYZA\u0300abc", - /* action del, output: */ 0, U"\u0300abc", - // ---- results ---- - /* action del, output: */ 1, U"Àabc", - /* app_context: */ u"XYZÀabc", - /* expected del */ U"A" - }, - - { - "BacktrackECombCirc2CharsToCombineAsNfc", - /* app context pre transform: */ u"abce\u0302", - /* cached context post transform: */ u"abce\u0323\u0302", - /* action del, output: */ 1, U"\u0323\u0302", - // ---- results ---- - /* action del, output: */ 2, U"ệ", - /* app_context: */ u"abcệ", - /* expected del */ U"e\u0302" - }, - - { - "OneBackspaceForNfdConvertsIntoOneCharInNfcAndRecombine", - /* app context pre transform: */ u"abcê", - /* cached context post transform: */ u"abce\u0323\u0302", - /* action del, output: */ 1, U"\u0323\u0302", // NFD input; delete 1: \u0302 - // ---- results ---- - /* action del, output: */ 1, U"ệ", // NFC output; delete 1: ê - /* app_context: */ u"abcệ", - /* expected del */ U"ê" - }, - - // a\u0300 should not be normalized because it is not otherwise impacted by - // the action. - { - "AvoidEditingTooFarBackInContextWhenFindingNormalizationBoundary", - /* app context pre transform: */ u"a\u0300bcê", - /* cached context post transform: */ u"a\u0300bce\u0323\u0302", - /* action del, output: */ 1, U"\u0323\u0302", // NFD input; delete 1: \u0302 - // ---- results ---- - /* action del, output: */ 1, U"ệ", // NFC output; delete 1: ê - /* app_context: */ u"a\u0300bcệ", - /* expected del */ U"ê" - }, - - // If we don't reach a normalization boundary, we still should continue to work - { - "NormalizableLettersAtStartOfContext", - /* app context pre transform: */ u"\u0300", - /* cached context post transform: */ u"\u0323\u0300\u0302", - /* action del, output: */ 1, U"\u0323\u0300\u0302", // NFD input; - // ---- results ---- - /* action del, output: */ 1, U"\u0323\u0300\u0302", // NFC output is still decomposed because there is no base - /* app_context: */ u"\u0323\u0300\u0302", - /* expected del */ U"\u0300" - }, - - // Modifies the base as well as diacritic - - { - "TwoBackspacesForNfdConvertsIntoOneCharInNfcAndRecombine", - /* app context pre transform: */ u"abcê", - /* cached context post transform: */ u"abca\u0323\u0302", - /* action del, output: */ 2, U"a\u0323\u0302", // NFD input; delete 2: e\u0302 - // ---- results ---- - /* action del, output: */ 1, U"ậ", // NFC output; delete 1: ê - /* app_context: */ u"abcậ", - /* expected del */ U"ê" - }, - - // surrogate pair tests - - { - "SurrogatePairInContext", - /* app context pre transform: */ u"abc\U0001F607ê", - /* cached context post transform: */ u"abc\U0001F607a\u0323\u0302", - /* action del, output: */ 2, U"a\u0323\u0302", - // ---- results ---- - /* action del, output: */ 1, U"ậ", - /* app_context: */ u"abc\U0001F607ậ", - /* expected del */ U"ê" - }, - - { - "SurrogatePairInOutput", - /* app context pre transform: */ u"abc", - /* cached context post transform: */ u"abc\U0001F607", - /* action del, output: */ 0, U"\U0001F607", - // ---- results ---- - /* action del, output: */ 0, U"\U0001F607", - /* app_context: */ u"abc\U0001F607", - /* expected del */ U"" - }, - - { - "SurrogatePairsInBothContextAndOutput", - /* app context pre transform: */ u"a\U0001F607bcê", - /* cached context post transform: */ u"a\U0001F607bca\U0001F60E", - /* action del, output: */ 2, U"a\U0001F60E", - // ---- results ---- - /* action del, output: */ 1, U"a\U0001F60E", - /* app_context: */ u"a\U0001F607bca\U0001F60E", - /* expected del */ U"ê" - } -}; - -INSTANTIATE_TEST_SUITE_P(KeymanCore, GetActionApiTest, testing::ValuesIn(values), GenerateTestName); +INSTANTIATE_TEST_SUITE_P(KeymanCore, GetActionApiTest, testing::ValuesIn(actionsTestData), GenerateTestName); diff --git a/core/tests/unit/api/actions_normalize.tests.cpp b/core/tests/unit/api/actions_normalize.tests.cpp index 22d6644e61..13aba519d7 100644 --- a/core/tests/unit/api/actions_normalize.tests.cpp +++ b/core/tests/unit/api/actions_normalize.tests.cpp @@ -15,68 +15,15 @@ #include "context.hpp" #include "../helpers/core_test_helpers.h" +#include "./actions_test_data.h" -struct TestData { - const char* test_name; - - /** - * the app context stored in the state, _before_ transform is applied -- NFU - */ - const km_core_cu *initial_app_context; - - /** - * cached context _after_ actions have been applied -- guaranteed NFD - * (essentially, this is initial_cached_context - - * actions_code_points_to_delete + actions_output) - no markers supported - */ - const km_core_cu *final_cached_context_string; - - /** - * cached context _after_ actions have been applied -- guaranteed NFD - * (essentially, this is initial_cached_context - - * actions_code_points_to_delete + actions_output) - markers supported - */ - const km_core_context_item *final_cached_context_items; - - /** - * number of NFD code points that the keyboard processor has asked to remove in its actions - */ - int actions_code_points_to_delete; - - /** - * NFD string that the keyboard processor has asked to insert in its actions - */ - const std::u32string actions_output; - - /** - * expected: NFU code points to ask app to remove - */ - const unsigned int expected_delete; - - /** - * expected: adjusted NFC output to insert into the app - */ - const std::u32string expected_output; - - /** - * expected: NFU adjusted final app context, which will be NFC from the - * boundary of the transform, but will not have been modified prior to that. - * Should match char-for-char what the app ends up with in its text buffer. - */ - const km_core_cu *expected_final_app_context; -}; - -std::string GenerateTestName(const testing::TestParamInfo& info) { - return info.param.test_name; -} - -class ActionsNormalizeApiTest : public testing::TestWithParam { +class ActionsNormalizeApiTest : public testing::TestWithParam { protected: km_core_keyboard * test_kb = nullptr; km_core_state * test_state = nullptr; km_core_actions test_actions = {0}; - void Initialize(TestData const& data) { + void Initialize(ActionsTestData const& data) { km::core::path path = km::core::path::join(test_dir, "..", "ldml", "fixtures", "keyboards", "17.0", "k_001_tiny.kmx"); auto blob = km::tests::load_kmx_file(path.native().c_str()); ASSERT_STATUS_OK(km_core_keyboard_load_from_blob(path.stem().c_str(), blob.data(), blob.size(), &test_kb)); @@ -143,282 +90,4 @@ TEST_P(ActionsNormalizeApiTest, TestActionsNormalize) { ASSERT_NO_FATAL_FAILURE(km::tests::compare_context(km_core_state_app_context(test_state), data.expected_final_app_context)); } -const km_core_context_item items_1[] = { //u"a\U0001F607b\uFFFF\u0008\u0001ca\U0001F60E", - { KM_CORE_CT_CHAR, {0,}, { 0x0061 } }, - { KM_CORE_CT_CHAR, {0,}, { 0x1F607 } }, - { KM_CORE_CT_CHAR, {0,}, { 0x0062 } }, - { KM_CORE_CT_MARKER, {0,}, { 0x1 } }, - { KM_CORE_CT_CHAR, {0,}, { 0x0063 } }, - { KM_CORE_CT_CHAR, {0,}, { 0x0061 } }, - { KM_CORE_CT_CHAR, {0,}, { 0x1F60E } }, - KM_CORE_CONTEXT_ITEM_END -}; - -const km_core_context_item items_2[] = { //u"a\U0001F607bca\U0001F60E\uFFFF\u0008\u0001", - { KM_CORE_CT_CHAR, {0,}, { 0x0061 } }, - { KM_CORE_CT_CHAR, {0,}, { 0x1F607 } }, - { KM_CORE_CT_CHAR, {0,}, { 0x0062 } }, - { KM_CORE_CT_CHAR, {0,}, { 0x0063 } }, - { KM_CORE_CT_CHAR, {0,}, { 0x0061 } }, - { KM_CORE_CT_CHAR, {0,}, { 0x1F60E } }, - { KM_CORE_CT_MARKER, {0,}, { 0x1 } }, - KM_CORE_CONTEXT_ITEM_END -}; - -const km_core_context_item items_11067[] = { - { KM_CORE_CT_CHAR, {0,}, { U'𐒻' } }, - { KM_CORE_CT_CHAR, {0,}, { U'𐒷' } }, - KM_CORE_CONTEXT_ITEM_END -}; - -const TestData values[] = { - // Null boundary tests - { - "Noop", - /* app context pre transform: */ u"", - /* cached context post transform: */ u"", - /* cached context post transform: */ nullptr, - /* action del, output: */ 0, U"", - // ---- results ---- - /* action del, output: */ 0, U"", - /* app_context: */ u"" - }, - - { - "NoOutput", - /* app context pre transform: */ u"abc", - /* cached context post transform: */ u"abc", - /* cached context post transform: */ nullptr, - /* action del, output: */ 0, U"", - // ---- results ---- - /* action del, output: */ 0, U"", - /* app_context: */ u"abc" - }, - - { - "NoContext", - /* app context pre transform: */ u"", - /* cached context post transform: */ u"def", - /* cached context post transform: */ nullptr, - /* action del, output: */ 0, U"def", - // ---- results ---- - /* action del, output: */ 0, U"def", - /* app_context: */ u"def" - }, - - // Simple tests -- no deletions involved - - { - "NoNormalization", - /* app context pre transform: */ u"abc", - /* cached context post transform: */ u"abcdef", - /* cached context post transform: */ nullptr, - /* action del, output: */ 0, U"def", - // ---- results ---- - /* action del, output: */ 0, U"def", - /* app_context: */ u"abcdef" - }, - - { - "OutputToNfcBasic", - /* app context pre transform: */ u"abc", - /* cached context post transform: */ u"abcde\u0300f", - /* cached context post transform: */ nullptr, - /* action del, output: */ 0, U"de\u0300f", - // ---- results ---- - /* action del, output: */ 0, U"dèf", - /* app_context: */ u"abcdèf" - }, - - { - "OutputToNfcHefty", - /* app context pre transform: */ u"abc", - /* cached context post transform: */ u"abcA\u0300" u"e\u0316\u0301" u"\u0073\u0323\u0307" u"\u0041\u030a" u"\U000114B9\U000114B0", - /* cached context post transform: */ nullptr, - /* action del, output: */ 0, U"A\u0300" U"e\u0316\u0301" U"\u0073\u0323\u0307" U"\u0041\u030a" U"\U000114B9\U000114B0", - // ---- results ---- - /* action del, output: */ 0, U"À" U"é̖" U"\u1e69" U"\u00c5" U"\U000114BC", - /* app_context: */ u"abcÀé̖\u1e69\u00c5\U000114BC" - }, - - // Interaction with input context when not on normalization boundary - - { - "BacktrackOneCharacterToCombineAsNfc", - /* app context pre transform: */ u"XYZA", - /* cached context post transform: */ u"XYZA\u0300abc", - /* cached context post transform: */ nullptr, - /* action del, output: */ 0, U"\u0300abc", - // ---- results ---- - /* action del, output: */ 1, U"Àabc", - /* app_context: */ u"XYZÀabc" - }, - - { - "BacktrackEcombCirc2CharsToCombineAsNfc", - /* app context pre transform: */ u"abce\u0302", - /* cached context post transform: */ u"abce\u0323\u0302", - /* cached context post transform: */ nullptr, - /* action del, output: */ 1, U"\u0323\u0302", - // ---- results ---- - /* action del, output: */ 2, U"ệ", - /* app_context: */ u"abcệ" - }, - - { - "OneBackspaceToDeleteLastNfdCharacter15487", - /* app context pre transform: */ u"abcê", // NFC - /* cached context post transform: */ u"abce", - /* cached context post transform: */ nullptr, - /* action del, output: */ 1, U"", // NFD input; delete 1: \u0302 - // ---- results ---- - /* action del, output: */ 1, U"e", // NFC output; delete 1: e - /* app_context: */ u"abce" - }, - - { - "OneBackspaceToDeleteLastNfdCharacterWithNfcAppContext15487", - /* app context pre transform: */ u"abce\u0302", // NFD - /* cached context post transform: */ u"abce", - /* cached context post transform: */ nullptr, - /* action del, output: */ 1, U"", // NFD input; delete 1: \u0302 - // ---- results ---- - /* action del, output: */ 1, U"", // NFC output; delete 1: e - /* app_context: */ u"abce" - }, - - { - "OneBackspaceForNfdConvertsIntoOneCharInNfcAndRecombine", - /* app context pre transform: */ u"abcê", - /* cached context post transform: */ u"abce\u0323\u0302", - /* cached context post transform: */ nullptr, - /* action del, output: */ 1, U"\u0323\u0302", // NFD input; delete 1: \u0302 - // ---- results ---- - /* action del, output: */ 1, U"ệ", // NFC output; delete 1: ê - /* app_context: */ u"abcệ" - }, - - // a\u0300 should not be normalized because it is not otherwise impacted by - // the action. - { - "AvoidEditingTooFarBackInContextWhenFindingNormalizationBoundary", - /* app context pre transform: */ u"a\u0300bcê", - /* cached context post transform: */ u"a\u0300bce\u0323\u0302", - /* cached context post transform: */ nullptr, - /* action del, output: */ 1, U"\u0323\u0302", // NFD input; delete 1: \u0302 - // ---- results ---- - /* action del, output: */ 1, U"ệ", // NFC output; delete 1: ê - /* app_context: */ u"a\u0300bcệ" - }, - - // If we don't reach a normalization boundary, we still should continue to work - { - "NormalizableLettersAtStartOfContext", - /* app context pre transform: */ u"\u0300", - /* cached context post transform: */ u"\u0323\u0300\u0302", - /* cached context post transform: */ nullptr, - /* action del, output: */ 1, U"\u0323\u0300\u0302", // NFD input; - // ---- results ---- - /* action del, output: */ 1, U"\u0323\u0300\u0302", // NFC output is still decomposed because there is no base - /* app_context: */ u"\u0323\u0300\u0302" - }, - - // #15505 - normalization of Bengali characters - { - "BengaliNormalizationOfU09C7U09D7U09CC", - /* app context pre transform: */ u"\u0995\u09C7", - /* cached context post transform: */ u"\u0995\u09C7\u09D7", - /* cached context post transform: */ nullptr, - /* action del, output: */ 0, U"\u09D7", - // ---- results ---- - /* action del, output: */ 1, U"\u09CC", - /* app_context: */ u"\u0995\u09CC" - }, - - // Modifies the base as well as diacritic - - { - "TwoBackspacesForNfdConvertsIntoOneCharInNfcAndRecombine", - /* app context pre transform: */ u"abcê", - /* cached context post transform: */ u"abca\u0323\u0302", - /* cached context post transform: */ nullptr, - /* action del, output: */ 2, U"a\u0323\u0302", // NFD input; delete 2: e\u0302 - // ---- results ---- - /* action del, output: */ 1, U"ậ", // NFC output; delete 1: ê - /* app_context: */ u"abcậ" - }, - - // surrogate pair tests - - { - "SurrogatePairInContext", - /* app context pre transform: */ u"abc\U0001F607ê", - /* cached context post transform: */ u"abc\U0001F607a\u0323\u0302", - /* cached context post transform: */ nullptr, - /* action del, output: */ 2, U"a\u0323\u0302", - // ---- results ---- - /* action del, output: */ 1, U"ậ", - /* app_context: */ u"abc\U0001F607ậ" - }, - - { - "SurrogatePairInOutput", - /* app context pre transform: */ u"abc", - /* cached context post transform: */ u"abc\U0001F607", - /* cached context post transform: */ nullptr, - /* action del, output: */ 0, U"\U0001F607", - // ---- results ---- - /* action del, output: */ 0, U"\U0001F607", - /* app_context: */ u"abc\U0001F607" - }, - - { - "SurrogatePairsInBothContextAndOutput", - /* app context pre transform: */ u"a\U0001F607bcê", - /* cached context post transform: */ u"a\U0001F607bca\U0001F60E", - /* cached context post transform: */ nullptr, - /* action del, output: */ 2, U"a\U0001F60E", - // ---- results ---- - /* action del, output: */ 1, U"a\U0001F60E", - /* app_context: */ u"a\U0001F607bca\U0001F60E" - }, - - // Marker tests - - { - "AMarkerInTheCachedContextShouldNotShowUpInAppContext", - /* app context pre transform: */ u"a\U0001F607bcê", - /* cached context post transform: */ nullptr, - /* cached context post transform: */ &items_1[0], - - /* action del, output: */ 2, U"a\U0001F60E", - // ---- results ---- - /* action del, output: */ 1, U"a\U0001F60E", - /* app_context: */ u"a\U0001F607bca\U0001F60E" - }, - - { - "AMarkerInTheModifiedSectionOfCachedContextShouldNotShowUpInAppContext", - /* app context pre transform: */ u"a\U0001F607bcê", - /* cached context post transform: */ nullptr, - /* cached context post transform: */ &items_2[0], - /* action del, output: */ 2, U"a\U0001F60E", - // ---- results ---- - /* action del, output: */ 1, U"a\U0001F60E", - /* app_context: */ u"a\U0001F607bca\U0001F60E" - }, - - // regression #11067 - { - "ANonBmpCharInContext11067", - /* app context pre transform: */ u"𐒻", - /* cached context post transform: */ u"𐒻𐒷", - /* cached context post transform: */ &items_11067[0], - /* action del, output: */ 0, U"𐒻𐒷", - // ---- results ---- - /* action del, output: */ 1, U"𐒻𐒷", - /* app_context: */ u"𐒻𐒷" - } -}; - -INSTANTIATE_TEST_SUITE_P(KeymanCore, ActionsNormalizeApiTest, testing::ValuesIn(values), GenerateTestName); +INSTANTIATE_TEST_SUITE_P(KeymanCore, ActionsNormalizeApiTest, testing::ValuesIn(actionsTestData), GenerateTestName); diff --git a/core/tests/unit/api/actions_test_data.cpp b/core/tests/unit/api/actions_test_data.cpp new file mode 100644 index 0000000000..2566fe02ce --- /dev/null +++ b/core/tests/unit/api/actions_test_data.cpp @@ -0,0 +1,306 @@ + +#include "./actions_test_data.h" + +const km_core_context_item items_1[] = { //u"a\U0001F607b\uFFFF\u0008\u0001ca\U0001F60E", + { KM_CORE_CT_CHAR, {0,}, { 0x0061 } }, + { KM_CORE_CT_CHAR, {0,}, { 0x1F607 } }, + { KM_CORE_CT_CHAR, {0,}, { 0x0062 } }, + { KM_CORE_CT_MARKER, {0,}, { 0x1 } }, + { KM_CORE_CT_CHAR, {0,}, { 0x0063 } }, + { KM_CORE_CT_CHAR, {0,}, { 0x0061 } }, + { KM_CORE_CT_CHAR, {0,}, { 0x1F60E } }, + KM_CORE_CONTEXT_ITEM_END +}; + +const km_core_context_item items_2[] = { //u"a\U0001F607bca\U0001F60E\uFFFF\u0008\u0001", + { KM_CORE_CT_CHAR, {0,}, { 0x0061 } }, + { KM_CORE_CT_CHAR, {0,}, { 0x1F607 } }, + { KM_CORE_CT_CHAR, {0,}, { 0x0062 } }, + { KM_CORE_CT_CHAR, {0,}, { 0x0063 } }, + { KM_CORE_CT_CHAR, {0,}, { 0x0061 } }, + { KM_CORE_CT_CHAR, {0,}, { 0x1F60E } }, + { KM_CORE_CT_MARKER, {0,}, { 0x1 } }, + KM_CORE_CONTEXT_ITEM_END +}; + +const km_core_context_item items_11067[] = { + { KM_CORE_CT_CHAR, {0,}, { U'𐒻' } }, + { KM_CORE_CT_CHAR, {0,}, { U'𐒷' } }, + KM_CORE_CONTEXT_ITEM_END +}; + +const std::vector actionsTestData = { + // Null boundary tests + + { + "Noop", + /* app context pre transform: */ u"", + /* cached context post transform: */ u"", + /* cached context post transform: */ nullptr, + /* action del, output: */ 0, U"", + // ---- results ---- + /* expected action del, output: */ 0, U"", + /* expected app_context: */ u"", + /* expected del */ U"" + }, + + { + "NoOutput", + /* app context pre transform: */ u"abc", + /* cached context post transform: */ u"abc", + /* cached context post transform: */ nullptr, + /* action del, output: */ 0, U"", + // ---- results ---- + /* expected action del, output: */ 0, U"", + /* expected app_context: */ u"abc", + /* expected del */ U"" + }, + + { + "NoContext", + /* app context pre transform: */ u"", + /* cached context post transform: */ u"def", + /* cached context post transform: */ nullptr, + /* action del, output: */ 0, U"def", + // ---- results ---- + /* expected action del, output: */ 0, U"def", + /* expected app_context: */ u"def", + /* expected del */ U"" + }, + + // Simple tests -- no deletions involved + + { + "NoNormalization", + /* app context pre transform: */ u"abc", + /* cached context post transform: */ u"abcdef", + /* cached context post transform: */ nullptr, + /* action del, output: */ 0, U"def", + // ---- results ---- + /* expected action del, output: */ 0, U"def", + /* expected app_context: */ u"abcdef", + /* expected del */ U"" + }, + + { + "OutputToNfcBasic", + /* app context pre transform: */ u"abc", + /* cached context post transform: */ u"abcde\u0300f", + /* cached context post transform: */ nullptr, + /* action del, output: */ 0, U"de\u0300f", + // ---- results ---- + /* expected action del, output: */ 0, U"dèf", + /* expected app_context: */ u"abcdèf", + /* expected del */ U"" + }, + + { + "OutputToNfcHefty", + /* app context pre transform: */ u"abc", + /* cached context post transform: */ u"abcA\u0300" u"e\u0316\u0301" u"\u0073\u0323\u0307" u"\u0041\u030a" u"\U000114B9\U000114B0", + /* cached context post transform: */ nullptr, + /* action del, output: */ 0, U"A\u0300" U"e\u0316\u0301" U"\u0073\u0323\u0307" U"\u0041\u030a" U"\U000114B9\U000114B0", + // ---- results ---- + /* expected action del, output: */ 0, U"À" U"é̖" U"\u1e69" U"\u00c5" U"\U000114BC", + /* expected app_context: */ u"abcÀé̖\u1e69\u00c5\U000114BC", + /* expected del */ U"" + }, + + // Interaction with input context when not on normalization boundary + + { + "BacktrackOneCharacterToCombineAsNfc", + /* app context pre transform: */ u"XYZA", + /* cached context post transform: */ u"XYZA\u0300abc", + /* cached context post transform: */ nullptr, + /* action del, output: */ 0, U"\u0300abc", + // ---- results ---- + /* expected action del, output: */ 1, U"Àabc", + /* expected app_context: */ u"XYZÀabc", + /* expected del */ U"A" + }, + + { + "BacktrackECombCirc2CharsToCombineAsNfc", + /* app context pre transform: */ u"abce\u0302", + /* cached context post transform: */ u"abce\u0323\u0302", + /* cached context post transform: */ nullptr, + /* action del, output: */ 1, U"\u0323\u0302", + // ---- results ---- + /* expected action del, output: */ 2, U"ệ", + /* expected app_context: */ u"abcệ", + /* expected del */ U"e\u0302" + }, + + { + "OneBackspaceToDeleteLastNfdCharacter15487", + /* app context pre transform: */ u"abcê", // NFC + /* cached context post transform: */ u"abce", + /* cached context post transform: */ nullptr, + /* action del, output: */ 1, U"", // NFD input; delete 1: \u0302 + // ---- results ---- + /* expected action del, output: */ 1, U"e", // NFC output; delete 1: e + /* expected app_context: */ u"abce", + /* expected del */ U"ê" + }, + + { + "OneBackspaceToDeleteLastNfdCharacterWithNfdAppContext15487", + /* app context pre transform: */ u"abce\u0302", // NFD + /* cached context post transform: */ u"abce", + /* cached context post transform: */ nullptr, + /* action del, output: */ 1, U"", // NFD input; delete 1: \u0302 + // ---- results ---- + /* expected action del, output: */ 1, U"", // NFC output; delete 1: e + /* expected app_context: */ u"abce", + /* expected del */ U"\u0302" + }, + + { + "OneBackspaceForNfdConvertsIntoOneCharInNfcAndRecombine", + /* app context pre transform: */ u"abcê", + /* cached context post transform: */ u"abce\u0323\u0302", + /* cached context post transform: */ nullptr, + /* action del, output: */ 1, U"\u0323\u0302", // NFD input; delete 1: \u0302 + // ---- results ---- + /* expected action del, output: */ 1, U"ệ", // NFC output; delete 1: ê + /* expected app_context: */ u"abcệ", + /* expected del */ U"ê" + }, + + // a\u0300 should not be normalized because it is not otherwise impacted by + // the action. + { + "AvoidEditingTooFarBackInContextWhenFindingNormalizationBoundary", + /* app context pre transform: */ u"a\u0300bcê", + /* cached context post transform: */ u"a\u0300bce\u0323\u0302", + /* cached context post transform: */ nullptr, + /* action del, output: */ 1, U"\u0323\u0302", // NFD input; delete 1: \u0302 + // ---- results ---- + /* expected action del, output: */ 1, U"ệ", // NFC output; delete 1: ê + /* expected app_context: */ u"a\u0300bcệ", + /* expected del */ U"ê" + }, + + // If we don't reach a normalization boundary, we still should continue to work + { + "NormalizableLettersAtStartOfContext", + /* app context pre transform: */ u"\u0300", + /* cached context post transform: */ u"\u0323\u0300\u0302", + /* cached context post transform: */ nullptr, + /* action del, output: */ 1, U"\u0323\u0300\u0302", // NFD input; + // ---- results ---- + /* expected action del, output: */ 1, U"\u0323\u0300\u0302", // NFC output is still decomposed because there is no base + /* expected app_context: */ u"\u0323\u0300\u0302", + /* expected del */ U"\u0300" + }, + + // #15505 - normalization of Bengali characters + { + "BengaliNormalizationOfU09C7U09D7U09CC", + /* app context pre transform: */ u"\u0995\u09C7", + /* cached context post transform: */ u"\u0995\u09C7\u09D7", + /* cached context post transform: */ nullptr, + /* action del, output: */ 0, U"\u09D7", + // ---- results ---- + /* expected action del, output: */ 1, U"\u09CC", + /* expected app_context: */ u"\u0995\u09CC", + /* expected del */ U"\u09C7" + }, + + // Modifies the base as well as diacritic + + { + "TwoBackspacesForNfdConvertsIntoOneCharInNfcAndRecombine", + /* app context pre transform: */ u"abcê", + /* cached context post transform: */ u"abca\u0323\u0302", + /* cached context post transform: */ nullptr, + /* action del, output: */ 2, U"a\u0323\u0302", // NFD input; delete 2: e\u0302 + // ---- results ---- + /* expected action del, output: */ 1, U"ậ", // NFC output; delete 1: ê + /* expected app_context: */ u"abcậ", + /* expected del */ U"ê" + }, + + // surrogate pair tests + + { + "SurrogatePairInContext", + /* app context pre transform: */ u"abc\U0001F607ê", + /* cached context post transform: */ u"abc\U0001F607a\u0323\u0302", + /* cached context post transform: */ nullptr, + /* action del, output: */ 2, U"a\u0323\u0302", + // ---- results ---- + /* expected action del, output: */ 1, U"ậ", + /* expected app_context: */ u"abc\U0001F607ậ", + /* expected del */ U"ê" + }, + + { + "SurrogatePairInOutput", + /* app context pre transform: */ u"abc", + /* cached context post transform: */ u"abc\U0001F607", + /* cached context post transform: */ nullptr, + /* action del, output: */ 0, U"\U0001F607", + // ---- results ---- + /* expected action del, output: */ 0, U"\U0001F607", + /* expected app_context: */ u"abc\U0001F607", + /* expected del */ U"" + }, + + { + "SurrogatePairsInBothContextAndOutput", + /* app context pre transform: */ u"a\U0001F607bcê", + /* cached context post transform: */ u"a\U0001F607bca\U0001F60E", + /* cached context post transform: */ nullptr, + /* action del, output: */ 2, U"a\U0001F60E", + // ---- results ---- + /* expected action del, output: */ 1, U"a\U0001F60E", + /* expected app_context: */ u"a\U0001F607bca\U0001F60E", + /* expected del */ U"ê" + }, + + // Marker tests + + { + "AMarkerInTheCachedContextShouldNotShowUpInAppContext", + /* app context pre transform: */ u"a\U0001F607bcê", + /* cached context post transform: */ nullptr, + /* cached context post transform: */ &items_1[0], + + /* action del, output: */ 2, U"a\U0001F60E", + // ---- results ---- + /* expected action del, output: */ 1, U"a\U0001F60E", + /* expected app_context: */ u"a\U0001F607bca\U0001F60E", + /* expected del */ U"ê" + }, + + { + "AMarkerInTheModifiedSectionOfCachedContextShouldNotShowUpInAppContext", + /* app context pre transform: */ u"a\U0001F607bcê", + /* cached context post transform: */ nullptr, + /* cached context post transform: */ &items_2[0], + /* action del, output: */ 2, U"a\U0001F60E", + // ---- results ---- + /* expected action del, output: */ 1, U"a\U0001F60E", + /* expected app_context: */ u"a\U0001F607bca\U0001F60E", + /* expected del */ U"ê" + }, + + // regression #11067 + { + "ANonBmpCharInContext11067", + /* app context pre transform: */ u"𐒻", + /* cached context post transform: */ u"𐒻𐒷", + /* cached context post transform: */ &items_11067[0], + /* action del, output: */ 0, U"𐒻𐒷", + // ---- results ---- + /* expected action del, output: */ 1, U"𐒻𐒷", + /* expected app_context: */ u"𐒻𐒷", + /* expected del: */ U"\x104BB" + } +}; + +std::string GenerateTestName(const testing::TestParamInfo& info) { + return info.param.test_name; +} diff --git a/core/tests/unit/api/actions_test_data.h b/core/tests/unit/api/actions_test_data.h new file mode 100644 index 0000000000..92b8db5108 --- /dev/null +++ b/core/tests/unit/api/actions_test_data.h @@ -0,0 +1,64 @@ +#pragma once +#include +#include +#include +#include "keyman_core.h" + +struct ActionsTestData { + const char* test_name; + + /** + * the app context stored in the state, _before_ transform is applied -- NFU + */ + const km_core_cu *initial_app_context; + + /** + * cached context _after_ actions have been applied -- guaranteed NFD + * (essentially, this is initial_cached_context - + * actions_code_points_to_delete + actions_output) - no markers supported + */ + const km_core_cu *final_cached_context_string; + + /** + * cached context _after_ actions have been applied -- guaranteed NFD + * (essentially, this is initial_cached_context - + * actions_code_points_to_delete + actions_output) - markers supported + */ + const km_core_context_item *final_cached_context_items; + + /** + * number of NFD code points that the keyboard processor has asked to remove in its actions + */ + int actions_code_points_to_delete; + + /** + * NFD string that the keyboard processor has asked to insert in its actions + */ + const std::u32string actions_output; + + /** + * expected: NFU code points to ask app to remove + */ + const unsigned int expected_delete; + + /** + * expected: adjusted NFC output to insert into the app + */ + const std::u32string expected_output; + + /** + * expected: NFU adjusted final app context, which will be NFC from the + * boundary of the transform, but will not have been modified prior to that. + * Should match char-for-char what the app ends up with in its text buffer. + */ + const km_core_cu *expected_final_app_context; + + /** + * expected: the characters deleted from the context + */ + const std::u32string expected_deleted_context; +}; + +extern const std::vector actionsTestData; + +std::string GenerateTestName(const testing::TestParamInfo& info); diff --git a/core/tests/unit/api/actions_update_app_context_nfu.tests.cpp b/core/tests/unit/api/actions_update_app_context_nfu.tests.cpp index 36330daea7..bf98601c34 100644 --- a/core/tests/unit/api/actions_update_app_context_nfu.tests.cpp +++ b/core/tests/unit/api/actions_update_app_context_nfu.tests.cpp @@ -15,72 +15,15 @@ #include "context.hpp" #include "../helpers/core_test_helpers.h" +#include "./actions_test_data.h" -// TODO-WEB-CORE: merge with actions_normalize.tests.cpp? These are identical; -// note that actions_get_api and actions_set_api data are subtly different, -// opportunity to merge those too? - -struct TestData { - const char* test_name; - - /** - * the app context stored in the state, _before_ transform is applied -- NFU - */ - const km_core_cu *initial_app_context; - - /** - * cached context _after_ actions have been applied -- guaranteed NFD - * (essentially, this is initial_cached_context - - * actions_code_points_to_delete + actions_output) - no markers supported - */ - const km_core_cu *final_cached_context_string; - - /** - * cached context _after_ actions have been applied -- guaranteed NFD - * (essentially, this is initial_cached_context - - * actions_code_points_to_delete + actions_output) - markers supported - */ - const km_core_context_item *final_cached_context_items; - - /** - * number of NFD code points that the keyboard processor has asked to remove in its actions - */ - int actions_code_points_to_delete; - - /** - * NFD string that the keyboard processor has asked to insert in its actions - */ - const std::u32string actions_output; - - /** - * expected: NFU code points to ask app to remove - */ - const unsigned int expected_delete; - - /** - * expected: adjusted NFC output to insert into the app - */ - const std::u32string expected_output; - - /** - * expected: NFU adjusted final app context, which will be NFC from the - * boundary of the transform, but will not have been modified prior to that. - * Should match char-for-char what the app ends up with in its text buffer. - */ - const km_core_cu *expected_final_app_context; -}; - -std::string GenerateTestName(const testing::TestParamInfo& info) { - return info.param.test_name; -} - -class ActionsUpdateAppContextNfuApiTest : public testing::TestWithParam { +class ActionsUpdateAppContextNfuApiTest : public testing::TestWithParam { protected: km_core_keyboard * test_kb = nullptr; km_core_state * test_state = nullptr; km_core_actions test_actions = {0}; - void Initialize(TestData const& data) { + void Initialize(ActionsTestData const& data) { km::core::path path = km::core::path::join(test_dir, "..", "ldml", "fixtures", "keyboards", "17.0", "k_001_tiny.kmx"); auto blob = km::tests::load_kmx_file(path.native().c_str()); ASSERT_STATUS_OK(km_core_keyboard_load_from_blob(path.stem().c_str(), blob.data(), blob.size(), &test_kb)); @@ -172,7 +115,23 @@ const km_core_context_item items_2[] = { //u"a\U0001F607bca\U0001F60E\uFFFF\u000 KM_CORE_CONTEXT_ITEM_END }; -const TestData values[] = { +const km_core_context_item items_11067[] = { + { KM_CORE_CT_CHAR, {0,}, { U'𐒻' } }, + { KM_CORE_CT_CHAR, {0,}, { U'𐒷' } }, + KM_CORE_CONTEXT_ITEM_END +}; + +/** + * Test cases for non-normalized app context update. These look superficially + * similar to the data in `actionsTestData`, but diverge when taking into + * account normalization of output. Several normalization-related tests in + * `actionsTestData` are therefore also currently excluded here. + * + * @todo: (low priority) `actionsTestData` and `actionsTestDataNFU` could be + * combined with additional entries for each NFU result. + */ + +const std::vector actionsTestDataNFU = { // Null boundary tests { @@ -182,8 +141,9 @@ const TestData values[] = { /* cached context post transform: */ nullptr, /* action del, output: */ 0, U"", // ---- results ---- - /* action del, output: */ 0, U"", - /* app_context: */ u"" + /* expected action del, output: */ 0, U"", + /* expected app_context: */ u"", + /* expected del */ U"" }, { @@ -193,8 +153,9 @@ const TestData values[] = { /* cached context post transform: */ nullptr, /* action del, output: */ 0, U"", // ---- results ---- - /* action del, output: */ 0, U"", - /* app_context: */ u"abc" + /* expected action del, output: */ 0, U"", + /* expected app_context: */ u"abc", + /* expected del */ U"" }, { @@ -204,10 +165,26 @@ const TestData values[] = { /* cached context post transform: */ nullptr, /* action del, output: */ 0, U"def", // ---- results ---- - /* action del, output: */ 0, U"def", - /* app_context: */ u"def" + /* expected action del, output: */ 0, U"def", + /* expected app_context: */ u"def", + /* expected del */ U"" }, + // Simple tests -- no deletions involved + + { + "NoNormalization", + /* app context pre transform: */ u"abc", + /* cached context post transform: */ u"abcdef", + /* cached context post transform: */ nullptr, + /* action del, output: */ 0, U"def", + // ---- results ---- + /* expected action del, output: */ 0, U"def", + /* expected app_context: */ u"abcdef", + /* expected del */ U"" + }, + + // surrogate pair tests { @@ -217,8 +194,9 @@ const TestData values[] = { /* cached context post transform: */ nullptr, /* action del, output: */ 1, U"a\u0323\u0302", // ---- results ---- - /* action del, output: */ 1, U"a\u0323\u0302", - /* app_context: */ u"abc\U0001F607a\u0323\u0302" + /* expected action del, output: */ 1, U"a\u0323\u0302", + /* expected app_context: */ u"abc\U0001F607a\u0323\u0302", + /* expected del */ U"ê" }, { @@ -228,8 +206,9 @@ const TestData values[] = { /* cached context post transform: */ nullptr, /* action del, output: */ 0, U"\U0001F607", // ---- results ---- - /* action del, output: */ 0, U"\U0001F607", - /* app_context: */ u"abc\U0001F607" + /* expected action del, output: */ 0, U"\U0001F607", + /* expected app_context: */ u"abc\U0001F607", + /* expected del */ U"" }, { @@ -239,8 +218,9 @@ const TestData values[] = { /* cached context post transform: */ nullptr, /* action del, output: */ 1, U"a\U0001F60E", // ---- results ---- - /* action del, output: */ 1, U"a\U0001F60E", - /* app_context: */ u"a\U0001F607bca\U0001F60E" + /* expected action del, output: */ 1, U"a\U0001F60E", + /* expected app_context: */ u"a\U0001F607bca\U0001F60E", + /* expected del */ U"ê" }, // Marker tests @@ -252,8 +232,9 @@ const TestData values[] = { /* cached context post transform: */ &items_1[0], /* action del, output: */ 1, U"a\U0001F60E", // ---- results ---- - /* action del, output: */ 1, U"a\U0001F60E", - /* app_context: */ u"a\U0001F607bca\U0001F60E" + /* expected action del, output: */ 1, U"a\U0001F60E", + /* expected app_context: */ u"a\U0001F607bca\U0001F60E", + /* expected del */ U"ê" }, { @@ -263,9 +244,23 @@ const TestData values[] = { /* cached context post transform: */ &items_2[0], /* action del, output: */ 1, U"a\U0001F60E", // ---- results ---- - /* action del, output: */ 1, U"a\U0001F60E", - /* app_context: */ u"a\U0001F607bca\U0001F60E" + /* expected action del, output: */ 1, U"a\U0001F60E", + /* expected app_context: */ u"a\U0001F607bca\U0001F60E", + /* expected del */ U"ê" + }, + + // regression #11067 + { + "ANonBmpCharInContext11067", + /* app context pre transform: */ u"𐒻", + /* cached context post transform: */ u"𐒻𐒷", + /* cached context post transform: */ &items_11067[0], + /* action del, output: */ 1, U"𐒻𐒷", + // ---- results ---- + /* expected action del, output: */ 1, U"𐒻𐒷", + /* expected app_context: */ u"𐒻𐒷", + /* expected del: */ U"\x104BB" } }; -INSTANTIATE_TEST_SUITE_P(KeymanCore, ActionsUpdateAppContextNfuApiTest, testing::ValuesIn(values), GenerateTestName); +INSTANTIATE_TEST_SUITE_P(KeymanCore, ActionsUpdateAppContextNfuApiTest, testing::ValuesIn(actionsTestDataNFU), GenerateTestName); diff --git a/core/tests/unit/api/meson.build b/core/tests/unit/api/meson.build index 97e04b5002..04d2802d6e 100644 --- a/core/tests/unit/api/meson.build +++ b/core/tests/unit/api/meson.build @@ -40,15 +40,15 @@ tests = [ ['debug-api-tests', 'debug_api.tests.cpp'], ['kmx_xstring-tests', 'kmx_xstring.tests.cpp'], ['kmx_context-tests', 'kmx_context.tests.cpp'], - ['actions_normalize-tests', 'actions_normalize.tests.cpp'], + ['actions_normalize-tests', ['actions_normalize.tests.cpp', 'actions_test_data.cpp']], # renamed to avoid heuristic that causes "update" in a filename to require # elevation on Windows! this could also be avoided by adding a manifest # resource to each executable (e.g. with mt.exe) # https://learn.microsoft.com/en-us/windows/security/application-security/application-control/user-account-control/architecture#installer-detection-technology - ['actions_updatx_app_context_nfu-tests', 'actions_update_app_context_nfu.tests.cpp'], + ['actions_updatx_app_context_nfu-tests', ['actions_update_app_context_nfu.tests.cpp', 'actions_test_data.cpp']], - ['actions_get_api-tests', 'actions_get_api.tests.cpp'], + ['actions_get_api-tests', ['actions_get_api.tests.cpp', 'actions_test_data.cpp']], ['km_core_keyboard_api-tests', 'km_core_keyboard_api.tests.cpp'], ['km_core_process_event-tests', 'km_core_process_event.tests.cpp'], ['key_list-tests', 'kmx_key_list.tests.cpp'], From e0fd08ca0fba4a1d96d42ff966c832855655b21c Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Thu, 7 May 2026 15:17:08 +0200 Subject: [PATCH 8/8] chore(core): cleanup leaks in tests and skipped tests --- core/tests/unit/api/actions_get_api.tests.cpp | 5 ++--- core/tests/unit/api/state_context_api.tests.cpp | 4 ++++ core/tests/unit/ldml/transforms.tests.cpp | 1 - 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/core/tests/unit/api/actions_get_api.tests.cpp b/core/tests/unit/api/actions_get_api.tests.cpp index 8d975a94f1..9ccc139336 100644 --- a/core/tests/unit/api/actions_get_api.tests.cpp +++ b/core/tests/unit/api/actions_get_api.tests.cpp @@ -68,6 +68,7 @@ protected: } if(test_actions) { delete [] test_actions->output; + delete [] test_actions->persist_options; delete test_actions; test_actions = nullptr; } @@ -86,12 +87,10 @@ TEST_P(GetActionApiTest, TestActionsApi) { ASSERT_NO_FATAL_FAILURE(Initialize(data)); if(data.final_cached_context_string == nullptr) { - // We skip the shared tests that don't have a input context string + GTEST_SKIP() << "Skip the shared tests that don't have a input context string"; return; } - // setup(initial_app_context, final_cached_context, actions_code_points_to_delete, actions_output); - auto actual_actions = km_core_state_get_actions(test_state); std::cout << " (" << data.test_name << "): delete: " << data.expected_delete << " output: |" << std::u32string(actual_actions->output) << "|" << std::endl; diff --git a/core/tests/unit/api/state_context_api.tests.cpp b/core/tests/unit/api/state_context_api.tests.cpp index 946fc1700f..59c94c3a8c 100644 --- a/core/tests/unit/api/state_context_api.tests.cpp +++ b/core/tests/unit/api/state_context_api.tests.cpp @@ -66,6 +66,8 @@ void assert_identical_context(const km_core_context *actual_context, km_core_cu ASSERT_STATUS_OK(context_items_to_utf16(actual_context_items, nullptr, &buf_size)); km_core_cu *actual_context_cu = new km_core_cu[buf_size]; ASSERT_STATUS_OK(context_items_to_utf16(actual_context_items, actual_context_cu, &buf_size)); + km_core_context_items_dispose(actual_context_items); + ASSERT_EQ(actual_context_cu, std::u16string(expected_context)); delete[] actual_context_cu; } @@ -77,6 +79,8 @@ void assert_different_context(const km_core_context *actual_context, km_core_cu ASSERT_STATUS_OK(context_items_to_utf16(actual_context_items, nullptr, &buf_size)); km_core_cu *actual_context_cu = new km_core_cu[buf_size]; ASSERT_STATUS_OK(context_items_to_utf16(actual_context_items, actual_context_cu, &buf_size)); + km_core_context_items_dispose(actual_context_items); + ASSERT_NE(actual_context_cu, std::u16string(expected_context)); delete[] actual_context_cu; } diff --git a/core/tests/unit/ldml/transforms.tests.cpp b/core/tests/unit/ldml/transforms.tests.cpp index 4dbdcb73e8..422d21fb63 100644 --- a/core/tests/unit/ldml/transforms.tests.cpp +++ b/core/tests/unit/ldml/transforms.tests.cpp @@ -1017,7 +1017,6 @@ TEST(TransformsTests, TestNormalizeMarkerBeforeDoubleGreek) { } void TEST_NFD_PLAIN(std::u32string x, std::u32string y) { - marker_map map; const std::u32string src = x; const std::u32string expect_nfd = y; std::u32string dst_nfd = src;