diff --git a/core/src/kmx/kmx_xstring.h b/core/src/kmx/kmx_xstring.h index e6dc1c7b0d..56547e56eb 100644 --- a/core/src/kmx/kmx_xstring.h +++ b/core/src/kmx/kmx_xstring.h @@ -98,6 +98,39 @@ Utf32CharToUtf16(const KMX_DWORD ch32, char16_single &ch16) { return len; } +/** + * Convert a u16 string to a u32 string. U+FFFD instead of mismatched surrogates or sliced surrogate + * @param source UTF-16 string + * @return a UTF-32 string + */ +inline std::u32string +u16string_to_u32string(const std::u16string &source) { + std::u32string out; + + for (auto ptr = source.begin(); ptr < source.end(); ptr++) { + const char16_t lead = *ptr; + if (Uni_IsSurrogate1(lead)) { + ptr++; + if (ptr == source.end()) { + // DebugLog("End of string during surrogate pair"); + out.push_back(0xFFFD); // error + return out; + } + const char16_t trail = *ptr; + if (!Uni_IsSurrogate2(trail)) { + out.push_back(0xFFFD); // error, mismatched lead surrogate + ptr--; // reprocess remaining char + } else { + out.push_back(Uni_SurrogateToUTF32(lead, trail)); + } + } else if (Uni_IsSurrogate2(lead)) { + out.push_back(0xFFFD); // error - mismatched trail surrogate + } else { + out.push_back(lead); + } + } + return out; +} } // namespace kmx } // namespace kbp diff --git a/core/tests/unit/kmnkbd/test_kmx_xstring.cpp b/core/tests/unit/kmnkbd/test_kmx_xstring.cpp index 93ee46914e..5ae5628a19 100644 --- a/core/tests/unit/kmnkbd/test_kmx_xstring.cpp +++ b/core/tests/unit/kmnkbd/test_kmx_xstring.cpp @@ -1268,6 +1268,56 @@ test_utf32() { assert(s1 == std::u16string(u"🙀")); } +void +test_u16string_to_u32string() { + // normal cases + { + const std::u32string str = u16string_to_u32string(u""); + assert_equal(str.length(), 0); + } + { + const std::u32string str = u16string_to_u32string(u"e"); + assert_equal(str.length(), 1); + assert_equal(str.at(0), 0x0065); + } + { + const std::u32string str = u16string_to_u32string(u"🙀"); + assert_equal(str.length(), 1); + assert_equal(str.at(0), 0x0001F640); + } + { + const std::u32string str = u16string_to_u32string(u"Ω🙀"); + assert_equal(str.length(), 2); + assert_equal(str.at(0), u'Ω'); + assert_equal(str.at(1), 0x0001F640); + } + + // error cases + { + std::u16string half_cat; + half_cat.push_back(0xD83D); // mismatched lead surrogate + const std::u32string str = u16string_to_u32string(half_cat); + assert_equal(str.length(), 1); + assert_equal(str.at(0), 0xFFFD); + } + { + std::u16string half_cat; + half_cat.push_back(0xD83D); // mismatched lead surrogate + half_cat.push_back(u'Ω'); // with following text + const std::u32string str = u16string_to_u32string(half_cat); + assert_equal(str.length(), 2); + assert_equal(str.at(0), 0xFFFD); + assert_equal(str.at(1), u'Ω'); + } + { + std::u16string half_cat; + half_cat.push_back(0xDE40); // mismatched trail surrogate + const std::u32string str = u16string_to_u32string(half_cat); + assert_equal(str.length(), 1); + assert_equal(str.at(0), 0xFFFD); + } +} + constexpr const auto help_str = u"\ test_kmx_xstring [--color]\n\ \n\ @@ -1289,6 +1339,7 @@ int main(int argc, char *argv []) { test_xstrlen(); test_xstrlen_ignoreifopt(); test_utf32(); + test_u16string_to_u32string(); return 0; }