/* * Keyman is copyright (C) SIL International. MIT License. * * Created by Steven R. Loomis on 2024-05-03 * * Keyman Core - Tests for versions of Unicode resources */ #include #include #include // Ensure that ICU gets included even on wasm. #define KMN_IN_LDML_TESTS #include "keyman_core.h" #include "path.hpp" #include "action.hpp" #include "json.hpp" #include "../helpers/core_test_helpers.h" #include "core_icu.h" #include #include #include "util_normalize.hpp" #include "kmx/kmx_xstring.h" #ifdef __EMSCRIPTEN__ // Pull this in to verify versions #include "util_normalize_table.h" #endif //------------------------------------------------------------------------------------- // Unicode version tests //------------------------------------------------------------------------------------- /** * Load a .json file into a json object * @param jsonpath path to the .json file * @param out json object */ bool load_json(const km::core::path &jsonpath, nlohmann::json &out) { std::cout << "== " << __FUNCTION__ << " loading " << jsonpath << std::endl; std::ifstream json_file(jsonpath.native()); if (!json_file) { std::cerr << "ERROR Could not load: " << jsonpath << std::endl; return false; } out = nlohmann::json::parse(json_file); return !out.empty(); } /** @param out the major version of 'ver', skipping initial '^'. empty on err */ void get_major(const std::string& ver, std::string& out) { ASSERT_FALSE(ver.empty()); auto start = 0; // skip leading '^' if (ver[start] == '^') { start++; } // find first '.' auto end = ver.find('.', start); ASSERT_NE(end, std::string::npos); out = ver.substr(start, end - start); } /** * @return the Unicode version from a Blocks.txt file, such as `15.1.0` */ void get_block_unicode_ver(km::core::path const & blocks_path, std::string& result) { // open Blocks.txt std::ifstream blocks_file(blocks_path.native()); ASSERT_TRUE(blocks_file.good()); std::string block_line; ASSERT_TRUE(std::getline(blocks_file, block_line)); // first line // The first line is something such as '# Blocks-15.1.0.txt' // We skip the prefix, and then stop before the suffix const std::string prefix = "# Blocks-"; const std::string txt_suffix = ".txt"; // find and skip the prefix - "15.1.0.txt" ASSERT_TRUE(block_line.length() > prefix.length()); result = block_line.substr(prefix.length()); // "15.1.0" // find and trim before the suffix auto txt_pos = result.find(txt_suffix, 0); ASSERT_NE(txt_pos, std::string::npos); result.resize(txt_pos); } TEST(UnicodeTests, TestUnicodeVersions) { km::core::path jsonpath = test_dir / "nodeversions.json"; km::core::path packagepath = test_dir / "package.json"; km::core::path blockspath = test_dir / "Blocks.txt"; // load the dump of node's process.versions which the meson.build file generated nlohmann::json versions, package; ASSERT_TRUE(load_json(jsonpath, versions)); // load our top level package.json ASSERT_TRUE(load_json(packagepath, package)); std::string block_unicode_ver; ASSERT_NO_FATAL_FAILURE(get_block_unicode_ver(blockspath, block_unicode_ver)); // 'raw' versions const std::string cxx_icu_unicode(U_UNICODE_VERSION); const std::string cxx_icu(U_ICU_VERSION); const std::string node_engine(package["engines"]["node"].template get()); const std::string node_icu_unicode(versions["unicode"].template get()); const std::string node_icu(versions["icu"].template get()); const std::string node(versions["node"].template get()); // calculated versions std::string block_ver_major ; ASSERT_NO_FATAL_FAILURE(get_major(block_unicode_ver, block_ver_major)); std::string cxx_icu_major ; ASSERT_NO_FATAL_FAILURE(get_major(cxx_icu, cxx_icu_major)); std::string cxx_icu_unicode_major ; ASSERT_NO_FATAL_FAILURE(get_major(cxx_icu_unicode, cxx_icu_unicode_major)); std::string node_engine_major ; ASSERT_NO_FATAL_FAILURE(get_major(node_engine, node_engine_major)); std::string node_icu_major ; ASSERT_NO_FATAL_FAILURE(get_major(node_icu, node_icu_major)); std::string node_icu_unicode_major ; ASSERT_NO_FATAL_FAILURE(get_major(node_icu_unicode, node_icu_unicode_major)); std::string node_major ; ASSERT_NO_FATAL_FAILURE(get_major(node, node_major)); // macro to output string value std::cout << "ICU Versions:" << std::endl; std::cout << "* " << cxx_icu << "\t" << "..linked from C++" << std::endl; std::cout << "* " << node_icu << "\t" << "..in Node.js" << std::endl; std::cout << std::endl; std::cout << "Unicode Versions:" << std::endl; std::cout << "* " << cxx_icu_unicode << "\t" << "..in ICU linked from C++" << std::endl; std::cout << "* " << node_icu_unicode << "\t" << "..in ICU in Node.js" << std::endl; std::cout << "* " << block_unicode_ver << "\t" << "..in Keyman repo Blocks.txt" << std::endl; std::cout << std::endl; std::cout << "Node.js" << std::endl; std::cout << "* " << versions["node"] << "\t" << "Actual version of Node.js" << std::endl; std::cout << "* " << node_engine << "\t" << "Version of Node.js requested by package.json" << std::endl; std::cout << std::endl; // ---- tests ------ // allow the Node.js version to be >= required auto node_engine_num = std::atoi(node_engine_major.c_str()); auto node_num = std::atoi(node_major.c_str()); ASSERT_GE(node_num, node_engine_num); // the cxx_icu can come from the Ubuntu environment, so do not depend on it // for now. // TODO: Resolve with ICU4C 76 in #12398 //assert_basic_equal(node_icu_unicode_major, cxx_icu_unicode_major); //assert_basic_equal(node_icu_unicode_major, block_ver_major); // seems less important if the C++ ICU verison matches the Node.js ICU version. //assert_basic_equal(cxx_icu_major, node_icu_major); std::cout << "All OK!" << std::endl; std::cout << std::endl; } #ifdef __EMSCRIPTEN__ inline const char *boolstr(bool b) { return b?"T":"f"; } #endif TEST(UnicodeTests, TestHasBoundaryBefore) { std::cout << "= " << __FUNCTION__ << std::endl; // Latin - #15505 ASSERT_TRUE(km::core::util::has_nfc_boundary_before(0x0065)); ASSERT_FALSE(km::core::util::has_nfc_boundary_before(0x0301)); // Bengali - #15505 ASSERT_TRUE(km::core::util::has_nfc_boundary_before(0x0995)); ASSERT_FALSE(km::core::util::has_nfc_boundary_before(0x09d7)); #ifdef __EMSCRIPTEN__ std::cout << "I see we are on Emscripten / wasm! Now we will do some additional tests." << std::endl; std::string icu4c_unicode(U_UNICODE_VERSION), header_unicode(KM_HASBOUNDARYBEFORE_UNICODE_VERSION), icu4c_icu(U_ICU_VERSION), header_icu(KM_HASBOUNDARYBEFORE_ICU_VERSION); std::cout << "Unicode: " << U_UNICODE_VERSION << ", and from the table file: " << KM_HASBOUNDARYBEFORE_UNICODE_VERSION << std::endl; std::cout << "It would be very strange for these versions to be out of sync. Some sort of build or tool problem." << std::endl; ASSERT_EQ(icu4c_unicode, header_unicode); ASSERT_EQ(icu4c_icu, header_icu); std::cout << std::endl << "Now, let's make sure has_nfc_boundary_before() matches ICU." << std::endl; UErrorCode status = U_ZERO_ERROR; const icu::Normalizer2 *nfc = icu::Normalizer2::getNFCInstance(status); UASSERT_SUCCESS(status); // now, test that hasBoundaryBefore is the same for (km_core_usv cp = 0; cp < km::core::kmx::Uni_MAX_CODEPOINT; cp++) { auto km_hbb = km::core::util::has_nfc_boundary_before(cp); auto icu_hbb = nfc->hasBoundaryBefore(cp); if (km_hbb != icu_hbb) { std::cerr << "Error: util_normalize_table.h said " << boolstr(km_hbb) << " but ICU said " << boolstr(icu_hbb) << " for " << "has_nfc_boundary_before(0x" << std::hex << cp << std::dec << ")" << std::endl; } ASSERT_EQ(km_hbb, icu_hbb); } #endif std::cout << "All OK!" << std::endl; }