diff --git a/core/build.sh b/core/build.sh index ad133ac33a..168cc1cbcf 100755 --- a/core/build.sh +++ b/core/build.sh @@ -68,7 +68,6 @@ Libraries will be built in 'build///src'. "uninstall uninstall libraries from current system" \ "${archtargets[@]}" \ "--no-tests do not configure tests (used by other projects)" \ - "--update-unicode rebuild tables if the ICU4C Unicode version changes" \ "--test,-t=opt_tests test[s] to run (space separated)" builder_parse "$@" @@ -178,12 +177,6 @@ if builder_start_action test:mac; then builder_finish_action success test:mac fi -if builder_has_option --update-unicode; then - # TODO: only works under mac. What's the right way to get the arch here? - meson compile -C ${KEYMAN_ROOT}/core/build/mac-x86_64/$BUILDER_CONFIGURATION tools/norm_data && mv -v ${KEYMAN_ROOT}/core/build/mac-x86_64/$BUILDER_CONFIGURATION/tools/nfd_table.h ${KEYMAN_ROOT}/resources/standards-data/unicode-character-database/ -fi - - # ------------------------------------------------------------------------------- do_action install diff --git a/core/meson.build b/core/meson.build index ff306991e9..0049c95741 100644 --- a/core/meson.build +++ b/core/meson.build @@ -44,4 +44,3 @@ subdir('doc') subdir('include') subdir('src') subdir('tests') -subdir('tools') diff --git a/core/src/meson.build b/core/src/meson.build index 5632c22257..a4160842eb 100644 --- a/core/src/meson.build +++ b/core/src/meson.build @@ -40,6 +40,29 @@ if icu_uc.found() defns += '-DHAVE_ICU4C' endif +# On wasm, generate util_normalize_table.h automatically from ICU +generated_headers = [] + +if cpp_compiler.get_id() == 'emscripten' + +util_normalize_table_generator = executable('util_normalize_table_generator', + ['util_normalize_table_generator.cpp'], + cpp_args: defns + warns, + include_directories: [inc], + link_args: links, + dependencies: [icu_uc, icu_i18n], + ) + +util_normalize_table_h = custom_target('util_normalize_table.h', + output: 'util_normalize_table.h', + command: [util_normalize_table_generator], + capture:true) + +generated_headers += util_normalize_table_h + + +endif + kmx_files = files( 'actions_normalize.cpp', @@ -111,6 +134,7 @@ lib = library('keymancore', kmx_files, mock_files, version_res, + generated_headers, cpp_args: defns + warns + flags, link_args: links, version: lib_version, diff --git a/core/src/util_normalize.cpp b/core/src/util_normalize.cpp index 7d0d76d8ab..947dbe0f85 100644 --- a/core/src/util_normalize.cpp +++ b/core/src/util_normalize.cpp @@ -31,7 +31,7 @@ EM_JS(char*, NormalizeNFC, (const char* input), { }); // pull in the generated table -#include "../../resources/standards-data/unicode-character-database/nfd_table.h" +#include "util_normalize_table.h" #endif diff --git a/core/src/util_normalize_table_generator.cpp b/core/src/util_normalize_table_generator.cpp new file mode 100644 index 0000000000..b5895a588b --- /dev/null +++ b/core/src/util_normalize_table_generator.cpp @@ -0,0 +1,108 @@ +/* + Copyright: © SIL International. + Description: Generator for util_normalize_table.h + Create Date: 5 Jun 2024 + Authors: Steven R. Loomis + + util_normalize_table.h is used under wasm by utilities in util_normalize.cpp to implement + normalization functions without needing ICU4C linked. + + This generator is invoked automatically by meson as part of the build. +*/ + +#include "kmx/kmx_plus.h" +#include "kmx/kmx_xstring.h" + +#define KMN_NO_ICU 0 // we will need ICU.. + +#include "core_icu.h" + +#include +#include +#include +#include + +#include + +#include + + + +int +write_nfd_table() { +#ifndef __EMSCRIPTEN__ + std::cerr << "Note: This is unusual - this generator is usually only run under emscripten!" << std::endl; +#endif + + // We write to stdout instead of to a file to avoid dealing with the filesystem under emscripten. + + std::cerr << "Writing to stdout." << std::endl; + + // write preamble + std::cout << "// GENERATED FILE: DO NOT EDIT" << std::endl; + std::cout << "//" << std::endl; + std::cout << "// util_normalize_table.h is generated by util_normalize_table_generator.cpp" << std::endl; + std::cout << "// and used by util_normalize.cpp" << std::endl; + std::cout << std::endl; + std::cout << "#pragma once" << std::endl; + std::cout << "#define KM_HASBOUNDARYBEFORE_UNICODE_VERSION \"" << U_UNICODE_VERSION << "\"" << std::endl; + std::cout << "#define KM_HASBOUNDARYBEFORE_ICU_VERSION \"" << U_ICU_VERSION << "\"" << std::endl; + std::cout << std::endl; + // we're going to need an NFD normalizer + UErrorCode status = U_ZERO_ERROR; + const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(status); + assert(U_SUCCESS(status)); + + // collect the raw list of chars that do NOT have a boundary before them. + std::vector noBoundary; + for (km_core_usv ch = 0; ch < 0x10FFFF; ch++) { + bool bb = nfd->hasBoundaryBefore(ch); + assert(!(ch == 0 && !bb)); // assert that we can use U+0000 as a terminator + if (bb) continue; //only emit nonboundary + noBoundary.push_back(ch); + } + + // now, compress these into runs + std::vector> runs; // start,len + + km_core_usv first = 0; + km_core_usv last = 0; + for(auto i = noBoundary.begin(); i <= noBoundary.end(); i++) { + if (first == 0) { + first = last = *i; + } else { + last++; + if(i == noBoundary.end() || *i != last) { + // end of a run + runs.emplace_back(first, last - first); + if (i != noBoundary.end()) { + // setup for next + first = last = *i; + } + } + } + } + + // finally, write out metadata and the runs themselves. + std::cout << "#define km_noBoundaryBefore_entries " << runs.size() << "\n"; + + std::cout << "static char32_t km_noBoundaryBefore[km_noBoundaryBefore_entries * 2 ] = {" << std::endl; + + std::cout << "/* start codepoint, count (inclusive), ...range end */" << std::endl; + + for (auto i = runs.begin(); i < runs.end(); i++) { + std::cout << "\t0x" << std::hex << i->first << std::dec << ",\t " << i->second << ", // ...0x" << std::hex << (i->first+i->second-1) << std::endl; + } + + // termination + std::cout << "};" << std::endl; + std::cout << "// end" << std::endl; + std::cerr << "Wrote " << runs.size() << " runs representing " << noBoundary.size() << " entries." << std::endl; + return 0; +} + +int +main(int /*argc*/, const char * /*argv*/[]) { + write_nfd_table(); + return 0; +} diff --git a/core/tests/unit/ldml/meson.build b/core/tests/unit/ldml/meson.build index da60757750..42f2d7778e 100644 --- a/core/tests/unit/ldml/meson.build +++ b/core/tests/unit/ldml/meson.build @@ -132,7 +132,7 @@ test('test_context_normalization', test_context_normalization, suite: 'ldml') # Build and run additional test_unicode test test_unicode = executable('test_unicode', 'test_unicode.cpp', - ['test_unicode.cpp', common_test_files], + ['test_unicode.cpp', common_test_files, generated_headers], cpp_args: defns + warns, include_directories: [inc, libsrc, '../../../../developer/src/ext/json'], link_args: links + tests_flags, diff --git a/core/tests/unit/ldml/test_unicode.cpp b/core/tests/unit/ldml/test_unicode.cpp index 50844d635b..b36f1a3e38 100644 --- a/core/tests/unit/ldml/test_unicode.cpp +++ b/core/tests/unit/ldml/test_unicode.cpp @@ -47,7 +47,7 @@ #ifdef __EMSCRIPTEN__ // Pull this in to verify versions -#include "../../../../resources/standards-data/unicode-character-database/nfd_table.h" +#include "util_normalize_table.h" #endif //------------------------------------------------------------------------------------- @@ -189,9 +189,15 @@ inline const char *boolstr(bool b) { void test_has_boundary_before() { std::cout << "= " << __FUNCTION__ << std::endl; - std::cout << "(this test only runs under emscripten. congratulations.)" << std::endl; - // static_assert(U_UNICODE_VERSION == KM_HASBOUNDARYBEFORE_UNICODE_VERSION, "nfd_table.h Unicode version does not match ICU's - see nfd_table.h"); - std::cout << U_UNICODE_VERSION << "≈≈" << KM_HASBOUNDARYBEFORE_UNICODE_VERSION << std::endl; + std::cout << "I see we are on Emscripten / wasm! Now we will do some additional tests." << std::endl; + std::string icu4c_unicode(U_UNICODE_VERSION), header_unicode(KM_HASBOUNDARYBEFORE_UNICODE_VERSION), + icu4c_icu(U_ICU_VERSION), header_icu(KM_HASBOUNDARYBEFORE_ICU_VERSION); + std::cout << "Unicode: " << U_UNICODE_VERSION << ", and from the table file: " << KM_HASBOUNDARYBEFORE_UNICODE_VERSION << std::endl; + std::cout << "It would be very strange for these versions to be out of sync. Some sort of build or tool problem." << std::endl; + assert_basic_equal(icu4c_unicode, header_unicode); + assert_basic_equal(icu4c_icu, header_icu); + + std::cout << std::endl << "Now, let's make sure has_nfd_boundary_before() matches ICU." << std::endl; UErrorCode status = U_ZERO_ERROR; const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(status); @@ -203,7 +209,7 @@ void test_has_boundary_before() { auto icu_hbb = nfd->hasBoundaryBefore(cp); if (km_hbb != icu_hbb) { - std::cerr << "Error: nfd_table.h said " << boolstr(km_hbb) << " but ICU said " << boolstr(icu_hbb) << " for " + std::cerr << "Error: util_normalize_table.h said " << boolstr(km_hbb) << " but ICU said " << boolstr(icu_hbb) << " for " << "has_nfd_boundary_before(0x" << std::hex << cp << std::dec << ")" << std::endl; } assert(km_hbb == icu_hbb); diff --git a/core/tools/meson.build b/core/tools/meson.build deleted file mode 100644 index 5221b2a6af..0000000000 --- a/core/tools/meson.build +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright: © 2024 SIL International. -# Description: Cross platform build script to compile tool(s). -# Create Date: 31 May 2024 -# Authors: Steven R. Loomis (SRL) -# - - -# TODO -- why are these differing from the standard.meson.build flags? -if cpp_compiler.get_id() == 'gcc' or cpp_compiler.get_id() == 'clang' or cpp_compiler.get_id() == 'emscripten' - warns = [ - '-Wno-missing-field-initializers', - '-Wno-unused-parameter' - ] -else - warns = [] -endif - -norm_unicode_update = executable('norm_unicode_update', - ['norm_unicode_update.cpp'], - cpp_args: defns + warns, - include_directories: [inc, libsrc, '../../developer/src/ext/json'], - link_args: links, - dependencies: [icu_uc, icu_i18n], - # link_with: [lib], - objects: lib.extract_all_objects(recursive: false), - ) - - -# ../../resources/standards-data/unicode-character-database/ -norm_data = custom_target('norm_data', output: 'nfd_table.h', command: [norm_unicode_update, '@OUTPUT@']) - - -# TODO: execute it diff --git a/core/tools/norm_unicode_update.cpp b/core/tools/norm_unicode_update.cpp deleted file mode 100644 index 408dbb0c98..0000000000 --- a/core/tools/norm_unicode_update.cpp +++ /dev/null @@ -1,84 +0,0 @@ -#include "kmx/kmx_plus.h" -#include "kmx/kmx_xstring.h" -#include "core_icu.h" - -#include -#include -#include -#include - -#include - -#include - -#ifndef __EMSCRIPTEN__ - -int -write_nfd_table(const char *NFD_FILE) { - std::cout << " writing: " << NFD_FILE << std::endl; - auto f = std::ofstream(NFD_FILE); - assert(f.good()); - - // write preamble - f << "//NFD hasBoundaryBefore" << std::endl; - f << "#pragma once" << std::endl; - f << "#define KM_HASBOUNDARYBEFORE_UNICODE_VERSION \"" << U_UNICODE_VERSION << "\"" << std::endl; - f << "#define KM_HASBOUNDARYBEFORE_ICU_VERSION \"" << U_ICU_VERSION << "\"" << std::endl; - // we're going to need an NFD normalizer - UErrorCode status = U_ZERO_ERROR; - const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(status); - assert(U_SUCCESS(status)); - - std::vector noBoundary; - - for (km_core_usv ch = 0; ch < 0x10FFFF; ch++) { - bool bb = nfd->hasBoundaryBefore(ch); - assert(!(ch == 0 && !bb)); // assert that we can use U+0000 as a terminator - if (bb) continue; //only emit nonboundary - noBoundary.push_back(ch); - } - - std::vector> runs; // start,len - - km_core_usv first = 0; - km_core_usv last = 0; - for(auto i = noBoundary.begin(); i <= noBoundary.end(); i++) { - if (first == 0) { - first = last = *i; - } else { - last++; - if(i == noBoundary.end() || *i != last) { - // end of a run - runs.emplace_back(first, last - first); - if (i != noBoundary.end()) { - // setup for next - first = last = *i; - } - } - } - } - f << "#define km_noBoundaryBefore_entries " << runs.size() << "\n"; - - f << "static char32_t km_noBoundaryBefore[km_noBoundaryBefore_entries * 2 ] = {" << std::endl; - - for (auto i = runs.begin(); i < runs.end(); i++) { - f << "\t0x" << std::hex << i->first << std::dec << ",\t " << i->second << ", // ...0x" << std::hex << (i->first+i->second-1) << std::endl; - } - - // termination - f << "};" << std::endl; - return 0; -} - -int -main(int argc, const char *argv[]) { - assert(argc == 2); // call with one param: @OUTPUT@ - write_nfd_table(argv[1]); - return 0; -} -#else -int main(int argc, const char *argv[]) { - std::cerr << "Can't run this under Emscripten - run under another platform." << std::endl; - return 1; -} -#endif diff --git a/resources/standards-data/unicode-character-database/nfd_table.h b/resources/standards-data/unicode-character-database/nfd_table.h deleted file mode 100644 index 068240fc30..0000000000 --- a/resources/standards-data/unicode-character-database/nfd_table.h +++ /dev/null @@ -1,197 +0,0 @@ -//NFD hasBoundaryBefore -#pragma once -#define KM_HASBOUNDARYBEFORE_UNICODE_VERSION "15.0" -#define KM_HASBOUNDARYBEFORE_ICU_VERSION "73.1" -#define km_noBoundaryBefore_entries 190 -static char32_t km_noBoundaryBefore[km_noBoundaryBefore_entries * 2 ] = { - 0x300, 79, // ...0x34e - 0x350, 32, // ...0x36f - 0x483, 5, // ...0x487 - 0x591, 45, // ...0x5bd - 0x5bf, 1, // ...0x5bf - 0x5c1, 2, // ...0x5c2 - 0x5c4, 2, // ...0x5c5 - 0x5c7, 1, // ...0x5c7 - 0x610, 11, // ...0x61a - 0x64b, 21, // ...0x65f - 0x670, 1, // ...0x670 - 0x6d6, 7, // ...0x6dc - 0x6df, 6, // ...0x6e4 - 0x6e7, 2, // ...0x6e8 - 0x6ea, 4, // ...0x6ed - 0x711, 1, // ...0x711 - 0x730, 27, // ...0x74a - 0x7eb, 9, // ...0x7f3 - 0x7fd, 1, // ...0x7fd - 0x816, 4, // ...0x819 - 0x81b, 9, // ...0x823 - 0x825, 3, // ...0x827 - 0x829, 5, // ...0x82d - 0x859, 3, // ...0x85b - 0x898, 8, // ...0x89f - 0x8ca, 24, // ...0x8e1 - 0x8e3, 29, // ...0x8ff - 0x93c, 1, // ...0x93c - 0x94d, 1, // ...0x94d - 0x951, 4, // ...0x954 - 0x9bc, 1, // ...0x9bc - 0x9cd, 1, // ...0x9cd - 0x9fe, 1, // ...0x9fe - 0xa3c, 1, // ...0xa3c - 0xa4d, 1, // ...0xa4d - 0xabc, 1, // ...0xabc - 0xacd, 1, // ...0xacd - 0xb3c, 1, // ...0xb3c - 0xb4d, 1, // ...0xb4d - 0xbcd, 1, // ...0xbcd - 0xc3c, 1, // ...0xc3c - 0xc4d, 1, // ...0xc4d - 0xc55, 2, // ...0xc56 - 0xcbc, 1, // ...0xcbc - 0xccd, 1, // ...0xccd - 0xd3b, 2, // ...0xd3c - 0xd4d, 1, // ...0xd4d - 0xdca, 1, // ...0xdca - 0xe38, 3, // ...0xe3a - 0xe48, 4, // ...0xe4b - 0xeb8, 3, // ...0xeba - 0xec8, 4, // ...0xecb - 0xf18, 2, // ...0xf19 - 0xf35, 1, // ...0xf35 - 0xf37, 1, // ...0xf37 - 0xf39, 1, // ...0xf39 - 0xf71, 5, // ...0xf75 - 0xf7a, 4, // ...0xf7d - 0xf80, 5, // ...0xf84 - 0xf86, 2, // ...0xf87 - 0xfc6, 1, // ...0xfc6 - 0x1037, 1, // ...0x1037 - 0x1039, 2, // ...0x103a - 0x108d, 1, // ...0x108d - 0x135d, 3, // ...0x135f - 0x1714, 2, // ...0x1715 - 0x1734, 1, // ...0x1734 - 0x17d2, 1, // ...0x17d2 - 0x17dd, 1, // ...0x17dd - 0x18a9, 1, // ...0x18a9 - 0x1939, 3, // ...0x193b - 0x1a17, 2, // ...0x1a18 - 0x1a60, 1, // ...0x1a60 - 0x1a75, 8, // ...0x1a7c - 0x1a7f, 1, // ...0x1a7f - 0x1ab0, 14, // ...0x1abd - 0x1abf, 16, // ...0x1ace - 0x1b34, 1, // ...0x1b34 - 0x1b44, 1, // ...0x1b44 - 0x1b6b, 9, // ...0x1b73 - 0x1baa, 2, // ...0x1bab - 0x1be6, 1, // ...0x1be6 - 0x1bf2, 2, // ...0x1bf3 - 0x1c37, 1, // ...0x1c37 - 0x1cd0, 3, // ...0x1cd2 - 0x1cd4, 13, // ...0x1ce0 - 0x1ce2, 7, // ...0x1ce8 - 0x1ced, 1, // ...0x1ced - 0x1cf4, 1, // ...0x1cf4 - 0x1cf8, 2, // ...0x1cf9 - 0x1dc0, 64, // ...0x1dff - 0x20d0, 13, // ...0x20dc - 0x20e1, 1, // ...0x20e1 - 0x20e5, 12, // ...0x20f0 - 0x2cef, 3, // ...0x2cf1 - 0x2d7f, 1, // ...0x2d7f - 0x2de0, 32, // ...0x2dff - 0x302a, 6, // ...0x302f - 0x3099, 2, // ...0x309a - 0xa66f, 1, // ...0xa66f - 0xa674, 10, // ...0xa67d - 0xa69e, 2, // ...0xa69f - 0xa6f0, 2, // ...0xa6f1 - 0xa806, 1, // ...0xa806 - 0xa82c, 1, // ...0xa82c - 0xa8c4, 1, // ...0xa8c4 - 0xa8e0, 18, // ...0xa8f1 - 0xa92b, 3, // ...0xa92d - 0xa953, 1, // ...0xa953 - 0xa9b3, 1, // ...0xa9b3 - 0xa9c0, 1, // ...0xa9c0 - 0xaab0, 1, // ...0xaab0 - 0xaab2, 3, // ...0xaab4 - 0xaab7, 2, // ...0xaab8 - 0xaabe, 2, // ...0xaabf - 0xaac1, 1, // ...0xaac1 - 0xaaf6, 1, // ...0xaaf6 - 0xabed, 1, // ...0xabed - 0xfb1e, 1, // ...0xfb1e - 0xfe20, 16, // ...0xfe2f - 0x101fd, 1, // ...0x101fd - 0x102e0, 1, // ...0x102e0 - 0x10376, 5, // ...0x1037a - 0x10a0d, 1, // ...0x10a0d - 0x10a0f, 1, // ...0x10a0f - 0x10a38, 3, // ...0x10a3a - 0x10a3f, 1, // ...0x10a3f - 0x10ae5, 2, // ...0x10ae6 - 0x10d24, 4, // ...0x10d27 - 0x10eab, 2, // ...0x10eac - 0x10efd, 3, // ...0x10eff - 0x10f46, 11, // ...0x10f50 - 0x10f82, 4, // ...0x10f85 - 0x11046, 1, // ...0x11046 - 0x11070, 1, // ...0x11070 - 0x1107f, 1, // ...0x1107f - 0x110b9, 2, // ...0x110ba - 0x11100, 3, // ...0x11102 - 0x11133, 2, // ...0x11134 - 0x11173, 1, // ...0x11173 - 0x111c0, 1, // ...0x111c0 - 0x111ca, 1, // ...0x111ca - 0x11235, 2, // ...0x11236 - 0x112e9, 2, // ...0x112ea - 0x1133b, 2, // ...0x1133c - 0x1134d, 1, // ...0x1134d - 0x11366, 7, // ...0x1136c - 0x11370, 5, // ...0x11374 - 0x11442, 1, // ...0x11442 - 0x11446, 1, // ...0x11446 - 0x1145e, 1, // ...0x1145e - 0x114c2, 2, // ...0x114c3 - 0x115bf, 2, // ...0x115c0 - 0x1163f, 1, // ...0x1163f - 0x116b6, 2, // ...0x116b7 - 0x1172b, 1, // ...0x1172b - 0x11839, 2, // ...0x1183a - 0x1193d, 2, // ...0x1193e - 0x11943, 1, // ...0x11943 - 0x119e0, 1, // ...0x119e0 - 0x11a34, 1, // ...0x11a34 - 0x11a47, 1, // ...0x11a47 - 0x11a99, 1, // ...0x11a99 - 0x11c3f, 1, // ...0x11c3f - 0x11d42, 1, // ...0x11d42 - 0x11d44, 2, // ...0x11d45 - 0x11d97, 1, // ...0x11d97 - 0x11f41, 2, // ...0x11f42 - 0x16af0, 5, // ...0x16af4 - 0x16b30, 7, // ...0x16b36 - 0x16ff0, 2, // ...0x16ff1 - 0x1bc9e, 1, // ...0x1bc9e - 0x1d165, 5, // ...0x1d169 - 0x1d16d, 6, // ...0x1d172 - 0x1d17b, 8, // ...0x1d182 - 0x1d185, 7, // ...0x1d18b - 0x1d1aa, 4, // ...0x1d1ad - 0x1d242, 3, // ...0x1d244 - 0x1e000, 7, // ...0x1e006 - 0x1e008, 17, // ...0x1e018 - 0x1e01b, 7, // ...0x1e021 - 0x1e023, 2, // ...0x1e024 - 0x1e026, 5, // ...0x1e02a - 0x1e08f, 1, // ...0x1e08f - 0x1e130, 7, // ...0x1e136 - 0x1e2ae, 1, // ...0x1e2ae - 0x1e2ec, 4, // ...0x1e2ef - 0x1e4ec, 4, // ...0x1e4ef - 0x1e8d0, 7, // ...0x1e8d6 - 0x1e944, 7, // ...0x1e94a -};