From 465c4bfdc0c934e60d9f1553a8779b6c06695b21 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 24 May 2024 09:24:53 -0500 Subject: [PATCH 01/64] feat(core): add a KMN_NO_ICU internal switch to start being able to turn off ICU - always set to 0 for now (keep ICU around) - set KMN_IN_LDML_TESTS in tests to keep ICU there for test and comparison - add core_icu.cpp and put some utils there. #9467 --- core/src/actions_normalize.cpp | 26 +-------- core/src/core_icu.cpp | 69 +++++++++++++++++++++++ core/src/core_icu.h | 61 +++++++++++++++++++- core/src/meson.build | 1 + core/src/util_normalize.cpp | 29 ---------- core/tests/unit/ldml/ldml_test_source.cpp | 3 + core/tests/unit/ldml/test_unicode.cpp | 3 + 7 files changed, 137 insertions(+), 55 deletions(-) create mode 100644 core/src/core_icu.cpp diff --git a/core/src/actions_normalize.cpp b/core/src/actions_normalize.cpp index 3384fda975..799b84425e 100644 --- a/core/src/actions_normalize.cpp +++ b/core/src/actions_normalize.cpp @@ -20,7 +20,6 @@ // forward declarations icu::UnicodeString context_items_to_unicode_string(km::core::context const *context); -km_core_usv *unicode_string_to_usv(icu::UnicodeString& src); /** * Normalize the output from an action to NFC, across the context | output @@ -182,7 +181,7 @@ bool km::core::actions_normalize( return false; } - auto new_output = unicode_string_to_usv(output_nfc); + auto new_output = km::core::util::unicode_string_to_usv(output_nfc); if(!new_output) { // error logging handled in unicode_string_to_usv return false; @@ -255,29 +254,6 @@ icu::UnicodeString context_items_to_unicode_string(km::core::context const *cont return result; } -/** - * Helper to convert icu::UnicodeString to a UTF-32 km_core_usv buffer, - * nul-terminated - */ -km_core_usv *unicode_string_to_usv(icu::UnicodeString& src) { - UErrorCode icu_status = U_ZERO_ERROR; - - km_core_usv *dst = new km_core_usv[src.length() + 1]; - - src.toUTF32(reinterpret_cast(dst), src.length(), icu_status); - - assert(U_SUCCESS(icu_status)); - if(!U_SUCCESS(icu_status)) { - DebugLog("toUTF32 failed with %x", icu_status); - delete[] dst; - return nullptr; - } - - dst[src.length()] = 0; - return dst; -} - - /** * Refresh app_context to match the cached_context. Does not do normalization, diff --git a/core/src/core_icu.cpp b/core/src/core_icu.cpp new file mode 100644 index 0000000000..e8b1e4ae6b --- /dev/null +++ b/core/src/core_icu.cpp @@ -0,0 +1,69 @@ +/* + Copyright: © SIL International. + Description: Common LDML utilities + Create Date: 24 May 2024 + Authors: Steven R. Loomis +*/ + +#include "core_icu.h" + +#if !KMN_NO_ICU + +namespace km { +namespace core { +namespace util { + +/** + * Helper to convert icu::UnicodeString to a UTF-32 km_core_usv buffer, + * nul-terminated + */ +km_core_usv *unicode_string_to_usv(icu::UnicodeString& src) { + UErrorCode icu_status = U_ZERO_ERROR; + + km_core_usv *dst = new km_core_usv[src.length() + 1]; + + src.toUTF32(reinterpret_cast(dst), src.length(), icu_status); + + assert(U_SUCCESS(icu_status)); + if(!U_SUCCESS(icu_status)) { + DebugLog("toUTF32 failed with %x", icu_status); + delete[] dst; + return nullptr; + } + + dst[src.length()] = 0; + return dst; +} + +/** + * Internal function to normalize with a specified mode. + * Note: that this function _does_ assert failure, so it is not + * required to assert its return code. The return is provided so + * that callers can exit (such as making no change) if there was failure. + * + * Also note that "failure" here is something catastrophic: ICU not initialized, + * or, more likely, some low memory situation. Does not fail on "bad" data. + * @param n the ICU Normalizer to use + * @param str input/output string + * @param status error code, must be initialized on input + * @return false if failure + */ +bool normalize(const icu::Normalizer2 *n, std::u16string &str, UErrorCode &status) { + UASSERT_SUCCESS(status); + assert(n != nullptr); + icu::UnicodeString dest; + icu::UnicodeString src = icu::UnicodeString(str.data(), (int32_t)str.length()); + n->normalize(src, dest, status); + // the next line here will assert + if (UASSERT_SUCCESS(status)) { + str.assign(dest.getBuffer(), dest.length()); + } + return U_SUCCESS(status); +} + + +} +} +} /* end km::core::util */ + +#endif diff --git a/core/src/core_icu.h b/core/src/core_icu.h index d95d198637..dadf4e8448 100644 --- a/core/src/core_icu.h +++ b/core/src/core_icu.h @@ -3,8 +3,32 @@ */ #pragma once +#define KMN_NO_ICU 0 /* Temporary - keep ICU in for now while checking out build issues */ + +#ifdef __EMSCRIPTEN__ +// define this in tests to keep ICU around +# if !defined(KMN_IN_LDML_TESTS) +# if !defined(KMN_NO_ICU) +// under wasm, turn off ICU except in tests. +# define KMN_NO_ICU 1 +# endif +# endif +#elif !defined(KMN_NO_ICU) +# define KMN_NO_ICU 0 +#endif + +#if KMN_NO_ICU + +// NO ICU + +// any shims needed here for disabling ICU + +#else + +// YES ICU + #if !defined(HAVE_ICU4C) -#error icu4c is required for this code +# error icu4c is required for this code #endif #define U_FALLTHROUGH @@ -12,6 +36,7 @@ #include "unicode/unistr.h" #include "unicode/normalizer2.h" +#include "keyman_core.h" #include "debuglog.h" #include @@ -31,3 +56,37 @@ inline bool uassert_success(const char *file, int line, const char *function, UE * the first assert is for debug builds, the second triggers the debuglog and has the return value. * */ #define UASSERT_SUCCESS(status) (assert(U_SUCCESS(status)), uassert_success(__FILE__, __LINE__, __FUNCTION__, status)) + +// ------------------ some ICU C++ utilities ---------------------------- + +namespace km { +namespace core { +namespace util { + +/** + * Convert a UnicodeString to a km_core_usv array + * @return the 0-terminated array. Caller owns storage. + */ +km_core_usv *unicode_string_to_usv(icu::UnicodeString& src); + +/** + * Internal function to normalize with a specified mode. + * Note: that this function _does_ assert failure, so it is not + * required to assert its return code. The return is provided so + * that callers can exit (such as making no change) if there was failure. + * + * Also note that "failure" here is something catastrophic: ICU not initialized, + * or, more likely, some low memory situation. Does not fail on "bad" data. + * @param n the ICU Normalizer to use + * @param str input/output string + * @param status error code, must be initialized on input + * @return false if failure + */ +bool normalize(const icu::Normalizer2 *n, std::u16string &str, UErrorCode &status); + + +} +} +} + +#endif /* KMN_NO_ICU */ diff --git a/core/src/meson.build b/core/src/meson.build index e8008fb78a..5632c22257 100644 --- a/core/src/meson.build +++ b/core/src/meson.build @@ -60,6 +60,7 @@ kmx_files = files( 'km_core_processevent_api.cpp', 'jsonpp.cpp', 'util_normalize.cpp', + 'core_icu.cpp', 'ldml/ldml_processor.cpp', 'ldml/ldml_transforms.cpp', 'ldml/ldml_markers.cpp', diff --git a/core/src/util_normalize.cpp b/core/src/util_normalize.cpp index c21b75ea5b..c0367b5ef9 100644 --- a/core/src/util_normalize.cpp +++ b/core/src/util_normalize.cpp @@ -27,35 +27,6 @@ namespace km { namespace core { namespace util { -#ifndef __EMSCRIPTEN__ - -/** - * Internal function to normalize with a specified mode. - * Note: that this function _does_ assert failure, so it is not - * required to assert its return code. The return is provided so - * that callers can exit (such as making no change) if there was failure. - * - * Also note that "failure" here is something catastrophic: ICU not initialized, - * or, more likely, some low memory situation. Does not fail on "bad" data. - * @param n the ICU Normalizer to use - * @param str input/output string - * @param status error code, must be initialized on input - * @return false if failure - */ -static bool normalize(const icu::Normalizer2 *n, std::u16string &str, UErrorCode &status) { - UASSERT_SUCCESS(status); - assert(n != nullptr); - icu::UnicodeString dest; - icu::UnicodeString src = icu::UnicodeString(str.data(), (int32_t)str.length()); - n->normalize(src, dest, status); - // the next line here will assert - if (UASSERT_SUCCESS(status)) { - str.assign(dest.getBuffer(), dest.length()); - } - return U_SUCCESS(status); -} -#endif - bool normalize_nfd(std::u32string &str) { std::u16string rstr = km::core::kmx::u32string_to_u16string(str); if(!km::core::util::normalize_nfd(rstr)) { diff --git a/core/tests/unit/ldml/ldml_test_source.cpp b/core/tests/unit/ldml/ldml_test_source.cpp index 439236fd77..f2cca9ee5f 100644 --- a/core/tests/unit/ldml/ldml_test_source.cpp +++ b/core/tests/unit/ldml/ldml_test_source.cpp @@ -18,6 +18,9 @@ #include +// Ensure that ICU gets included even on wasm. +#define KMN_IN_LDML_TESTS + #include // for char to vk mapping tables #include // for surrogate pair macros #include diff --git a/core/tests/unit/ldml/test_unicode.cpp b/core/tests/unit/ldml/test_unicode.cpp index 77ae0202ac..ae41853153 100644 --- a/core/tests/unit/ldml/test_unicode.cpp +++ b/core/tests/unit/ldml/test_unicode.cpp @@ -10,6 +10,9 @@ #include #include +// Ensure that ICU gets included even on wasm. +#define KMN_IN_LDML_TESTS + #include "keyman_core.h" #include "path.hpp" From a80a0a7dcba90b2c33cbd2ad6ad57f08cfd1feec Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 24 May 2024 10:42:57 -0500 Subject: [PATCH 02/64] feat(core): move more normalization logic into JS - add a normalize_nfd() which takes a single codepoint - temporarily keep ICU in actions_normalize.cpp and ldml_transforms.cpp - expand wasm opts in unit tests --- core/src/actions_normalize.cpp | 30 ++++++++++++++++++++++ core/src/core_icu.h | 2 -- core/src/ldml/ldml_markers.cpp | 35 ++++++++++--------------- core/src/ldml/ldml_transforms.cpp | 2 ++ core/src/util_normalize.cpp | 41 ++++++++++++++++++++++++++++++ core/src/util_normalize.hpp | 3 +++ core/tests/unit/kmnkbd/meson.build | 2 +- 7 files changed, 90 insertions(+), 25 deletions(-) diff --git a/core/src/actions_normalize.cpp b/core/src/actions_normalize.cpp index 799b84425e..8a7cf57eaf 100644 --- a/core/src/actions_normalize.cpp +++ b/core/src/actions_normalize.cpp @@ -1,3 +1,5 @@ +// TEMP +#define KMN_NO_ICU 0 /* Copyright: © 2024 SIL International. Description: Implementation of the action output normalization. @@ -21,6 +23,34 @@ icu::UnicodeString context_items_to_unicode_string(km::core::context const *context); + +// TEMP +namespace km { +namespace core { +namespace util { + +/** + * Helper to convert icu::UnicodeString to a UTF-32 km_core_usv buffer, + * nul-terminated + */ +inline km_core_usv *unicode_string_to_usv(icu::UnicodeString& src) { + UErrorCode icu_status = U_ZERO_ERROR; + + km_core_usv *dst = new km_core_usv[src.length() + 1]; + + src.toUTF32(reinterpret_cast(dst), src.length(), icu_status); + + assert(U_SUCCESS(icu_status)); + if(!U_SUCCESS(icu_status)) { + DebugLog("toUTF32 failed with %x", icu_status); + delete[] dst; + return nullptr; + } + + dst[src.length()] = 0; + return dst; +} +}}} /** * Normalize the output from an action to NFC, across the context | output * boundary, fixing up the app_context and the output actions to take into diff --git a/core/src/core_icu.h b/core/src/core_icu.h index dadf4e8448..45941bc8b8 100644 --- a/core/src/core_icu.h +++ b/core/src/core_icu.h @@ -3,8 +3,6 @@ */ #pragma once -#define KMN_NO_ICU 0 /* Temporary - keep ICU in for now while checking out build issues */ - #ifdef __EMSCRIPTEN__ // define this in tests to keep ICU around # if !defined(KMN_IN_LDML_TESTS) diff --git a/core/src/ldml/ldml_markers.cpp b/core/src/ldml/ldml_markers.cpp index b1d4604dbc..5d4fb4ff46 100644 --- a/core/src/ldml/ldml_markers.cpp +++ b/core/src/ldml/ldml_markers.cpp @@ -268,15 +268,14 @@ add_pending_markers( marker_map *markers, marker_list &last_markers, const std::u32string::const_iterator &last, - const std::u32string::const_iterator &end, - const icu::Normalizer2 *nfd) { + const std::u32string::const_iterator &end) { // quick check to see if there's no work to do. if(markers == nullptr) { return; } /** which character this marker is 'glued' to. */ char32_t marker_ch; - icu::UnicodeString decomposition; + std::u32string decomposition; if (last == end) { // at end of text, so use a special value to indicate 'EOT'. marker_ch = MARKER_BEFORE_EOT; @@ -285,17 +284,13 @@ add_pending_markers( // if the character is composed, we need to use the first decomposed char // as the 'glue'. - if(!nfd->getDecomposition(ch, decomposition)) { + if(!km::core::util::normalize_nfd(ch, decomposition)) { // char does not have a decomposition - so it may be used for the glue - marker_ch = ch; - decomposition.remove(); // no other entries needed - } else { - // 'glue' is the first codepoint of the decomposition. - marker_ch = decomposition.char32At(0); - if (decomposition.countChar32() == 1) { - decomposition.remove(); // no other entries needed - } // else: will add the remainder below + // the 'if' is only for the assertions here. + assert(decomposition.length() == 1); // should be a single UTF-32 char + assert(decomposition.at(0) == ch); // should be the same char } + marker_ch = decomposition.at(0); // always the first char } markers->emplace_back(marker_ch); // now, update the map with these markers (in order) on this character. @@ -304,10 +299,10 @@ add_pending_markers( markers->emplace_back(marker_ch, *i); } // add any further entries due to decomposition - if (!decomposition.isEmpty()) { - // We already added the base char above, add teh rest - for (auto i=1; iemplace_back(decomposition.char32At(i)); + if (decomposition.length() > 1) { + // We already added the base char above, add the rest + for (size_t i=1; iemplace_back(decomposition.at(i)); } } // clear the list @@ -318,10 +313,6 @@ std::u32string remove_markers(const std::u32string &str, marker_map *markers, marker_encoding encoding) { std::u32string out; marker_list last_markers; - UErrorCode status = U_ZERO_ERROR; - const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(status); - UASSERT_SUCCESS(status); - auto last = str.begin(); // points to the part of the string after the last matched marker for (auto i = str.begin(); i != str.end();) { auto marker_no = parse_next_marker(i, str.end(), encoding); @@ -329,7 +320,7 @@ remove_markers(const std::u32string &str, marker_map *markers, marker_encoding e // add any markers found before this entry, but only if there is intervening // text. This prevents the sentinel or the '\u' from becoming the attachment char. if (i != last) { - add_pending_markers(markers, last_markers, last, str.end(), nfd); + add_pending_markers(markers, last_markers, last, str.end()); out.append(last, i); // append any non-marker text since the end of the last marker last = i; // advance over text we've already appended } @@ -347,7 +338,7 @@ remove_markers(const std::u32string &str, marker_map *markers, marker_encoding e // add any remaining pending markers. // if last == str.end() then this wil be MARKER_BEFORE_EOT // otherwise it will be the glue character - add_pending_markers(markers, last_markers, last, str.end(), nfd); + add_pending_markers(markers, last_markers, last, str.end()); // get the suffix between the last marker and the end (could be nothing) out.append(last, str.end()); return out; diff --git a/core/src/ldml/ldml_transforms.cpp b/core/src/ldml/ldml_transforms.cpp index 7b161ff4f9..42953c604e 100644 --- a/core/src/ldml/ldml_transforms.cpp +++ b/core/src/ldml/ldml_transforms.cpp @@ -1,3 +1,5 @@ +// TEMP +#define KMN_NO_ICU 0 /* Copyright: © SIL International. Description: This is an implementation of the LDML keyboard spec 3.0. diff --git a/core/src/util_normalize.cpp b/core/src/util_normalize.cpp index c0367b5ef9..98fc3e5fb8 100644 --- a/core/src/util_normalize.cpp +++ b/core/src/util_normalize.cpp @@ -13,6 +13,7 @@ #ifdef __EMSCRIPTEN__ #include #include "utfcodec.hpp" +#include // JS implementations EM_JS(char*, NormalizeNFD, (const char* input), { @@ -62,6 +63,10 @@ bool normalize_nfd(std::u16string &str) { * Normalize the input string using ICU, out of place */ bool normalize_nfd(km_core_cu const * src, std::u16string &dst) { +#ifdef __EMSCRIPTEN__ + dst = std::u16string(src); + return normalize_nfd(dst); // vector to above fcn +#else UErrorCode icu_status = U_ZERO_ERROR; const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(icu_status); assert(U_SUCCESS(icu_status)); @@ -80,6 +85,42 @@ bool normalize_nfd(km_core_cu const * src, std::u16string &dst) { dst.assign(udst.getBuffer(), udst.length()); return true; +#endif +} + +bool +normalize_nfd(km_core_usv cp, std::u32string &dst) { + // set the output string to the original string + dst.clear(); + dst.append(1, cp); +#ifdef __EMSCRIPTEN__ + auto str16 = convert(dst); + if (!normalize_nfd(str16)) { + return false; // failed, retain original str + } else { + dst = convert(str16); + return true; + } +#else + UErrorCode icu_status = U_ZERO_ERROR; + const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(icu_status); + assert(U_SUCCESS(icu_status)); + if (!U_SUCCESS(icu_status)) { + // TODO: log the failure code + return false; + } + icu::UnicodeString decomposition; + if (!nfd->getDecomposition(cp, decomposition)) { + return false; // no error, just no decomposition + } else { + dst.clear(); + auto len = decomposition.countChar32(); + for (int i = 0; i < len; i++) { + dst.append(1, decomposition.char32At(i)); + } + return true; + } +#endif } } diff --git a/core/src/util_normalize.hpp b/core/src/util_normalize.hpp index 90ed650b08..d417617565 100644 --- a/core/src/util_normalize.hpp +++ b/core/src/util_normalize.hpp @@ -23,6 +23,9 @@ bool normalize_nfd(std::u16string &str); /** normalize src to dst in NFD. @return false on failure */ bool normalize_nfd(km_core_cu const * src, std::u16string &dst); +/** normalize (decompose) a single cp to string. @return false on failure */ +bool normalize_nfd(km_core_usv cp, std::u32string &dst); + } } } diff --git a/core/tests/unit/kmnkbd/meson.build b/core/tests/unit/kmnkbd/meson.build index 0adda0a35e..7285b9bf11 100644 --- a/core/tests/unit/kmnkbd/meson.build +++ b/core/tests/unit/kmnkbd/meson.build @@ -34,7 +34,7 @@ test_path = join_paths(meson.current_build_dir(), '..', 'kmx') tests_flags = [] if cpp_compiler.get_id() == 'emscripten' - tests_flags += ['-lnodefs.js', '-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\']'] + tests_flags += ['-lnodefs.js', wasm_exported_runtime_methods] endif foreach t : tests From 5d420248eba9fe12211c1b8127337964a35c3d2f Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 24 May 2024 18:20:30 -0500 Subject: [PATCH 03/64] feat(core): move more normalization logic into JS - major redo of actions_normalize - into UTF-32 and not using ICU directly - add some utilities: u32len, u32dup, context_items_from_utf32 #9467 --- core/src/actions_normalize.cpp | 152 ++++++++++--------------------- core/src/context.hpp | 24 +++++ core/src/core_icu.cpp | 13 ++- core/src/core_icu.h | 1 + core/src/km_core_context_api.cpp | 7 ++ core/src/kmx/kmx_xstring.cpp | 14 +++ core/src/kmx/kmx_xstring.h | 3 + core/src/util_normalize.cpp | 126 ++++++++++++++++++++++--- core/src/util_normalize.hpp | 18 ++++ 9 files changed, 239 insertions(+), 119 deletions(-) diff --git a/core/src/actions_normalize.cpp b/core/src/actions_normalize.cpp index 8a7cf57eaf..78bada7b6f 100644 --- a/core/src/actions_normalize.cpp +++ b/core/src/actions_normalize.cpp @@ -1,5 +1,3 @@ -// TEMP -#define KMN_NO_ICU 0 /* Copyright: © 2024 SIL International. Description: Implementation of the action output normalization. @@ -17,40 +15,13 @@ #include "state.hpp" #include "option.hpp" #include "debuglog.h" -#include "core_icu.h" +#include "util_normalize.hpp" +#include "utfcodec.hpp" +#include "kmx/kmx_xstring.h" -// forward declarations +// forward declaration +bool context_items_to_unicode_string(km::core::context const *context, std::u32string &str); -icu::UnicodeString context_items_to_unicode_string(km::core::context const *context); - - -// TEMP -namespace km { -namespace core { -namespace util { - -/** - * Helper to convert icu::UnicodeString to a UTF-32 km_core_usv buffer, - * nul-terminated - */ -inline km_core_usv *unicode_string_to_usv(icu::UnicodeString& src) { - UErrorCode icu_status = U_ZERO_ERROR; - - km_core_usv *dst = new km_core_usv[src.length() + 1]; - - src.toUTF32(reinterpret_cast(dst), src.length(), icu_status); - - assert(U_SUCCESS(icu_status)); - if(!U_SUCCESS(icu_status)) { - DebugLog("toUTF32 failed with %x", icu_status); - delete[] dst; - return nullptr; - } - - dst[src.length()] = 0; - return dst; -} -}}} /** * Normalize the output from an action to NFC, across the context | output * boundary, fixing up the app_context and the output actions to take into @@ -94,32 +65,12 @@ bool km::core::actions_normalize( cached_context. */ - /* - Initialization - */ - - UErrorCode icu_status = U_ZERO_ERROR; - const icu::Normalizer2 *nfc = icu::Normalizer2::getNFCInstance(icu_status); - assert(U_SUCCESS(icu_status)); - if(!U_SUCCESS(icu_status)) { - DebugLog("getNFCInstance failed with %x", icu_status); + std::u32string output(actions.output); + std::u32string cached_context_string, app_context_string; + if (!context_items_to_unicode_string(cached_context, cached_context_string)) { return false; } - - const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(icu_status); - assert(U_SUCCESS(icu_status)); - if(!U_SUCCESS(icu_status)) { - DebugLog("getNFDInstance failed with %x", icu_status); - return false; - } - - icu::UnicodeString output = icu::UnicodeString::fromUTF32(reinterpret_cast(actions.output), -1); - icu::UnicodeString cached_context_string = context_items_to_unicode_string(cached_context); - icu::UnicodeString app_context_string = context_items_to_unicode_string(app_context); - assert(!output.isBogus()); - assert(!cached_context_string.isBogus()); - assert(!app_context_string.isBogus()); - if(output.isBogus() || cached_context_string.isBogus() || app_context_string.isBogus()) { + if (!context_items_to_unicode_string(app_context, app_context_string)) { return false; } int nfu_to_delete = 0; @@ -127,20 +78,25 @@ bool km::core::actions_normalize( /* Further debug assertion of inputs */ - - assert(nfd->isNormalized(output, icu_status) && U_SUCCESS(icu_status)); - assert(nfd->isNormalized(cached_context_string, icu_status) && U_SUCCESS(icu_status)); + assert(km::core::util::is_nfd(output)); + assert(km::core::util::is_nfd(cached_context_string)); /* The keyboard processor will have updated the cached_context already, - applying the transform to it, so we need to rewind this. Remove the output - from cached_context_string to start + applying the transform to it, so we need to rewind this. + + Assert that 'cached_context_string' ends with 'output' + + Remove the output + from cached_context_string to start. */ assert(cached_context_string.length() >= output.length()); - int n = cached_context_string.length() - output.length(); + size_t n = cached_context_string.length() - output.length(); + // auto end_cached = cached_context_string.substr(n, output.length()); + // assert(end_cached == output); assert(cached_context_string.compare(n, output.length(), output) == 0); - cached_context_string.remove(n); + cached_context_string.resize(n); /* While cached_context is guaranteed to be normalized, actions->output may not @@ -148,21 +104,22 @@ bool km::core::actions_normalize( normalization in our output, we now need to look for a normalization boundary prior to the intersection of the cached_context and the output. */ - if(!output.isEmpty()) { - while(n > 0 && !nfd->hasBoundaryBefore(output[0])) { + if(!output.empty()) { + while(n > 0 && !km::core::util::has_nfd_boundary_before(output[0])) { // The output may interact with the context further in normalization. We // need to copy characters back further until we reach a normalization // boundary. // Remove last code point from the context ... - n = cached_context_string.moveIndex32(n, -1); - UChar32 chr = cached_context_string.char32At(n); - cached_context_string.remove(n); + auto len = cached_context_string.length(); + assert(len>0); + auto chr = cached_context_string.at(len-1); + cached_context_string.resize(len-1); // And prepend it to the output ... - output.insert(0, chr); + output.insert(0, 1, chr); } } @@ -177,25 +134,20 @@ bool km::core::actions_normalize( its normalized form matches the cached_context normalized form. */ - while(app_context_string.countChar32()) { - icu::UnicodeString app_context_nfd; - nfd->normalize(app_context_string, app_context_nfd, icu_status); - assert(U_SUCCESS(icu_status)); - if(!U_SUCCESS(icu_status)) { - DebugLog("nfd->normalize failed with %x", icu_status); + while(!app_context_string.empty()) { + auto app_context_nfd = app_context_string; + if(!km::core::util::normalize_nfd(app_context_nfd)) { + DebugLog("nfd->normalize failed"); return false; } - if(app_context_nfd.compare(cached_context_string) == 0) { + if(app_context_nfd == cached_context_string) { break; } - // remove the last UChar32 - int32_t lastUChar32 = app_context_string.length()-1; - // adjust pointer to get the entire char (i.e. so we don't slice a non-BMP char) - lastUChar32 = app_context_string.getChar32Start(lastUChar32); - // remove the UChar32 (1 or 2 code units) - app_context_string.remove(lastUChar32); + size_t len = app_context_string.length(); + // remove the cp at end + app_context_string.resize(len-1); nfu_to_delete++; } @@ -203,17 +155,15 @@ bool km::core::actions_normalize( Normalize our output string */ - icu::UnicodeString output_nfc; - nfc->normalize(output, output_nfc, icu_status); - assert(U_SUCCESS(icu_status)); - if(!U_SUCCESS(icu_status)) { - DebugLog("nfc->normalize failed with %x", icu_status); + auto output_nfc = output; + if(!km::core::util::normalize_nfc(output_nfc)) { + DebugLog("nfc->normalize failed"); return false; } - auto new_output = km::core::util::unicode_string_to_usv(output_nfc); - if(!new_output) { - // error logging handled in unicode_string_to_usv + auto new_output = km::core::util::string_to_usv(output_nfc); + assert(new_output != nullptr); + if(new_output == nullptr) { return false; } @@ -226,8 +176,8 @@ bool km::core::actions_normalize( app_context_string.append(output_nfc); km_core_context_item *app_context_items = nullptr; km_core_status status = KM_CORE_STATUS_OK; - if((status = context_items_from_utf16(app_context_string.getTerminatedBuffer(), &app_context_items)) != KM_CORE_STATUS_OK) { - DebugLog("context_items_from_utf16 failed with %x", status); + if((status = context_items_from_utf32(app_context_string.c_str(), &app_context_items)) != KM_CORE_STATUS_OK) { + DebugLog("context_items_from_string failed with %x", status); delete [] new_output; return false; } @@ -253,21 +203,19 @@ bool km::core::actions_normalize( /** * Helper to convert km_core_context list into a icu::UnicodeString */ -icu::UnicodeString context_items_to_unicode_string(km::core::context const *context) { - icu::UnicodeString nullString; - nullString.setToBogus(); +bool context_items_to_unicode_string(km::core::context const *context, std::u32string &str) { km_core_context_item *items = nullptr; km_core_status status; if((status = km_core_context_get(static_cast(context), &items)) != KM_CORE_STATUS_OK) { DebugLog("Failed to retrieve context with %s", status); - return nullString; + return false; } size_t buf_size = 0; if((status = context_items_to_utf32(items, nullptr, &buf_size)) != KM_CORE_STATUS_OK) { DebugLog("Failed to retrieve context size with %s", status); km_core_context_items_dispose(items); - return nullString; + return false; } km_core_usv *buf = new km_core_usv[buf_size]; @@ -275,13 +223,13 @@ icu::UnicodeString context_items_to_unicode_string(km::core::context const *cont DebugLog("Failed to retrieve context with %s", status); km_core_context_items_dispose(items); delete [] buf; - return nullString; + return false; } - auto result = icu::UnicodeString::fromUTF32(reinterpret_cast(buf), -1); + str = std::u32string(buf, buf_size - 1); // don't include terminating null km_core_context_items_dispose(items); delete [] buf; - return result; + return true; } diff --git a/core/src/context.hpp b/core/src/context.hpp index 91cc9688cd..f56d637eff 100644 --- a/core/src/context.hpp +++ b/core/src/context.hpp @@ -87,6 +87,30 @@ km_core_status context_items_from_utf16(km_core_cu const *text, km_core_context_item **out_ptr); +/** + * Convert a UTF32 encoded Unicode string into an array of `km_core_context_item` + * structures. Allocates memory as needed. + * + * @return km_core_status + * * `KM_CORE_STATUS_OK`: On success. + * * `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are + * null. + * * `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be + * allocated for the output buffer. + * * `KM_CORE_STATUS_INVALID_UTF`: In the event the UTF32 string cannot + * be decoded. + * + * @param text a pointer to a null terminated array of utf32 encoded data. + * @param out_ptr a pointer to the result variable: A pointer to the start of + * the `km_core_context_item` array containing the representation + * of the input string. Terminated with a type of + * `KM_CORE_CT_END`. Must be disposed of with + * `km_core_context_items_dispose`. + */ +km_core_status +context_items_from_utf32(km_core_usv const *text, + km_core_context_item **out_ptr); + /** * Convert a context item array into a UTF-16 encoded string placing it into the * supplied buffer of specified size, and return the number of code units diff --git a/core/src/core_icu.cpp b/core/src/core_icu.cpp index e8b1e4ae6b..cfaac47c50 100644 --- a/core/src/core_icu.cpp +++ b/core/src/core_icu.cpp @@ -24,8 +24,7 @@ km_core_usv *unicode_string_to_usv(icu::UnicodeString& src) { src.toUTF32(reinterpret_cast(dst), src.length(), icu_status); - assert(U_SUCCESS(icu_status)); - if(!U_SUCCESS(icu_status)) { + if(!UASSERT_SUCCESS(icu_status)) { DebugLog("toUTF32 failed with %x", icu_status); delete[] dst; return nullptr; @@ -49,16 +48,20 @@ km_core_usv *unicode_string_to_usv(icu::UnicodeString& src) { * @return false if failure */ bool normalize(const icu::Normalizer2 *n, std::u16string &str, UErrorCode &status) { - UASSERT_SUCCESS(status); + if(!UASSERT_SUCCESS(status)) { + return false; + } assert(n != nullptr); icu::UnicodeString dest; icu::UnicodeString src = icu::UnicodeString(str.data(), (int32_t)str.length()); n->normalize(src, dest, status); // the next line here will assert - if (UASSERT_SUCCESS(status)) { + if (!UASSERT_SUCCESS(status)) { + return false; + } else { str.assign(dest.getBuffer(), dest.length()); + return true; } - return U_SUCCESS(status); } diff --git a/core/src/core_icu.h b/core/src/core_icu.h index 45941bc8b8..edfc209802 100644 --- a/core/src/core_icu.h +++ b/core/src/core_icu.h @@ -52,6 +52,7 @@ inline bool uassert_success(const char *file, int line, const char *function, UE /** * Assert an ICU4C UErrorCode * the first assert is for debug builds, the second triggers the debuglog and has the return value. + * @returns true on success * */ #define UASSERT_SUCCESS(status) (assert(U_SUCCESS(status)), uassert_success(__FILE__, __LINE__, __FUNCTION__, status)) diff --git a/core/src/km_core_context_api.cpp b/core/src/km_core_context_api.cpp index dd27ab7e5c..594080d8d0 100644 --- a/core/src/km_core_context_api.cpp +++ b/core/src/km_core_context_api.cpp @@ -115,6 +115,13 @@ context_items_from_utf16(km_core_cu const *text, } +km_core_status +context_items_from_utf32(km_core_usv const *text, + km_core_context_item **out_ptr) +{ + return _context_items_from(reinterpret_cast(text), out_ptr); +} + km_core_status context_items_to_utf8(km_core_context_item const *ci, char *buf, size_t * sz_ptr) { diff --git a/core/src/kmx/kmx_xstring.cpp b/core/src/kmx/kmx_xstring.cpp index a1d5ff518a..794ba26889 100644 --- a/core/src/kmx/kmx_xstring.cpp +++ b/core/src/kmx/kmx_xstring.cpp @@ -50,6 +50,15 @@ size_t km::core::kmx::u16len(const km_core_cu *p) { return i; } +size_t km::core::kmx::u32len(const km_core_usv *p) { + int i = 0; + while (*p) { + p++; + i++; + } + return i; +} + int km::core::kmx::u16cmp(const km_core_cu *p, const km_core_cu *q) { while (*p && *q) { if (*p != *q) return *p - *q; @@ -107,6 +116,11 @@ km_core_cu *km::core::kmx::u16dup(km_core_cu *src) { memcpy(dup, src, (u16len(src) + 1) * sizeof(km_core_cu)); return dup; } +km_core_usv *km::core::kmx::u32dup(const km_core_usv *src) { + km_core_usv *dup = new km_core_usv[u32len(src) + 1]; + memcpy(dup, src, (u32len(src) + 1) * sizeof(src[0])); + return dup; +} /* * int xstrlen( PKMX_BYTE p ); diff --git a/core/src/kmx/kmx_xstring.h b/core/src/kmx/kmx_xstring.h index a82e959e56..e17c003091 100644 --- a/core/src/kmx/kmx_xstring.h +++ b/core/src/kmx/kmx_xstring.h @@ -117,6 +117,9 @@ int u16ncmp(const km_core_cu *p, const km_core_cu *q, size_t count); km_core_cu *u16tok(km_core_cu *p, km_core_cu ch, km_core_cu **ctx); km_core_cu *u16dup(km_core_cu *src); +size_t u32len(const km_core_usv *p); +km_core_usv *u32dup(const km_core_usv *src); + //KMX_BOOL MapUSCharToVK(KMX_WORD ch, PKMX_WORD puKey, PKMX_DWORD puShiftFlags); // --- implementation --- diff --git a/core/src/util_normalize.cpp b/core/src/util_normalize.cpp index 98fc3e5fb8..b5c3b860d1 100644 --- a/core/src/util_normalize.cpp +++ b/core/src/util_normalize.cpp @@ -22,12 +22,33 @@ EM_JS(char*, NormalizeNFD, (const char* input), { const nfd = instr.normalize("NFD"); return stringToNewUTF8(nfd); }); + +EM_JS(char*, NormalizeNFC, (const char* input), { + if (!input) return input; // pass through null + const instr = Module.UTF8ToString(input); + const nfd = instr.normalize("NFC"); + return stringToNewUTF8(nfd); +}); + #endif namespace km { namespace core { namespace util { +#ifndef __EMSCRIPTEN__ +inline const icu::Normalizer2 *getNFD(UErrorCode &status) { + const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(status); + UASSERT_SUCCESS(status); + return nfd; +} +inline const icu::Normalizer2 *getNFC(UErrorCode &status) { + const icu::Normalizer2 *nfc = icu::Normalizer2::getNFCInstance(status); + UASSERT_SUCCESS(status); + return nfc; +} +#endif + bool normalize_nfd(std::u32string &str) { std::u16string rstr = km::core::kmx::u32string_to_u16string(str); if(!km::core::util::normalize_nfd(rstr)) { @@ -38,6 +59,16 @@ bool normalize_nfd(std::u32string &str) { } } +bool normalize_nfc(std::u32string &str) { + std::u16string rstr = km::core::kmx::u32string_to_u16string(str); + if(!km::core::util::normalize_nfc(rstr)) { + return false; + } else { + str = km::core::kmx::u16string_to_u32string(rstr); + return true; + } +} + bool normalize_nfd(std::u16string &str) { #ifdef __EMSCRIPTEN__ std::string instr = convert(str); @@ -53,9 +84,26 @@ bool normalize_nfd(std::u16string &str) { return true; #else UErrorCode status = U_ZERO_ERROR; - const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(status); - UASSERT_SUCCESS(status); - return normalize(nfd, str, status); + return normalize(getNFD(status), str, status); +#endif +} + +bool normalize_nfc(std::u16string &str) { +#ifdef __EMSCRIPTEN__ + std::string instr = convert(str); + const char *in = instr.c_str(); + char *out = NormalizeNFC(in); + if (out == nullptr) { + assert(out != nullptr); + return false; + } + std::string outstr(out); + str = convert(outstr); + free(out); + return true; +#else + UErrorCode status = U_ZERO_ERROR; + return normalize(getNFC(status), str, status); #endif } @@ -67,19 +115,15 @@ bool normalize_nfd(km_core_cu const * src, std::u16string &dst) { dst = std::u16string(src); return normalize_nfd(dst); // vector to above fcn #else - UErrorCode icu_status = U_ZERO_ERROR; - const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(icu_status); - assert(U_SUCCESS(icu_status)); - if(!U_SUCCESS(icu_status)) { - // TODO: log the failure code + UErrorCode status = U_ZERO_ERROR; + auto nfd = getNFD(status); + if (nfd == nullptr) { return false; } icu::UnicodeString udst; icu::UnicodeString usrc = icu::UnicodeString(src); - nfd->normalize(usrc, udst, icu_status); - assert(U_SUCCESS(icu_status)); - if(!U_SUCCESS(icu_status)) { - // TODO: log the failure code + nfd->normalize(usrc, udst, status); + if(!UASSERT_SUCCESS(status)) { return false; } @@ -123,6 +167,64 @@ normalize_nfd(km_core_usv cp, std::u32string &dst) { #endif } +bool is_nfd(const std::u16string& str) { +#ifdef __EMSCRIPTEN__ + std::u16string o = str; + normalize_nfd(o); + return (o == str); // false if changed +#else + UErrorCode status = U_ZERO_ERROR; + auto nfd = getNFD(status); + if (nfd == nullptr) return false; + auto ustr = icu::UnicodeString(false, str.c_str(), (int)str.length()); + auto result = nfd->isNormalized(ustr, status); + if (!UASSERT_SUCCESS(status)) { + return false; + } else { + return result; + } +#endif +} + +bool is_nfd(const std::u32string& str) { +#ifdef __EMSCRIPTEN__ + std::u32string o = str; + normalize_nfd(o); + return (o == str); // false if changed +#else + UErrorCode status = U_ZERO_ERROR; + auto nfd = getNFD(status); + if (nfd == nullptr) return false; + auto ustr = icu::UnicodeString::fromUTF32(reinterpret_cast(str.c_str()), (int)str.length()); + auto result = nfd->isNormalized(ustr, status); + if (!UASSERT_SUCCESS(status)) { + return false; + } else { + return result; + } +#endif +} + +bool has_nfd_boundary_before(km_core_usv cp) { +#ifdef __EMSCRIPTEN__ +#error TODO +#else + UErrorCode status = U_ZERO_ERROR; + auto nfd = getNFD(status); + if (nfd == nullptr) return false; + return nfd->hasBoundaryBefore(cp); +#endif +} + +/** + * Helper to convert icu::UnicodeString to a UTF-32 km_core_usv buffer, + * nul-terminated + */ +km_core_usv *string_to_usv(const std::u32string& src) { + return km::core::kmx::u32dup(src.c_str()); +} + + } } } diff --git a/core/src/util_normalize.hpp b/core/src/util_normalize.hpp index d417617565..6321ffacda 100644 --- a/core/src/util_normalize.hpp +++ b/core/src/util_normalize.hpp @@ -14,6 +14,12 @@ namespace km { namespace core { namespace util { +/** Normalize a u32string inplace to NFC. @return false on failure */ +bool normalize_nfc(std::u32string &str); + +/** Normalize a u16string inplace to NFC. @return false on failure */ +bool normalize_nfc(std::u16string &str); + /** Normalize a u32string inplace to NFD. @return false on failure */ bool normalize_nfd(std::u32string &str); @@ -26,6 +32,18 @@ bool normalize_nfd(km_core_cu const * src, std::u16string &dst); /** normalize (decompose) a single cp to string. @return false on failure */ bool normalize_nfd(km_core_usv cp, std::u32string &dst); +/** @return true if string is already NFD */ +bool is_nfd(const std::u16string& str); + +/** @return true if string is already NFD */ +bool is_nfd(const std::u32string& str); + +/** @return true if cp can interacts with prior chars */ +bool has_nfd_boundary_before(km_core_usv cp); + +/** convenience function, caller owns storage */ +km_core_usv *string_to_usv(const std::u32string& src); + } } } From e2147daac8be62b7d9beddb0096f03426890aea7 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Mon, 3 Jun 2024 21:58:26 +0200 Subject: [PATCH 04/64] chore(linux): Update debian changelog (cherry picked from commit e96becc26d12c0506e07de5a0d281902f92ce288) --- linux/debian/changelog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/linux/debian/changelog b/linux/debian/changelog index e5e2a939b6..2f89e332c0 100644 --- a/linux/debian/changelog +++ b/linux/debian/changelog @@ -1,3 +1,10 @@ +keyman (17.0.326-1) unstable; urgency=medium + + * New upstream release + * Re-release to Debian + + -- Eberhard Beilharz Mon, 03 Jun 2024 21:58:16 +0200 + keyman (17.0.295-1) unstable; urgency=medium * Remove ibus-keyman.post{inst,rm} (closes: #1034040) From 53a6638a70c243d6078b83eb51154318bc78bd5e Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Tue, 4 Jun 2024 11:15:17 -0500 Subject: [PATCH 05/64] feat(core): generate and use static table in wasm for NFD boundary - add core/tools build tree with custom targets - add to core/build.sh to generate nfd_table.h - test_unicode to validate Unicode version and compare NFD to actual ICU - currently, linear search of the table. --- core/build.sh | 6 + core/meson.build | 1 + core/src/util_normalize.cpp | 11 +- core/tests/unit/ldml/test_unicode.cpp | 41 + core/tools/meson.build | 40 + core/tools/norm_unicode_update.cpp | 67 ++ .../unicode-character-database/nfd_table.h | 932 ++++++++++++++++++ 7 files changed, 1097 insertions(+), 1 deletion(-) create mode 100644 core/tools/meson.build create mode 100644 core/tools/norm_unicode_update.cpp create mode 100644 resources/standards-data/unicode-character-database/nfd_table.h diff --git a/core/build.sh b/core/build.sh index 168cc1cbcf..016431cfdd 100755 --- a/core/build.sh +++ b/core/build.sh @@ -68,6 +68,7 @@ Libraries will be built in 'build///src'. "uninstall uninstall libraries from current system" \ "${archtargets[@]}" \ "--no-tests do not configure tests (used by other projects)" \ + "--update-unicode rebuild tables if the ICU4C Unicode version changes" \ "--test,-t=opt_tests test[s] to run (space separated)" builder_parse "$@" @@ -177,6 +178,11 @@ if builder_start_action test:mac; then builder_finish_action success test:mac fi +if builder_has_option --update-unicode; then + meson compile -C ${KEYMAN_ROOT}/core/build/mac-x86_64/$BUILDER_CONFIGURATION tools/norm_data && mv -v ${KEYMAN_ROOT}/core/build/mac-x86_64/$BUILDER_CONFIGURATION/tools/nfd_table.h ${KEYMAN_ROOT}/resources/standards-data/unicode-character-database/ +fi + + # ------------------------------------------------------------------------------- do_action install diff --git a/core/meson.build b/core/meson.build index 0049c95741..ff306991e9 100644 --- a/core/meson.build +++ b/core/meson.build @@ -44,3 +44,4 @@ subdir('doc') subdir('include') subdir('src') subdir('tests') +subdir('tools') diff --git a/core/src/util_normalize.cpp b/core/src/util_normalize.cpp index b5c3b860d1..7678a9d4b0 100644 --- a/core/src/util_normalize.cpp +++ b/core/src/util_normalize.cpp @@ -30,6 +30,9 @@ EM_JS(char*, NormalizeNFC, (const char* input), { return stringToNewUTF8(nfd); }); +// pull in the generated table +#include "../../resources/standards-data/unicode-character-database/nfd_table.h" + #endif namespace km { @@ -207,7 +210,13 @@ bool is_nfd(const std::u32string& str) { bool has_nfd_boundary_before(km_core_usv cp) { #ifdef __EMSCRIPTEN__ -#error TODO +// it's a negative table. entries in the table mean returning false. non-entries return true. + for (int i=0;;i++) { + auto t = km_noBoundaryBefore[i]; + if (t == 0) return true; + if (t > cp) return true; + if (t == cp) return false; + } #else UErrorCode status = U_ZERO_ERROR; auto nfd = getNFD(status); diff --git a/core/tests/unit/ldml/test_unicode.cpp b/core/tests/unit/ldml/test_unicode.cpp index ae41853153..ea1d9c6cca 100644 --- a/core/tests/unit/ldml/test_unicode.cpp +++ b/core/tests/unit/ldml/test_unicode.cpp @@ -25,6 +25,8 @@ #include #include #include "json.hpp" +#include "util_normalize.hpp" +#include "kmx/kmx_xstring.h" #include #include @@ -43,6 +45,11 @@ } \ } +#ifdef __EMSCRIPTEN__ +// Pull this in to verify versions +#include "../../../../resources/standards-data/unicode-character-database/nfd_table.h" +#endif + //------------------------------------------------------------------------------------- // Unicode version tests //------------------------------------------------------------------------------------- @@ -149,6 +156,36 @@ const std::string &block_unicode_ver) { std::cout << std::endl; } +#ifdef __EMSCRIPTEN__ +inline const char *boolstr(bool b) { + return b?"T":"f"; +} + +void test_has_boundary_before() { + std::cout << "= " << __FUNCTION__ << std::endl; + std::cout << "(this test only runs under emscripten. congratulations.)" << std::endl; + // static_assert(U_UNICODE_VERSION == KM_HASBOUNDARYBEFORE_UNICODE_VERSION, "nfd_table.h Unicode version does not match ICU's - see nfd_table.h"); + std::cout << U_UNICODE_VERSION << "≈≈" << KM_HASBOUNDARYBEFORE_UNICODE_VERSION << std::endl; + + UErrorCode status = U_ZERO_ERROR; + const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(status); + UASSERT_SUCCESS(status); + + // now, test that hasBoundaryBefore is the same + for (km_core_usv cp = 0; cp < km::core::kmx::Uni_MAX_CODEPOINT; cp++) { + auto km_hbb = km::core::util::has_nfd_boundary_before(cp); + auto icu_hbb = nfd->hasBoundaryBefore(cp); + + if (km_hbb != icu_hbb) { + std::cerr << "Error: nfd_table.h said " << boolstr(km_hbb) << " but ICU said " << boolstr(icu_hbb) << " for " + << "has_nfd_boundary_before(0x" << std::hex << cp << std::dec << ")" << std::endl; + } + assert(km_hbb == icu_hbb); + } + std::cout << "All OK!" << std::endl; +} +#endif + int test_all(const char *jsonpath, const char *packagepath, const char *blockspath) { std::cout << "= " << __FUNCTION__ << std::endl; @@ -164,6 +201,10 @@ int test_all(const char *jsonpath, const char *packagepath, const char *blockspa test_unicode_versions(versions, package, block_unicode_ver); +#ifdef __EMSCRIPTEN__ + test_has_boundary_before(); +#endif + return EXIT_SUCCESS; } diff --git a/core/tools/meson.build b/core/tools/meson.build new file mode 100644 index 0000000000..31f3a00848 --- /dev/null +++ b/core/tools/meson.build @@ -0,0 +1,40 @@ +# Copyright: © 2024 SIL International. +# Description: Cross platform build script to compile tool(s). +# Create Date: 31 May 2024 +# Authors: Steven R. Loomis (SRL) +# + + +# TODO -- why are these differing from the standard.meson.build flags? +if cpp_compiler.get_id() == 'gcc' or cpp_compiler.get_id() == 'clang' or cpp_compiler.get_id() == 'emscripten' + warns = [ + '-Wno-missing-field-initializers', + '-Wno-unused-parameter' + ] +else + warns = [] +endif + + +if cpp_compiler.get_id() == 'emscripten' + tests_flags += ['-lnodefs.js', + '-sNO_DISABLE_EXCEPTION_CATCHING', # for test exceptions + wasm_exported_runtime_methods] +endif + +norm_unicode_update = executable('norm_unicode_update', + ['norm_unicode_update.cpp'], + cpp_args: defns + warns, + include_directories: [inc, libsrc, '../../developer/src/ext/json'], + link_args: links + tests_flags, + dependencies: [icu_uc, icu_i18n], + # link_with: [lib], + objects: lib.extract_all_objects(recursive: false), + ) + + +# ../../resources/standards-data/unicode-character-database/ +norm_data = custom_target('norm_data', output: 'nfd_table.h', command: [norm_unicode_update, '@OUTPUT@']) + + +# TODO: execute it diff --git a/core/tools/norm_unicode_update.cpp b/core/tools/norm_unicode_update.cpp new file mode 100644 index 0000000000..c2c290ba9b --- /dev/null +++ b/core/tools/norm_unicode_update.cpp @@ -0,0 +1,67 @@ +#include "kmx/kmx_plus.h" +#include "kmx/kmx_xstring.h" +#include "core_icu.h" + +#include +#include +#include + +#include + +#include + +#ifndef __EMSCRIPTEN__ + +int +write_nfd_table(const char *NFD_FILE) { + std::cout << " writing: " << NFD_FILE << std::endl; + auto f = std::ofstream(NFD_FILE); + assert(f.good()); + + // write preamble + f << "//NFD hasBoundaryBefore" << std::endl; + f << "#pragma once" << std::endl; + f << "#define KM_HASBOUNDARYBEFORE_UNICODE_VERSION \"" << U_UNICODE_VERSION << "\"" << std::endl; + f << "#define KM_HASBOUNDARYBEFORE_ICU_VERSION \"" << U_ICU_VERSION << "\"" << std::endl; + f << "static char32_t km_noBoundaryBefore[] = {" << std::endl; + // we're going to need an NFD normalizer + UErrorCode status = U_ZERO_ERROR; + const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(status); + assert(U_SUCCESS(status)); + + for (km_core_usv ch = 0; ch < 0x10FFFF; ch++) { + bool bb = nfd->hasBoundaryBefore(ch); + assert(!(ch == 0 && !bb)); // assert that we can use U+0000 as a terminator + + // TODO: This test may be better in test_unicode + // icu::UnicodeString s; + // s.append((UChar32)ch); + // bool lccc = nfd->isNormalized(s, status) && u_getCombiningClass(ch) == 0; + // assert(U_SUCCESS(status)); + // if (bb != lccc) { + // printf("0x%04x - bb=%s but lccc=%s\n", (unsigned int)ch, bb ? "y" : "n", lccc ? "y" : "n"); + // } + // assert(bb == lccc); + if (bb) continue; //only emit nonboundary + // char key[10]; + // snprintf(key, 10, "%04X", (unsigned int)ch); + f << "\t0x" << std::hex << ch << "," << std::endl; + } + // termination + f << "\t0x" << std::hex << 0 << "," << std::endl; + f << "};" << std::endl; + return 0; +} + +int +main(int argc, const char *argv[]) { + assert(argc == 2); // call with one param: @OUTPUT@ + write_nfd_table(argv[1]); + return 0; +} +#else +int main(int argc, const char *argv[]) { + std::cerr << "Can't run this under Emscripten - run under another platform." << std::endl; + return 1; +} +#endif diff --git a/resources/standards-data/unicode-character-database/nfd_table.h b/resources/standards-data/unicode-character-database/nfd_table.h new file mode 100644 index 0000000000..4e157e5b6b --- /dev/null +++ b/resources/standards-data/unicode-character-database/nfd_table.h @@ -0,0 +1,932 @@ +//NFD hasBoundaryBefore +#pragma once +#define KM_HASBOUNDARYBEFORE_UNICODE_VERSION "15.0" +#define KM_HASBOUNDARYBEFORE_ICU_VERSION "73.1" +static char32_t km_noBoundaryBefore[] = { + 0x300, + 0x301, + 0x302, + 0x303, + 0x304, + 0x305, + 0x306, + 0x307, + 0x308, + 0x309, + 0x30a, + 0x30b, + 0x30c, + 0x30d, + 0x30e, + 0x30f, + 0x310, + 0x311, + 0x312, + 0x313, + 0x314, + 0x315, + 0x316, + 0x317, + 0x318, + 0x319, + 0x31a, + 0x31b, + 0x31c, + 0x31d, + 0x31e, + 0x31f, + 0x320, + 0x321, + 0x322, + 0x323, + 0x324, + 0x325, + 0x326, + 0x327, + 0x328, + 0x329, + 0x32a, + 0x32b, + 0x32c, + 0x32d, + 0x32e, + 0x32f, + 0x330, + 0x331, + 0x332, + 0x333, + 0x334, + 0x335, + 0x336, + 0x337, + 0x338, + 0x339, + 0x33a, + 0x33b, + 0x33c, + 0x33d, + 0x33e, + 0x33f, + 0x340, + 0x341, + 0x342, + 0x343, + 0x344, + 0x345, + 0x346, + 0x347, + 0x348, + 0x349, + 0x34a, + 0x34b, + 0x34c, + 0x34d, + 0x34e, + 0x350, + 0x351, + 0x352, + 0x353, + 0x354, + 0x355, + 0x356, + 0x357, + 0x358, + 0x359, + 0x35a, + 0x35b, + 0x35c, + 0x35d, + 0x35e, + 0x35f, + 0x360, + 0x361, + 0x362, + 0x363, + 0x364, + 0x365, + 0x366, + 0x367, + 0x368, + 0x369, + 0x36a, + 0x36b, + 0x36c, + 0x36d, + 0x36e, + 0x36f, + 0x483, + 0x484, + 0x485, + 0x486, + 0x487, + 0x591, + 0x592, + 0x593, + 0x594, + 0x595, + 0x596, + 0x597, + 0x598, + 0x599, + 0x59a, + 0x59b, + 0x59c, + 0x59d, + 0x59e, + 0x59f, + 0x5a0, + 0x5a1, + 0x5a2, + 0x5a3, + 0x5a4, + 0x5a5, + 0x5a6, + 0x5a7, + 0x5a8, + 0x5a9, + 0x5aa, + 0x5ab, + 0x5ac, + 0x5ad, + 0x5ae, + 0x5af, + 0x5b0, + 0x5b1, + 0x5b2, + 0x5b3, + 0x5b4, + 0x5b5, + 0x5b6, + 0x5b7, + 0x5b8, + 0x5b9, + 0x5ba, + 0x5bb, + 0x5bc, + 0x5bd, + 0x5bf, + 0x5c1, + 0x5c2, + 0x5c4, + 0x5c5, + 0x5c7, + 0x610, + 0x611, + 0x612, + 0x613, + 0x614, + 0x615, + 0x616, + 0x617, + 0x618, + 0x619, + 0x61a, + 0x64b, + 0x64c, + 0x64d, + 0x64e, + 0x64f, + 0x650, + 0x651, + 0x652, + 0x653, + 0x654, + 0x655, + 0x656, + 0x657, + 0x658, + 0x659, + 0x65a, + 0x65b, + 0x65c, + 0x65d, + 0x65e, + 0x65f, + 0x670, + 0x6d6, + 0x6d7, + 0x6d8, + 0x6d9, + 0x6da, + 0x6db, + 0x6dc, + 0x6df, + 0x6e0, + 0x6e1, + 0x6e2, + 0x6e3, + 0x6e4, + 0x6e7, + 0x6e8, + 0x6ea, + 0x6eb, + 0x6ec, + 0x6ed, + 0x711, + 0x730, + 0x731, + 0x732, + 0x733, + 0x734, + 0x735, + 0x736, + 0x737, + 0x738, + 0x739, + 0x73a, + 0x73b, + 0x73c, + 0x73d, + 0x73e, + 0x73f, + 0x740, + 0x741, + 0x742, + 0x743, + 0x744, + 0x745, + 0x746, + 0x747, + 0x748, + 0x749, + 0x74a, + 0x7eb, + 0x7ec, + 0x7ed, + 0x7ee, + 0x7ef, + 0x7f0, + 0x7f1, + 0x7f2, + 0x7f3, + 0x7fd, + 0x816, + 0x817, + 0x818, + 0x819, + 0x81b, + 0x81c, + 0x81d, + 0x81e, + 0x81f, + 0x820, + 0x821, + 0x822, + 0x823, + 0x825, + 0x826, + 0x827, + 0x829, + 0x82a, + 0x82b, + 0x82c, + 0x82d, + 0x859, + 0x85a, + 0x85b, + 0x898, + 0x899, + 0x89a, + 0x89b, + 0x89c, + 0x89d, + 0x89e, + 0x89f, + 0x8ca, + 0x8cb, + 0x8cc, + 0x8cd, + 0x8ce, + 0x8cf, + 0x8d0, + 0x8d1, + 0x8d2, + 0x8d3, + 0x8d4, + 0x8d5, + 0x8d6, + 0x8d7, + 0x8d8, + 0x8d9, + 0x8da, + 0x8db, + 0x8dc, + 0x8dd, + 0x8de, + 0x8df, + 0x8e0, + 0x8e1, + 0x8e3, + 0x8e4, + 0x8e5, + 0x8e6, + 0x8e7, + 0x8e8, + 0x8e9, + 0x8ea, + 0x8eb, + 0x8ec, + 0x8ed, + 0x8ee, + 0x8ef, + 0x8f0, + 0x8f1, + 0x8f2, + 0x8f3, + 0x8f4, + 0x8f5, + 0x8f6, + 0x8f7, + 0x8f8, + 0x8f9, + 0x8fa, + 0x8fb, + 0x8fc, + 0x8fd, + 0x8fe, + 0x8ff, + 0x93c, + 0x94d, + 0x951, + 0x952, + 0x953, + 0x954, + 0x9bc, + 0x9cd, + 0x9fe, + 0xa3c, + 0xa4d, + 0xabc, + 0xacd, + 0xb3c, + 0xb4d, + 0xbcd, + 0xc3c, + 0xc4d, + 0xc55, + 0xc56, + 0xcbc, + 0xccd, + 0xd3b, + 0xd3c, + 0xd4d, + 0xdca, + 0xe38, + 0xe39, + 0xe3a, + 0xe48, + 0xe49, + 0xe4a, + 0xe4b, + 0xeb8, + 0xeb9, + 0xeba, + 0xec8, + 0xec9, + 0xeca, + 0xecb, + 0xf18, + 0xf19, + 0xf35, + 0xf37, + 0xf39, + 0xf71, + 0xf72, + 0xf73, + 0xf74, + 0xf75, + 0xf7a, + 0xf7b, + 0xf7c, + 0xf7d, + 0xf80, + 0xf81, + 0xf82, + 0xf83, + 0xf84, + 0xf86, + 0xf87, + 0xfc6, + 0x1037, + 0x1039, + 0x103a, + 0x108d, + 0x135d, + 0x135e, + 0x135f, + 0x1714, + 0x1715, + 0x1734, + 0x17d2, + 0x17dd, + 0x18a9, + 0x1939, + 0x193a, + 0x193b, + 0x1a17, + 0x1a18, + 0x1a60, + 0x1a75, + 0x1a76, + 0x1a77, + 0x1a78, + 0x1a79, + 0x1a7a, + 0x1a7b, + 0x1a7c, + 0x1a7f, + 0x1ab0, + 0x1ab1, + 0x1ab2, + 0x1ab3, + 0x1ab4, + 0x1ab5, + 0x1ab6, + 0x1ab7, + 0x1ab8, + 0x1ab9, + 0x1aba, + 0x1abb, + 0x1abc, + 0x1abd, + 0x1abf, + 0x1ac0, + 0x1ac1, + 0x1ac2, + 0x1ac3, + 0x1ac4, + 0x1ac5, + 0x1ac6, + 0x1ac7, + 0x1ac8, + 0x1ac9, + 0x1aca, + 0x1acb, + 0x1acc, + 0x1acd, + 0x1ace, + 0x1b34, + 0x1b44, + 0x1b6b, + 0x1b6c, + 0x1b6d, + 0x1b6e, + 0x1b6f, + 0x1b70, + 0x1b71, + 0x1b72, + 0x1b73, + 0x1baa, + 0x1bab, + 0x1be6, + 0x1bf2, + 0x1bf3, + 0x1c37, + 0x1cd0, + 0x1cd1, + 0x1cd2, + 0x1cd4, + 0x1cd5, + 0x1cd6, + 0x1cd7, + 0x1cd8, + 0x1cd9, + 0x1cda, + 0x1cdb, + 0x1cdc, + 0x1cdd, + 0x1cde, + 0x1cdf, + 0x1ce0, + 0x1ce2, + 0x1ce3, + 0x1ce4, + 0x1ce5, + 0x1ce6, + 0x1ce7, + 0x1ce8, + 0x1ced, + 0x1cf4, + 0x1cf8, + 0x1cf9, + 0x1dc0, + 0x1dc1, + 0x1dc2, + 0x1dc3, + 0x1dc4, + 0x1dc5, + 0x1dc6, + 0x1dc7, + 0x1dc8, + 0x1dc9, + 0x1dca, + 0x1dcb, + 0x1dcc, + 0x1dcd, + 0x1dce, + 0x1dcf, + 0x1dd0, + 0x1dd1, + 0x1dd2, + 0x1dd3, + 0x1dd4, + 0x1dd5, + 0x1dd6, + 0x1dd7, + 0x1dd8, + 0x1dd9, + 0x1dda, + 0x1ddb, + 0x1ddc, + 0x1ddd, + 0x1dde, + 0x1ddf, + 0x1de0, + 0x1de1, + 0x1de2, + 0x1de3, + 0x1de4, + 0x1de5, + 0x1de6, + 0x1de7, + 0x1de8, + 0x1de9, + 0x1dea, + 0x1deb, + 0x1dec, + 0x1ded, + 0x1dee, + 0x1def, + 0x1df0, + 0x1df1, + 0x1df2, + 0x1df3, + 0x1df4, + 0x1df5, + 0x1df6, + 0x1df7, + 0x1df8, + 0x1df9, + 0x1dfa, + 0x1dfb, + 0x1dfc, + 0x1dfd, + 0x1dfe, + 0x1dff, + 0x20d0, + 0x20d1, + 0x20d2, + 0x20d3, + 0x20d4, + 0x20d5, + 0x20d6, + 0x20d7, + 0x20d8, + 0x20d9, + 0x20da, + 0x20db, + 0x20dc, + 0x20e1, + 0x20e5, + 0x20e6, + 0x20e7, + 0x20e8, + 0x20e9, + 0x20ea, + 0x20eb, + 0x20ec, + 0x20ed, + 0x20ee, + 0x20ef, + 0x20f0, + 0x2cef, + 0x2cf0, + 0x2cf1, + 0x2d7f, + 0x2de0, + 0x2de1, + 0x2de2, + 0x2de3, + 0x2de4, + 0x2de5, + 0x2de6, + 0x2de7, + 0x2de8, + 0x2de9, + 0x2dea, + 0x2deb, + 0x2dec, + 0x2ded, + 0x2dee, + 0x2def, + 0x2df0, + 0x2df1, + 0x2df2, + 0x2df3, + 0x2df4, + 0x2df5, + 0x2df6, + 0x2df7, + 0x2df8, + 0x2df9, + 0x2dfa, + 0x2dfb, + 0x2dfc, + 0x2dfd, + 0x2dfe, + 0x2dff, + 0x302a, + 0x302b, + 0x302c, + 0x302d, + 0x302e, + 0x302f, + 0x3099, + 0x309a, + 0xa66f, + 0xa674, + 0xa675, + 0xa676, + 0xa677, + 0xa678, + 0xa679, + 0xa67a, + 0xa67b, + 0xa67c, + 0xa67d, + 0xa69e, + 0xa69f, + 0xa6f0, + 0xa6f1, + 0xa806, + 0xa82c, + 0xa8c4, + 0xa8e0, + 0xa8e1, + 0xa8e2, + 0xa8e3, + 0xa8e4, + 0xa8e5, + 0xa8e6, + 0xa8e7, + 0xa8e8, + 0xa8e9, + 0xa8ea, + 0xa8eb, + 0xa8ec, + 0xa8ed, + 0xa8ee, + 0xa8ef, + 0xa8f0, + 0xa8f1, + 0xa92b, + 0xa92c, + 0xa92d, + 0xa953, + 0xa9b3, + 0xa9c0, + 0xaab0, + 0xaab2, + 0xaab3, + 0xaab4, + 0xaab7, + 0xaab8, + 0xaabe, + 0xaabf, + 0xaac1, + 0xaaf6, + 0xabed, + 0xfb1e, + 0xfe20, + 0xfe21, + 0xfe22, + 0xfe23, + 0xfe24, + 0xfe25, + 0xfe26, + 0xfe27, + 0xfe28, + 0xfe29, + 0xfe2a, + 0xfe2b, + 0xfe2c, + 0xfe2d, + 0xfe2e, + 0xfe2f, + 0x101fd, + 0x102e0, + 0x10376, + 0x10377, + 0x10378, + 0x10379, + 0x1037a, + 0x10a0d, + 0x10a0f, + 0x10a38, + 0x10a39, + 0x10a3a, + 0x10a3f, + 0x10ae5, + 0x10ae6, + 0x10d24, + 0x10d25, + 0x10d26, + 0x10d27, + 0x10eab, + 0x10eac, + 0x10efd, + 0x10efe, + 0x10eff, + 0x10f46, + 0x10f47, + 0x10f48, + 0x10f49, + 0x10f4a, + 0x10f4b, + 0x10f4c, + 0x10f4d, + 0x10f4e, + 0x10f4f, + 0x10f50, + 0x10f82, + 0x10f83, + 0x10f84, + 0x10f85, + 0x11046, + 0x11070, + 0x1107f, + 0x110b9, + 0x110ba, + 0x11100, + 0x11101, + 0x11102, + 0x11133, + 0x11134, + 0x11173, + 0x111c0, + 0x111ca, + 0x11235, + 0x11236, + 0x112e9, + 0x112ea, + 0x1133b, + 0x1133c, + 0x1134d, + 0x11366, + 0x11367, + 0x11368, + 0x11369, + 0x1136a, + 0x1136b, + 0x1136c, + 0x11370, + 0x11371, + 0x11372, + 0x11373, + 0x11374, + 0x11442, + 0x11446, + 0x1145e, + 0x114c2, + 0x114c3, + 0x115bf, + 0x115c0, + 0x1163f, + 0x116b6, + 0x116b7, + 0x1172b, + 0x11839, + 0x1183a, + 0x1193d, + 0x1193e, + 0x11943, + 0x119e0, + 0x11a34, + 0x11a47, + 0x11a99, + 0x11c3f, + 0x11d42, + 0x11d44, + 0x11d45, + 0x11d97, + 0x11f41, + 0x11f42, + 0x16af0, + 0x16af1, + 0x16af2, + 0x16af3, + 0x16af4, + 0x16b30, + 0x16b31, + 0x16b32, + 0x16b33, + 0x16b34, + 0x16b35, + 0x16b36, + 0x16ff0, + 0x16ff1, + 0x1bc9e, + 0x1d165, + 0x1d166, + 0x1d167, + 0x1d168, + 0x1d169, + 0x1d16d, + 0x1d16e, + 0x1d16f, + 0x1d170, + 0x1d171, + 0x1d172, + 0x1d17b, + 0x1d17c, + 0x1d17d, + 0x1d17e, + 0x1d17f, + 0x1d180, + 0x1d181, + 0x1d182, + 0x1d185, + 0x1d186, + 0x1d187, + 0x1d188, + 0x1d189, + 0x1d18a, + 0x1d18b, + 0x1d1aa, + 0x1d1ab, + 0x1d1ac, + 0x1d1ad, + 0x1d242, + 0x1d243, + 0x1d244, + 0x1e000, + 0x1e001, + 0x1e002, + 0x1e003, + 0x1e004, + 0x1e005, + 0x1e006, + 0x1e008, + 0x1e009, + 0x1e00a, + 0x1e00b, + 0x1e00c, + 0x1e00d, + 0x1e00e, + 0x1e00f, + 0x1e010, + 0x1e011, + 0x1e012, + 0x1e013, + 0x1e014, + 0x1e015, + 0x1e016, + 0x1e017, + 0x1e018, + 0x1e01b, + 0x1e01c, + 0x1e01d, + 0x1e01e, + 0x1e01f, + 0x1e020, + 0x1e021, + 0x1e023, + 0x1e024, + 0x1e026, + 0x1e027, + 0x1e028, + 0x1e029, + 0x1e02a, + 0x1e08f, + 0x1e130, + 0x1e131, + 0x1e132, + 0x1e133, + 0x1e134, + 0x1e135, + 0x1e136, + 0x1e2ae, + 0x1e2ec, + 0x1e2ed, + 0x1e2ee, + 0x1e2ef, + 0x1e4ec, + 0x1e4ed, + 0x1e4ee, + 0x1e4ef, + 0x1e8d0, + 0x1e8d1, + 0x1e8d2, + 0x1e8d3, + 0x1e8d4, + 0x1e8d5, + 0x1e8d6, + 0x1e944, + 0x1e945, + 0x1e946, + 0x1e947, + 0x1e948, + 0x1e949, + 0x1e94a, + 0x0, +}; From e9673867fa550d68ed7282c291af3198a1f5e603 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Tue, 4 Jun 2024 12:23:16 -0500 Subject: [PATCH 06/64] feat(core): speedup NFD boundary table - use RLE encoding, thanks @mcdurdin - much smaller table and faster lookup Fixes: #9467 --- core/src/util_normalize.cpp | 12 +- core/tools/norm_unicode_update.cpp | 47 +- .../unicode-character-database/nfd_table.h | 1119 +++-------------- 3 files changed, 231 insertions(+), 947 deletions(-) diff --git a/core/src/util_normalize.cpp b/core/src/util_normalize.cpp index 7678a9d4b0..7d0d76d8ab 100644 --- a/core/src/util_normalize.cpp +++ b/core/src/util_normalize.cpp @@ -211,12 +211,14 @@ bool is_nfd(const std::u32string& str) { bool has_nfd_boundary_before(km_core_usv cp) { #ifdef __EMSCRIPTEN__ // it's a negative table. entries in the table mean returning false. non-entries return true. - for (int i=0;;i++) { - auto t = km_noBoundaryBefore[i]; - if (t == 0) return true; - if (t > cp) return true; - if (t == cp) return false; + for (auto i=0;i<(km_noBoundaryBefore_entries*2);i+=2) { + auto start = km_noBoundaryBefore[i+0]; + if (start > cp) return true; + auto count = km_noBoundaryBefore[i+1]; + auto limit = start+count; + if (cp >= start && cp < limit) return false; } + return true; // fallthrough #else UErrorCode status = U_ZERO_ERROR; auto nfd = getNFD(status); diff --git a/core/tools/norm_unicode_update.cpp b/core/tools/norm_unicode_update.cpp index c2c290ba9b..408dbb0c98 100644 --- a/core/tools/norm_unicode_update.cpp +++ b/core/tools/norm_unicode_update.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include @@ -23,32 +24,48 @@ write_nfd_table(const char *NFD_FILE) { f << "#pragma once" << std::endl; f << "#define KM_HASBOUNDARYBEFORE_UNICODE_VERSION \"" << U_UNICODE_VERSION << "\"" << std::endl; f << "#define KM_HASBOUNDARYBEFORE_ICU_VERSION \"" << U_ICU_VERSION << "\"" << std::endl; - f << "static char32_t km_noBoundaryBefore[] = {" << std::endl; // we're going to need an NFD normalizer UErrorCode status = U_ZERO_ERROR; const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(status); assert(U_SUCCESS(status)); + std::vector noBoundary; + for (km_core_usv ch = 0; ch < 0x10FFFF; ch++) { bool bb = nfd->hasBoundaryBefore(ch); assert(!(ch == 0 && !bb)); // assert that we can use U+0000 as a terminator - - // TODO: This test may be better in test_unicode - // icu::UnicodeString s; - // s.append((UChar32)ch); - // bool lccc = nfd->isNormalized(s, status) && u_getCombiningClass(ch) == 0; - // assert(U_SUCCESS(status)); - // if (bb != lccc) { - // printf("0x%04x - bb=%s but lccc=%s\n", (unsigned int)ch, bb ? "y" : "n", lccc ? "y" : "n"); - // } - // assert(bb == lccc); if (bb) continue; //only emit nonboundary - // char key[10]; - // snprintf(key, 10, "%04X", (unsigned int)ch); - f << "\t0x" << std::hex << ch << "," << std::endl; + noBoundary.push_back(ch); } + + std::vector> runs; // start,len + + km_core_usv first = 0; + km_core_usv last = 0; + for(auto i = noBoundary.begin(); i <= noBoundary.end(); i++) { + if (first == 0) { + first = last = *i; + } else { + last++; + if(i == noBoundary.end() || *i != last) { + // end of a run + runs.emplace_back(first, last - first); + if (i != noBoundary.end()) { + // setup for next + first = last = *i; + } + } + } + } + f << "#define km_noBoundaryBefore_entries " << runs.size() << "\n"; + + f << "static char32_t km_noBoundaryBefore[km_noBoundaryBefore_entries * 2 ] = {" << std::endl; + + for (auto i = runs.begin(); i < runs.end(); i++) { + f << "\t0x" << std::hex << i->first << std::dec << ",\t " << i->second << ", // ...0x" << std::hex << (i->first+i->second-1) << std::endl; + } + // termination - f << "\t0x" << std::hex << 0 << "," << std::endl; f << "};" << std::endl; return 0; } diff --git a/resources/standards-data/unicode-character-database/nfd_table.h b/resources/standards-data/unicode-character-database/nfd_table.h index 4e157e5b6b..068240fc30 100644 --- a/resources/standards-data/unicode-character-database/nfd_table.h +++ b/resources/standards-data/unicode-character-database/nfd_table.h @@ -2,931 +2,196 @@ #pragma once #define KM_HASBOUNDARYBEFORE_UNICODE_VERSION "15.0" #define KM_HASBOUNDARYBEFORE_ICU_VERSION "73.1" -static char32_t km_noBoundaryBefore[] = { - 0x300, - 0x301, - 0x302, - 0x303, - 0x304, - 0x305, - 0x306, - 0x307, - 0x308, - 0x309, - 0x30a, - 0x30b, - 0x30c, - 0x30d, - 0x30e, - 0x30f, - 0x310, - 0x311, - 0x312, - 0x313, - 0x314, - 0x315, - 0x316, - 0x317, - 0x318, - 0x319, - 0x31a, - 0x31b, - 0x31c, - 0x31d, - 0x31e, - 0x31f, - 0x320, - 0x321, - 0x322, - 0x323, - 0x324, - 0x325, - 0x326, - 0x327, - 0x328, - 0x329, - 0x32a, - 0x32b, - 0x32c, - 0x32d, - 0x32e, - 0x32f, - 0x330, - 0x331, - 0x332, - 0x333, - 0x334, - 0x335, - 0x336, - 0x337, - 0x338, - 0x339, - 0x33a, - 0x33b, - 0x33c, - 0x33d, - 0x33e, - 0x33f, - 0x340, - 0x341, - 0x342, - 0x343, - 0x344, - 0x345, - 0x346, - 0x347, - 0x348, - 0x349, - 0x34a, - 0x34b, - 0x34c, - 0x34d, - 0x34e, - 0x350, - 0x351, - 0x352, - 0x353, - 0x354, - 0x355, - 0x356, - 0x357, - 0x358, - 0x359, - 0x35a, - 0x35b, - 0x35c, - 0x35d, - 0x35e, - 0x35f, - 0x360, - 0x361, - 0x362, - 0x363, - 0x364, - 0x365, - 0x366, - 0x367, - 0x368, - 0x369, - 0x36a, - 0x36b, - 0x36c, - 0x36d, - 0x36e, - 0x36f, - 0x483, - 0x484, - 0x485, - 0x486, - 0x487, - 0x591, - 0x592, - 0x593, - 0x594, - 0x595, - 0x596, - 0x597, - 0x598, - 0x599, - 0x59a, - 0x59b, - 0x59c, - 0x59d, - 0x59e, - 0x59f, - 0x5a0, - 0x5a1, - 0x5a2, - 0x5a3, - 0x5a4, - 0x5a5, - 0x5a6, - 0x5a7, - 0x5a8, - 0x5a9, - 0x5aa, - 0x5ab, - 0x5ac, - 0x5ad, - 0x5ae, - 0x5af, - 0x5b0, - 0x5b1, - 0x5b2, - 0x5b3, - 0x5b4, - 0x5b5, - 0x5b6, - 0x5b7, - 0x5b8, - 0x5b9, - 0x5ba, - 0x5bb, - 0x5bc, - 0x5bd, - 0x5bf, - 0x5c1, - 0x5c2, - 0x5c4, - 0x5c5, - 0x5c7, - 0x610, - 0x611, - 0x612, - 0x613, - 0x614, - 0x615, - 0x616, - 0x617, - 0x618, - 0x619, - 0x61a, - 0x64b, - 0x64c, - 0x64d, - 0x64e, - 0x64f, - 0x650, - 0x651, - 0x652, - 0x653, - 0x654, - 0x655, - 0x656, - 0x657, - 0x658, - 0x659, - 0x65a, - 0x65b, - 0x65c, - 0x65d, - 0x65e, - 0x65f, - 0x670, - 0x6d6, - 0x6d7, - 0x6d8, - 0x6d9, - 0x6da, - 0x6db, - 0x6dc, - 0x6df, - 0x6e0, - 0x6e1, - 0x6e2, - 0x6e3, - 0x6e4, - 0x6e7, - 0x6e8, - 0x6ea, - 0x6eb, - 0x6ec, - 0x6ed, - 0x711, - 0x730, - 0x731, - 0x732, - 0x733, - 0x734, - 0x735, - 0x736, - 0x737, - 0x738, - 0x739, - 0x73a, - 0x73b, - 0x73c, - 0x73d, - 0x73e, - 0x73f, - 0x740, - 0x741, - 0x742, - 0x743, - 0x744, - 0x745, - 0x746, - 0x747, - 0x748, - 0x749, - 0x74a, - 0x7eb, - 0x7ec, - 0x7ed, - 0x7ee, - 0x7ef, - 0x7f0, - 0x7f1, - 0x7f2, - 0x7f3, - 0x7fd, - 0x816, - 0x817, - 0x818, - 0x819, - 0x81b, - 0x81c, - 0x81d, - 0x81e, - 0x81f, - 0x820, - 0x821, - 0x822, - 0x823, - 0x825, - 0x826, - 0x827, - 0x829, - 0x82a, - 0x82b, - 0x82c, - 0x82d, - 0x859, - 0x85a, - 0x85b, - 0x898, - 0x899, - 0x89a, - 0x89b, - 0x89c, - 0x89d, - 0x89e, - 0x89f, - 0x8ca, - 0x8cb, - 0x8cc, - 0x8cd, - 0x8ce, - 0x8cf, - 0x8d0, - 0x8d1, - 0x8d2, - 0x8d3, - 0x8d4, - 0x8d5, - 0x8d6, - 0x8d7, - 0x8d8, - 0x8d9, - 0x8da, - 0x8db, - 0x8dc, - 0x8dd, - 0x8de, - 0x8df, - 0x8e0, - 0x8e1, - 0x8e3, - 0x8e4, - 0x8e5, - 0x8e6, - 0x8e7, - 0x8e8, - 0x8e9, - 0x8ea, - 0x8eb, - 0x8ec, - 0x8ed, - 0x8ee, - 0x8ef, - 0x8f0, - 0x8f1, - 0x8f2, - 0x8f3, - 0x8f4, - 0x8f5, - 0x8f6, - 0x8f7, - 0x8f8, - 0x8f9, - 0x8fa, - 0x8fb, - 0x8fc, - 0x8fd, - 0x8fe, - 0x8ff, - 0x93c, - 0x94d, - 0x951, - 0x952, - 0x953, - 0x954, - 0x9bc, - 0x9cd, - 0x9fe, - 0xa3c, - 0xa4d, - 0xabc, - 0xacd, - 0xb3c, - 0xb4d, - 0xbcd, - 0xc3c, - 0xc4d, - 0xc55, - 0xc56, - 0xcbc, - 0xccd, - 0xd3b, - 0xd3c, - 0xd4d, - 0xdca, - 0xe38, - 0xe39, - 0xe3a, - 0xe48, - 0xe49, - 0xe4a, - 0xe4b, - 0xeb8, - 0xeb9, - 0xeba, - 0xec8, - 0xec9, - 0xeca, - 0xecb, - 0xf18, - 0xf19, - 0xf35, - 0xf37, - 0xf39, - 0xf71, - 0xf72, - 0xf73, - 0xf74, - 0xf75, - 0xf7a, - 0xf7b, - 0xf7c, - 0xf7d, - 0xf80, - 0xf81, - 0xf82, - 0xf83, - 0xf84, - 0xf86, - 0xf87, - 0xfc6, - 0x1037, - 0x1039, - 0x103a, - 0x108d, - 0x135d, - 0x135e, - 0x135f, - 0x1714, - 0x1715, - 0x1734, - 0x17d2, - 0x17dd, - 0x18a9, - 0x1939, - 0x193a, - 0x193b, - 0x1a17, - 0x1a18, - 0x1a60, - 0x1a75, - 0x1a76, - 0x1a77, - 0x1a78, - 0x1a79, - 0x1a7a, - 0x1a7b, - 0x1a7c, - 0x1a7f, - 0x1ab0, - 0x1ab1, - 0x1ab2, - 0x1ab3, - 0x1ab4, - 0x1ab5, - 0x1ab6, - 0x1ab7, - 0x1ab8, - 0x1ab9, - 0x1aba, - 0x1abb, - 0x1abc, - 0x1abd, - 0x1abf, - 0x1ac0, - 0x1ac1, - 0x1ac2, - 0x1ac3, - 0x1ac4, - 0x1ac5, - 0x1ac6, - 0x1ac7, - 0x1ac8, - 0x1ac9, - 0x1aca, - 0x1acb, - 0x1acc, - 0x1acd, - 0x1ace, - 0x1b34, - 0x1b44, - 0x1b6b, - 0x1b6c, - 0x1b6d, - 0x1b6e, - 0x1b6f, - 0x1b70, - 0x1b71, - 0x1b72, - 0x1b73, - 0x1baa, - 0x1bab, - 0x1be6, - 0x1bf2, - 0x1bf3, - 0x1c37, - 0x1cd0, - 0x1cd1, - 0x1cd2, - 0x1cd4, - 0x1cd5, - 0x1cd6, - 0x1cd7, - 0x1cd8, - 0x1cd9, - 0x1cda, - 0x1cdb, - 0x1cdc, - 0x1cdd, - 0x1cde, - 0x1cdf, - 0x1ce0, - 0x1ce2, - 0x1ce3, - 0x1ce4, - 0x1ce5, - 0x1ce6, - 0x1ce7, - 0x1ce8, - 0x1ced, - 0x1cf4, - 0x1cf8, - 0x1cf9, - 0x1dc0, - 0x1dc1, - 0x1dc2, - 0x1dc3, - 0x1dc4, - 0x1dc5, - 0x1dc6, - 0x1dc7, - 0x1dc8, - 0x1dc9, - 0x1dca, - 0x1dcb, - 0x1dcc, - 0x1dcd, - 0x1dce, - 0x1dcf, - 0x1dd0, - 0x1dd1, - 0x1dd2, - 0x1dd3, - 0x1dd4, - 0x1dd5, - 0x1dd6, - 0x1dd7, - 0x1dd8, - 0x1dd9, - 0x1dda, - 0x1ddb, - 0x1ddc, - 0x1ddd, - 0x1dde, - 0x1ddf, - 0x1de0, - 0x1de1, - 0x1de2, - 0x1de3, - 0x1de4, - 0x1de5, - 0x1de6, - 0x1de7, - 0x1de8, - 0x1de9, - 0x1dea, - 0x1deb, - 0x1dec, - 0x1ded, - 0x1dee, - 0x1def, - 0x1df0, - 0x1df1, - 0x1df2, - 0x1df3, - 0x1df4, - 0x1df5, - 0x1df6, - 0x1df7, - 0x1df8, - 0x1df9, - 0x1dfa, - 0x1dfb, - 0x1dfc, - 0x1dfd, - 0x1dfe, - 0x1dff, - 0x20d0, - 0x20d1, - 0x20d2, - 0x20d3, - 0x20d4, - 0x20d5, - 0x20d6, - 0x20d7, - 0x20d8, - 0x20d9, - 0x20da, - 0x20db, - 0x20dc, - 0x20e1, - 0x20e5, - 0x20e6, - 0x20e7, - 0x20e8, - 0x20e9, - 0x20ea, - 0x20eb, - 0x20ec, - 0x20ed, - 0x20ee, - 0x20ef, - 0x20f0, - 0x2cef, - 0x2cf0, - 0x2cf1, - 0x2d7f, - 0x2de0, - 0x2de1, - 0x2de2, - 0x2de3, - 0x2de4, - 0x2de5, - 0x2de6, - 0x2de7, - 0x2de8, - 0x2de9, - 0x2dea, - 0x2deb, - 0x2dec, - 0x2ded, - 0x2dee, - 0x2def, - 0x2df0, - 0x2df1, - 0x2df2, - 0x2df3, - 0x2df4, - 0x2df5, - 0x2df6, - 0x2df7, - 0x2df8, - 0x2df9, - 0x2dfa, - 0x2dfb, - 0x2dfc, - 0x2dfd, - 0x2dfe, - 0x2dff, - 0x302a, - 0x302b, - 0x302c, - 0x302d, - 0x302e, - 0x302f, - 0x3099, - 0x309a, - 0xa66f, - 0xa674, - 0xa675, - 0xa676, - 0xa677, - 0xa678, - 0xa679, - 0xa67a, - 0xa67b, - 0xa67c, - 0xa67d, - 0xa69e, - 0xa69f, - 0xa6f0, - 0xa6f1, - 0xa806, - 0xa82c, - 0xa8c4, - 0xa8e0, - 0xa8e1, - 0xa8e2, - 0xa8e3, - 0xa8e4, - 0xa8e5, - 0xa8e6, - 0xa8e7, - 0xa8e8, - 0xa8e9, - 0xa8ea, - 0xa8eb, - 0xa8ec, - 0xa8ed, - 0xa8ee, - 0xa8ef, - 0xa8f0, - 0xa8f1, - 0xa92b, - 0xa92c, - 0xa92d, - 0xa953, - 0xa9b3, - 0xa9c0, - 0xaab0, - 0xaab2, - 0xaab3, - 0xaab4, - 0xaab7, - 0xaab8, - 0xaabe, - 0xaabf, - 0xaac1, - 0xaaf6, - 0xabed, - 0xfb1e, - 0xfe20, - 0xfe21, - 0xfe22, - 0xfe23, - 0xfe24, - 0xfe25, - 0xfe26, - 0xfe27, - 0xfe28, - 0xfe29, - 0xfe2a, - 0xfe2b, - 0xfe2c, - 0xfe2d, - 0xfe2e, - 0xfe2f, - 0x101fd, - 0x102e0, - 0x10376, - 0x10377, - 0x10378, - 0x10379, - 0x1037a, - 0x10a0d, - 0x10a0f, - 0x10a38, - 0x10a39, - 0x10a3a, - 0x10a3f, - 0x10ae5, - 0x10ae6, - 0x10d24, - 0x10d25, - 0x10d26, - 0x10d27, - 0x10eab, - 0x10eac, - 0x10efd, - 0x10efe, - 0x10eff, - 0x10f46, - 0x10f47, - 0x10f48, - 0x10f49, - 0x10f4a, - 0x10f4b, - 0x10f4c, - 0x10f4d, - 0x10f4e, - 0x10f4f, - 0x10f50, - 0x10f82, - 0x10f83, - 0x10f84, - 0x10f85, - 0x11046, - 0x11070, - 0x1107f, - 0x110b9, - 0x110ba, - 0x11100, - 0x11101, - 0x11102, - 0x11133, - 0x11134, - 0x11173, - 0x111c0, - 0x111ca, - 0x11235, - 0x11236, - 0x112e9, - 0x112ea, - 0x1133b, - 0x1133c, - 0x1134d, - 0x11366, - 0x11367, - 0x11368, - 0x11369, - 0x1136a, - 0x1136b, - 0x1136c, - 0x11370, - 0x11371, - 0x11372, - 0x11373, - 0x11374, - 0x11442, - 0x11446, - 0x1145e, - 0x114c2, - 0x114c3, - 0x115bf, - 0x115c0, - 0x1163f, - 0x116b6, - 0x116b7, - 0x1172b, - 0x11839, - 0x1183a, - 0x1193d, - 0x1193e, - 0x11943, - 0x119e0, - 0x11a34, - 0x11a47, - 0x11a99, - 0x11c3f, - 0x11d42, - 0x11d44, - 0x11d45, - 0x11d97, - 0x11f41, - 0x11f42, - 0x16af0, - 0x16af1, - 0x16af2, - 0x16af3, - 0x16af4, - 0x16b30, - 0x16b31, - 0x16b32, - 0x16b33, - 0x16b34, - 0x16b35, - 0x16b36, - 0x16ff0, - 0x16ff1, - 0x1bc9e, - 0x1d165, - 0x1d166, - 0x1d167, - 0x1d168, - 0x1d169, - 0x1d16d, - 0x1d16e, - 0x1d16f, - 0x1d170, - 0x1d171, - 0x1d172, - 0x1d17b, - 0x1d17c, - 0x1d17d, - 0x1d17e, - 0x1d17f, - 0x1d180, - 0x1d181, - 0x1d182, - 0x1d185, - 0x1d186, - 0x1d187, - 0x1d188, - 0x1d189, - 0x1d18a, - 0x1d18b, - 0x1d1aa, - 0x1d1ab, - 0x1d1ac, - 0x1d1ad, - 0x1d242, - 0x1d243, - 0x1d244, - 0x1e000, - 0x1e001, - 0x1e002, - 0x1e003, - 0x1e004, - 0x1e005, - 0x1e006, - 0x1e008, - 0x1e009, - 0x1e00a, - 0x1e00b, - 0x1e00c, - 0x1e00d, - 0x1e00e, - 0x1e00f, - 0x1e010, - 0x1e011, - 0x1e012, - 0x1e013, - 0x1e014, - 0x1e015, - 0x1e016, - 0x1e017, - 0x1e018, - 0x1e01b, - 0x1e01c, - 0x1e01d, - 0x1e01e, - 0x1e01f, - 0x1e020, - 0x1e021, - 0x1e023, - 0x1e024, - 0x1e026, - 0x1e027, - 0x1e028, - 0x1e029, - 0x1e02a, - 0x1e08f, - 0x1e130, - 0x1e131, - 0x1e132, - 0x1e133, - 0x1e134, - 0x1e135, - 0x1e136, - 0x1e2ae, - 0x1e2ec, - 0x1e2ed, - 0x1e2ee, - 0x1e2ef, - 0x1e4ec, - 0x1e4ed, - 0x1e4ee, - 0x1e4ef, - 0x1e8d0, - 0x1e8d1, - 0x1e8d2, - 0x1e8d3, - 0x1e8d4, - 0x1e8d5, - 0x1e8d6, - 0x1e944, - 0x1e945, - 0x1e946, - 0x1e947, - 0x1e948, - 0x1e949, - 0x1e94a, - 0x0, +#define km_noBoundaryBefore_entries 190 +static char32_t km_noBoundaryBefore[km_noBoundaryBefore_entries * 2 ] = { + 0x300, 79, // ...0x34e + 0x350, 32, // ...0x36f + 0x483, 5, // ...0x487 + 0x591, 45, // ...0x5bd + 0x5bf, 1, // ...0x5bf + 0x5c1, 2, // ...0x5c2 + 0x5c4, 2, // ...0x5c5 + 0x5c7, 1, // ...0x5c7 + 0x610, 11, // ...0x61a + 0x64b, 21, // ...0x65f + 0x670, 1, // ...0x670 + 0x6d6, 7, // ...0x6dc + 0x6df, 6, // ...0x6e4 + 0x6e7, 2, // ...0x6e8 + 0x6ea, 4, // ...0x6ed + 0x711, 1, // ...0x711 + 0x730, 27, // ...0x74a + 0x7eb, 9, // ...0x7f3 + 0x7fd, 1, // ...0x7fd + 0x816, 4, // ...0x819 + 0x81b, 9, // ...0x823 + 0x825, 3, // ...0x827 + 0x829, 5, // ...0x82d + 0x859, 3, // ...0x85b + 0x898, 8, // ...0x89f + 0x8ca, 24, // ...0x8e1 + 0x8e3, 29, // ...0x8ff + 0x93c, 1, // ...0x93c + 0x94d, 1, // ...0x94d + 0x951, 4, // ...0x954 + 0x9bc, 1, // ...0x9bc + 0x9cd, 1, // ...0x9cd + 0x9fe, 1, // ...0x9fe + 0xa3c, 1, // ...0xa3c + 0xa4d, 1, // ...0xa4d + 0xabc, 1, // ...0xabc + 0xacd, 1, // ...0xacd + 0xb3c, 1, // ...0xb3c + 0xb4d, 1, // ...0xb4d + 0xbcd, 1, // ...0xbcd + 0xc3c, 1, // ...0xc3c + 0xc4d, 1, // ...0xc4d + 0xc55, 2, // ...0xc56 + 0xcbc, 1, // ...0xcbc + 0xccd, 1, // ...0xccd + 0xd3b, 2, // ...0xd3c + 0xd4d, 1, // ...0xd4d + 0xdca, 1, // ...0xdca + 0xe38, 3, // ...0xe3a + 0xe48, 4, // ...0xe4b + 0xeb8, 3, // ...0xeba + 0xec8, 4, // ...0xecb + 0xf18, 2, // ...0xf19 + 0xf35, 1, // ...0xf35 + 0xf37, 1, // ...0xf37 + 0xf39, 1, // ...0xf39 + 0xf71, 5, // ...0xf75 + 0xf7a, 4, // ...0xf7d + 0xf80, 5, // ...0xf84 + 0xf86, 2, // ...0xf87 + 0xfc6, 1, // ...0xfc6 + 0x1037, 1, // ...0x1037 + 0x1039, 2, // ...0x103a + 0x108d, 1, // ...0x108d + 0x135d, 3, // ...0x135f + 0x1714, 2, // ...0x1715 + 0x1734, 1, // ...0x1734 + 0x17d2, 1, // ...0x17d2 + 0x17dd, 1, // ...0x17dd + 0x18a9, 1, // ...0x18a9 + 0x1939, 3, // ...0x193b + 0x1a17, 2, // ...0x1a18 + 0x1a60, 1, // ...0x1a60 + 0x1a75, 8, // ...0x1a7c + 0x1a7f, 1, // ...0x1a7f + 0x1ab0, 14, // ...0x1abd + 0x1abf, 16, // ...0x1ace + 0x1b34, 1, // ...0x1b34 + 0x1b44, 1, // ...0x1b44 + 0x1b6b, 9, // ...0x1b73 + 0x1baa, 2, // ...0x1bab + 0x1be6, 1, // ...0x1be6 + 0x1bf2, 2, // ...0x1bf3 + 0x1c37, 1, // ...0x1c37 + 0x1cd0, 3, // ...0x1cd2 + 0x1cd4, 13, // ...0x1ce0 + 0x1ce2, 7, // ...0x1ce8 + 0x1ced, 1, // ...0x1ced + 0x1cf4, 1, // ...0x1cf4 + 0x1cf8, 2, // ...0x1cf9 + 0x1dc0, 64, // ...0x1dff + 0x20d0, 13, // ...0x20dc + 0x20e1, 1, // ...0x20e1 + 0x20e5, 12, // ...0x20f0 + 0x2cef, 3, // ...0x2cf1 + 0x2d7f, 1, // ...0x2d7f + 0x2de0, 32, // ...0x2dff + 0x302a, 6, // ...0x302f + 0x3099, 2, // ...0x309a + 0xa66f, 1, // ...0xa66f + 0xa674, 10, // ...0xa67d + 0xa69e, 2, // ...0xa69f + 0xa6f0, 2, // ...0xa6f1 + 0xa806, 1, // ...0xa806 + 0xa82c, 1, // ...0xa82c + 0xa8c4, 1, // ...0xa8c4 + 0xa8e0, 18, // ...0xa8f1 + 0xa92b, 3, // ...0xa92d + 0xa953, 1, // ...0xa953 + 0xa9b3, 1, // ...0xa9b3 + 0xa9c0, 1, // ...0xa9c0 + 0xaab0, 1, // ...0xaab0 + 0xaab2, 3, // ...0xaab4 + 0xaab7, 2, // ...0xaab8 + 0xaabe, 2, // ...0xaabf + 0xaac1, 1, // ...0xaac1 + 0xaaf6, 1, // ...0xaaf6 + 0xabed, 1, // ...0xabed + 0xfb1e, 1, // ...0xfb1e + 0xfe20, 16, // ...0xfe2f + 0x101fd, 1, // ...0x101fd + 0x102e0, 1, // ...0x102e0 + 0x10376, 5, // ...0x1037a + 0x10a0d, 1, // ...0x10a0d + 0x10a0f, 1, // ...0x10a0f + 0x10a38, 3, // ...0x10a3a + 0x10a3f, 1, // ...0x10a3f + 0x10ae5, 2, // ...0x10ae6 + 0x10d24, 4, // ...0x10d27 + 0x10eab, 2, // ...0x10eac + 0x10efd, 3, // ...0x10eff + 0x10f46, 11, // ...0x10f50 + 0x10f82, 4, // ...0x10f85 + 0x11046, 1, // ...0x11046 + 0x11070, 1, // ...0x11070 + 0x1107f, 1, // ...0x1107f + 0x110b9, 2, // ...0x110ba + 0x11100, 3, // ...0x11102 + 0x11133, 2, // ...0x11134 + 0x11173, 1, // ...0x11173 + 0x111c0, 1, // ...0x111c0 + 0x111ca, 1, // ...0x111ca + 0x11235, 2, // ...0x11236 + 0x112e9, 2, // ...0x112ea + 0x1133b, 2, // ...0x1133c + 0x1134d, 1, // ...0x1134d + 0x11366, 7, // ...0x1136c + 0x11370, 5, // ...0x11374 + 0x11442, 1, // ...0x11442 + 0x11446, 1, // ...0x11446 + 0x1145e, 1, // ...0x1145e + 0x114c2, 2, // ...0x114c3 + 0x115bf, 2, // ...0x115c0 + 0x1163f, 1, // ...0x1163f + 0x116b6, 2, // ...0x116b7 + 0x1172b, 1, // ...0x1172b + 0x11839, 2, // ...0x1183a + 0x1193d, 2, // ...0x1193e + 0x11943, 1, // ...0x11943 + 0x119e0, 1, // ...0x119e0 + 0x11a34, 1, // ...0x11a34 + 0x11a47, 1, // ...0x11a47 + 0x11a99, 1, // ...0x11a99 + 0x11c3f, 1, // ...0x11c3f + 0x11d42, 1, // ...0x11d42 + 0x11d44, 2, // ...0x11d45 + 0x11d97, 1, // ...0x11d97 + 0x11f41, 2, // ...0x11f42 + 0x16af0, 5, // ...0x16af4 + 0x16b30, 7, // ...0x16b36 + 0x16ff0, 2, // ...0x16ff1 + 0x1bc9e, 1, // ...0x1bc9e + 0x1d165, 5, // ...0x1d169 + 0x1d16d, 6, // ...0x1d172 + 0x1d17b, 8, // ...0x1d182 + 0x1d185, 7, // ...0x1d18b + 0x1d1aa, 4, // ...0x1d1ad + 0x1d242, 3, // ...0x1d244 + 0x1e000, 7, // ...0x1e006 + 0x1e008, 17, // ...0x1e018 + 0x1e01b, 7, // ...0x1e021 + 0x1e023, 2, // ...0x1e024 + 0x1e026, 5, // ...0x1e02a + 0x1e08f, 1, // ...0x1e08f + 0x1e130, 7, // ...0x1e136 + 0x1e2ae, 1, // ...0x1e2ae + 0x1e2ec, 4, // ...0x1e2ef + 0x1e4ec, 4, // ...0x1e4ef + 0x1e8d0, 7, // ...0x1e8d6 + 0x1e944, 7, // ...0x1e94a }; From fdb2e95d20429e0f2453a541210bca62fd9ad033 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Tue, 4 Jun 2024 12:26:00 -0500 Subject: [PATCH 07/64] feat(core): build improvements for --update-unicode option Fixes: #9467 --- core/build.sh | 1 + core/tools/meson.build | 9 +-------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/core/build.sh b/core/build.sh index 016431cfdd..ad133ac33a 100755 --- a/core/build.sh +++ b/core/build.sh @@ -179,6 +179,7 @@ if builder_start_action test:mac; then fi if builder_has_option --update-unicode; then + # TODO: only works under mac. What's the right way to get the arch here? meson compile -C ${KEYMAN_ROOT}/core/build/mac-x86_64/$BUILDER_CONFIGURATION tools/norm_data && mv -v ${KEYMAN_ROOT}/core/build/mac-x86_64/$BUILDER_CONFIGURATION/tools/nfd_table.h ${KEYMAN_ROOT}/resources/standards-data/unicode-character-database/ fi diff --git a/core/tools/meson.build b/core/tools/meson.build index 31f3a00848..5221b2a6af 100644 --- a/core/tools/meson.build +++ b/core/tools/meson.build @@ -15,18 +15,11 @@ else warns = [] endif - -if cpp_compiler.get_id() == 'emscripten' - tests_flags += ['-lnodefs.js', - '-sNO_DISABLE_EXCEPTION_CATCHING', # for test exceptions - wasm_exported_runtime_methods] -endif - norm_unicode_update = executable('norm_unicode_update', ['norm_unicode_update.cpp'], cpp_args: defns + warns, include_directories: [inc, libsrc, '../../developer/src/ext/json'], - link_args: links + tests_flags, + link_args: links, dependencies: [icu_uc, icu_i18n], # link_with: [lib], objects: lib.extract_all_objects(recursive: false), From f156a7264dd769f73f6fbc83db4fc0970f888464 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Wed, 5 Jun 2024 15:47:20 -0500 Subject: [PATCH 08/64] feat(core): generator in core/src for util_normalize_table.h - temporary header file generated by wasm during build - built using icu - test_unicode verifies the contents and synchronization with running ICU. - this is used by util_normalize to provide normalization properties under wasm without needing to include ICU. Fixes: #9467 --- core/build.sh | 7 - core/meson.build | 1 - core/src/meson.build | 24 +++ core/src/util_normalize.cpp | 2 +- core/src/util_normalize_table_generator.cpp | 108 ++++++++++ core/tests/unit/ldml/meson.build | 2 +- core/tests/unit/ldml/test_unicode.cpp | 16 +- core/tools/meson.build | 33 --- core/tools/norm_unicode_update.cpp | 84 -------- .../unicode-character-database/nfd_table.h | 197 ------------------ 10 files changed, 145 insertions(+), 329 deletions(-) create mode 100644 core/src/util_normalize_table_generator.cpp delete mode 100644 core/tools/meson.build delete mode 100644 core/tools/norm_unicode_update.cpp delete mode 100644 resources/standards-data/unicode-character-database/nfd_table.h diff --git a/core/build.sh b/core/build.sh index ad133ac33a..168cc1cbcf 100755 --- a/core/build.sh +++ b/core/build.sh @@ -68,7 +68,6 @@ Libraries will be built in 'build///src'. "uninstall uninstall libraries from current system" \ "${archtargets[@]}" \ "--no-tests do not configure tests (used by other projects)" \ - "--update-unicode rebuild tables if the ICU4C Unicode version changes" \ "--test,-t=opt_tests test[s] to run (space separated)" builder_parse "$@" @@ -178,12 +177,6 @@ if builder_start_action test:mac; then builder_finish_action success test:mac fi -if builder_has_option --update-unicode; then - # TODO: only works under mac. What's the right way to get the arch here? - meson compile -C ${KEYMAN_ROOT}/core/build/mac-x86_64/$BUILDER_CONFIGURATION tools/norm_data && mv -v ${KEYMAN_ROOT}/core/build/mac-x86_64/$BUILDER_CONFIGURATION/tools/nfd_table.h ${KEYMAN_ROOT}/resources/standards-data/unicode-character-database/ -fi - - # ------------------------------------------------------------------------------- do_action install diff --git a/core/meson.build b/core/meson.build index ff306991e9..0049c95741 100644 --- a/core/meson.build +++ b/core/meson.build @@ -44,4 +44,3 @@ subdir('doc') subdir('include') subdir('src') subdir('tests') -subdir('tools') diff --git a/core/src/meson.build b/core/src/meson.build index 5632c22257..a4160842eb 100644 --- a/core/src/meson.build +++ b/core/src/meson.build @@ -40,6 +40,29 @@ if icu_uc.found() defns += '-DHAVE_ICU4C' endif +# On wasm, generate util_normalize_table.h automatically from ICU +generated_headers = [] + +if cpp_compiler.get_id() == 'emscripten' + +util_normalize_table_generator = executable('util_normalize_table_generator', + ['util_normalize_table_generator.cpp'], + cpp_args: defns + warns, + include_directories: [inc], + link_args: links, + dependencies: [icu_uc, icu_i18n], + ) + +util_normalize_table_h = custom_target('util_normalize_table.h', + output: 'util_normalize_table.h', + command: [util_normalize_table_generator], + capture:true) + +generated_headers += util_normalize_table_h + + +endif + kmx_files = files( 'actions_normalize.cpp', @@ -111,6 +134,7 @@ lib = library('keymancore', kmx_files, mock_files, version_res, + generated_headers, cpp_args: defns + warns + flags, link_args: links, version: lib_version, diff --git a/core/src/util_normalize.cpp b/core/src/util_normalize.cpp index 7d0d76d8ab..947dbe0f85 100644 --- a/core/src/util_normalize.cpp +++ b/core/src/util_normalize.cpp @@ -31,7 +31,7 @@ EM_JS(char*, NormalizeNFC, (const char* input), { }); // pull in the generated table -#include "../../resources/standards-data/unicode-character-database/nfd_table.h" +#include "util_normalize_table.h" #endif diff --git a/core/src/util_normalize_table_generator.cpp b/core/src/util_normalize_table_generator.cpp new file mode 100644 index 0000000000..b5895a588b --- /dev/null +++ b/core/src/util_normalize_table_generator.cpp @@ -0,0 +1,108 @@ +/* + Copyright: © SIL International. + Description: Generator for util_normalize_table.h + Create Date: 5 Jun 2024 + Authors: Steven R. Loomis + + util_normalize_table.h is used under wasm by utilities in util_normalize.cpp to implement + normalization functions without needing ICU4C linked. + + This generator is invoked automatically by meson as part of the build. +*/ + +#include "kmx/kmx_plus.h" +#include "kmx/kmx_xstring.h" + +#define KMN_NO_ICU 0 // we will need ICU.. + +#include "core_icu.h" + +#include +#include +#include +#include + +#include + +#include + + + +int +write_nfd_table() { +#ifndef __EMSCRIPTEN__ + std::cerr << "Note: This is unusual - this generator is usually only run under emscripten!" << std::endl; +#endif + + // We write to stdout instead of to a file to avoid dealing with the filesystem under emscripten. + + std::cerr << "Writing to stdout." << std::endl; + + // write preamble + std::cout << "// GENERATED FILE: DO NOT EDIT" << std::endl; + std::cout << "//" << std::endl; + std::cout << "// util_normalize_table.h is generated by util_normalize_table_generator.cpp" << std::endl; + std::cout << "// and used by util_normalize.cpp" << std::endl; + std::cout << std::endl; + std::cout << "#pragma once" << std::endl; + std::cout << "#define KM_HASBOUNDARYBEFORE_UNICODE_VERSION \"" << U_UNICODE_VERSION << "\"" << std::endl; + std::cout << "#define KM_HASBOUNDARYBEFORE_ICU_VERSION \"" << U_ICU_VERSION << "\"" << std::endl; + std::cout << std::endl; + // we're going to need an NFD normalizer + UErrorCode status = U_ZERO_ERROR; + const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(status); + assert(U_SUCCESS(status)); + + // collect the raw list of chars that do NOT have a boundary before them. + std::vector noBoundary; + for (km_core_usv ch = 0; ch < 0x10FFFF; ch++) { + bool bb = nfd->hasBoundaryBefore(ch); + assert(!(ch == 0 && !bb)); // assert that we can use U+0000 as a terminator + if (bb) continue; //only emit nonboundary + noBoundary.push_back(ch); + } + + // now, compress these into runs + std::vector> runs; // start,len + + km_core_usv first = 0; + km_core_usv last = 0; + for(auto i = noBoundary.begin(); i <= noBoundary.end(); i++) { + if (first == 0) { + first = last = *i; + } else { + last++; + if(i == noBoundary.end() || *i != last) { + // end of a run + runs.emplace_back(first, last - first); + if (i != noBoundary.end()) { + // setup for next + first = last = *i; + } + } + } + } + + // finally, write out metadata and the runs themselves. + std::cout << "#define km_noBoundaryBefore_entries " << runs.size() << "\n"; + + std::cout << "static char32_t km_noBoundaryBefore[km_noBoundaryBefore_entries * 2 ] = {" << std::endl; + + std::cout << "/* start codepoint, count (inclusive), ...range end */" << std::endl; + + for (auto i = runs.begin(); i < runs.end(); i++) { + std::cout << "\t0x" << std::hex << i->first << std::dec << ",\t " << i->second << ", // ...0x" << std::hex << (i->first+i->second-1) << std::endl; + } + + // termination + std::cout << "};" << std::endl; + std::cout << "// end" << std::endl; + std::cerr << "Wrote " << runs.size() << " runs representing " << noBoundary.size() << " entries." << std::endl; + return 0; +} + +int +main(int /*argc*/, const char * /*argv*/[]) { + write_nfd_table(); + return 0; +} diff --git a/core/tests/unit/ldml/meson.build b/core/tests/unit/ldml/meson.build index da60757750..42f2d7778e 100644 --- a/core/tests/unit/ldml/meson.build +++ b/core/tests/unit/ldml/meson.build @@ -132,7 +132,7 @@ test('test_context_normalization', test_context_normalization, suite: 'ldml') # Build and run additional test_unicode test test_unicode = executable('test_unicode', 'test_unicode.cpp', - ['test_unicode.cpp', common_test_files], + ['test_unicode.cpp', common_test_files, generated_headers], cpp_args: defns + warns, include_directories: [inc, libsrc, '../../../../developer/src/ext/json'], link_args: links + tests_flags, diff --git a/core/tests/unit/ldml/test_unicode.cpp b/core/tests/unit/ldml/test_unicode.cpp index 50844d635b..b36f1a3e38 100644 --- a/core/tests/unit/ldml/test_unicode.cpp +++ b/core/tests/unit/ldml/test_unicode.cpp @@ -47,7 +47,7 @@ #ifdef __EMSCRIPTEN__ // Pull this in to verify versions -#include "../../../../resources/standards-data/unicode-character-database/nfd_table.h" +#include "util_normalize_table.h" #endif //------------------------------------------------------------------------------------- @@ -189,9 +189,15 @@ inline const char *boolstr(bool b) { void test_has_boundary_before() { std::cout << "= " << __FUNCTION__ << std::endl; - std::cout << "(this test only runs under emscripten. congratulations.)" << std::endl; - // static_assert(U_UNICODE_VERSION == KM_HASBOUNDARYBEFORE_UNICODE_VERSION, "nfd_table.h Unicode version does not match ICU's - see nfd_table.h"); - std::cout << U_UNICODE_VERSION << "≈≈" << KM_HASBOUNDARYBEFORE_UNICODE_VERSION << std::endl; + std::cout << "I see we are on Emscripten / wasm! Now we will do some additional tests." << std::endl; + std::string icu4c_unicode(U_UNICODE_VERSION), header_unicode(KM_HASBOUNDARYBEFORE_UNICODE_VERSION), + icu4c_icu(U_ICU_VERSION), header_icu(KM_HASBOUNDARYBEFORE_ICU_VERSION); + std::cout << "Unicode: " << U_UNICODE_VERSION << ", and from the table file: " << KM_HASBOUNDARYBEFORE_UNICODE_VERSION << std::endl; + std::cout << "It would be very strange for these versions to be out of sync. Some sort of build or tool problem." << std::endl; + assert_basic_equal(icu4c_unicode, header_unicode); + assert_basic_equal(icu4c_icu, header_icu); + + std::cout << std::endl << "Now, let's make sure has_nfd_boundary_before() matches ICU." << std::endl; UErrorCode status = U_ZERO_ERROR; const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(status); @@ -203,7 +209,7 @@ void test_has_boundary_before() { auto icu_hbb = nfd->hasBoundaryBefore(cp); if (km_hbb != icu_hbb) { - std::cerr << "Error: nfd_table.h said " << boolstr(km_hbb) << " but ICU said " << boolstr(icu_hbb) << " for " + std::cerr << "Error: util_normalize_table.h said " << boolstr(km_hbb) << " but ICU said " << boolstr(icu_hbb) << " for " << "has_nfd_boundary_before(0x" << std::hex << cp << std::dec << ")" << std::endl; } assert(km_hbb == icu_hbb); diff --git a/core/tools/meson.build b/core/tools/meson.build deleted file mode 100644 index 5221b2a6af..0000000000 --- a/core/tools/meson.build +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright: © 2024 SIL International. -# Description: Cross platform build script to compile tool(s). -# Create Date: 31 May 2024 -# Authors: Steven R. Loomis (SRL) -# - - -# TODO -- why are these differing from the standard.meson.build flags? -if cpp_compiler.get_id() == 'gcc' or cpp_compiler.get_id() == 'clang' or cpp_compiler.get_id() == 'emscripten' - warns = [ - '-Wno-missing-field-initializers', - '-Wno-unused-parameter' - ] -else - warns = [] -endif - -norm_unicode_update = executable('norm_unicode_update', - ['norm_unicode_update.cpp'], - cpp_args: defns + warns, - include_directories: [inc, libsrc, '../../developer/src/ext/json'], - link_args: links, - dependencies: [icu_uc, icu_i18n], - # link_with: [lib], - objects: lib.extract_all_objects(recursive: false), - ) - - -# ../../resources/standards-data/unicode-character-database/ -norm_data = custom_target('norm_data', output: 'nfd_table.h', command: [norm_unicode_update, '@OUTPUT@']) - - -# TODO: execute it diff --git a/core/tools/norm_unicode_update.cpp b/core/tools/norm_unicode_update.cpp deleted file mode 100644 index 408dbb0c98..0000000000 --- a/core/tools/norm_unicode_update.cpp +++ /dev/null @@ -1,84 +0,0 @@ -#include "kmx/kmx_plus.h" -#include "kmx/kmx_xstring.h" -#include "core_icu.h" - -#include -#include -#include -#include - -#include - -#include - -#ifndef __EMSCRIPTEN__ - -int -write_nfd_table(const char *NFD_FILE) { - std::cout << " writing: " << NFD_FILE << std::endl; - auto f = std::ofstream(NFD_FILE); - assert(f.good()); - - // write preamble - f << "//NFD hasBoundaryBefore" << std::endl; - f << "#pragma once" << std::endl; - f << "#define KM_HASBOUNDARYBEFORE_UNICODE_VERSION \"" << U_UNICODE_VERSION << "\"" << std::endl; - f << "#define KM_HASBOUNDARYBEFORE_ICU_VERSION \"" << U_ICU_VERSION << "\"" << std::endl; - // we're going to need an NFD normalizer - UErrorCode status = U_ZERO_ERROR; - const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(status); - assert(U_SUCCESS(status)); - - std::vector noBoundary; - - for (km_core_usv ch = 0; ch < 0x10FFFF; ch++) { - bool bb = nfd->hasBoundaryBefore(ch); - assert(!(ch == 0 && !bb)); // assert that we can use U+0000 as a terminator - if (bb) continue; //only emit nonboundary - noBoundary.push_back(ch); - } - - std::vector> runs; // start,len - - km_core_usv first = 0; - km_core_usv last = 0; - for(auto i = noBoundary.begin(); i <= noBoundary.end(); i++) { - if (first == 0) { - first = last = *i; - } else { - last++; - if(i == noBoundary.end() || *i != last) { - // end of a run - runs.emplace_back(first, last - first); - if (i != noBoundary.end()) { - // setup for next - first = last = *i; - } - } - } - } - f << "#define km_noBoundaryBefore_entries " << runs.size() << "\n"; - - f << "static char32_t km_noBoundaryBefore[km_noBoundaryBefore_entries * 2 ] = {" << std::endl; - - for (auto i = runs.begin(); i < runs.end(); i++) { - f << "\t0x" << std::hex << i->first << std::dec << ",\t " << i->second << ", // ...0x" << std::hex << (i->first+i->second-1) << std::endl; - } - - // termination - f << "};" << std::endl; - return 0; -} - -int -main(int argc, const char *argv[]) { - assert(argc == 2); // call with one param: @OUTPUT@ - write_nfd_table(argv[1]); - return 0; -} -#else -int main(int argc, const char *argv[]) { - std::cerr << "Can't run this under Emscripten - run under another platform." << std::endl; - return 1; -} -#endif diff --git a/resources/standards-data/unicode-character-database/nfd_table.h b/resources/standards-data/unicode-character-database/nfd_table.h deleted file mode 100644 index 068240fc30..0000000000 --- a/resources/standards-data/unicode-character-database/nfd_table.h +++ /dev/null @@ -1,197 +0,0 @@ -//NFD hasBoundaryBefore -#pragma once -#define KM_HASBOUNDARYBEFORE_UNICODE_VERSION "15.0" -#define KM_HASBOUNDARYBEFORE_ICU_VERSION "73.1" -#define km_noBoundaryBefore_entries 190 -static char32_t km_noBoundaryBefore[km_noBoundaryBefore_entries * 2 ] = { - 0x300, 79, // ...0x34e - 0x350, 32, // ...0x36f - 0x483, 5, // ...0x487 - 0x591, 45, // ...0x5bd - 0x5bf, 1, // ...0x5bf - 0x5c1, 2, // ...0x5c2 - 0x5c4, 2, // ...0x5c5 - 0x5c7, 1, // ...0x5c7 - 0x610, 11, // ...0x61a - 0x64b, 21, // ...0x65f - 0x670, 1, // ...0x670 - 0x6d6, 7, // ...0x6dc - 0x6df, 6, // ...0x6e4 - 0x6e7, 2, // ...0x6e8 - 0x6ea, 4, // ...0x6ed - 0x711, 1, // ...0x711 - 0x730, 27, // ...0x74a - 0x7eb, 9, // ...0x7f3 - 0x7fd, 1, // ...0x7fd - 0x816, 4, // ...0x819 - 0x81b, 9, // ...0x823 - 0x825, 3, // ...0x827 - 0x829, 5, // ...0x82d - 0x859, 3, // ...0x85b - 0x898, 8, // ...0x89f - 0x8ca, 24, // ...0x8e1 - 0x8e3, 29, // ...0x8ff - 0x93c, 1, // ...0x93c - 0x94d, 1, // ...0x94d - 0x951, 4, // ...0x954 - 0x9bc, 1, // ...0x9bc - 0x9cd, 1, // ...0x9cd - 0x9fe, 1, // ...0x9fe - 0xa3c, 1, // ...0xa3c - 0xa4d, 1, // ...0xa4d - 0xabc, 1, // ...0xabc - 0xacd, 1, // ...0xacd - 0xb3c, 1, // ...0xb3c - 0xb4d, 1, // ...0xb4d - 0xbcd, 1, // ...0xbcd - 0xc3c, 1, // ...0xc3c - 0xc4d, 1, // ...0xc4d - 0xc55, 2, // ...0xc56 - 0xcbc, 1, // ...0xcbc - 0xccd, 1, // ...0xccd - 0xd3b, 2, // ...0xd3c - 0xd4d, 1, // ...0xd4d - 0xdca, 1, // ...0xdca - 0xe38, 3, // ...0xe3a - 0xe48, 4, // ...0xe4b - 0xeb8, 3, // ...0xeba - 0xec8, 4, // ...0xecb - 0xf18, 2, // ...0xf19 - 0xf35, 1, // ...0xf35 - 0xf37, 1, // ...0xf37 - 0xf39, 1, // ...0xf39 - 0xf71, 5, // ...0xf75 - 0xf7a, 4, // ...0xf7d - 0xf80, 5, // ...0xf84 - 0xf86, 2, // ...0xf87 - 0xfc6, 1, // ...0xfc6 - 0x1037, 1, // ...0x1037 - 0x1039, 2, // ...0x103a - 0x108d, 1, // ...0x108d - 0x135d, 3, // ...0x135f - 0x1714, 2, // ...0x1715 - 0x1734, 1, // ...0x1734 - 0x17d2, 1, // ...0x17d2 - 0x17dd, 1, // ...0x17dd - 0x18a9, 1, // ...0x18a9 - 0x1939, 3, // ...0x193b - 0x1a17, 2, // ...0x1a18 - 0x1a60, 1, // ...0x1a60 - 0x1a75, 8, // ...0x1a7c - 0x1a7f, 1, // ...0x1a7f - 0x1ab0, 14, // ...0x1abd - 0x1abf, 16, // ...0x1ace - 0x1b34, 1, // ...0x1b34 - 0x1b44, 1, // ...0x1b44 - 0x1b6b, 9, // ...0x1b73 - 0x1baa, 2, // ...0x1bab - 0x1be6, 1, // ...0x1be6 - 0x1bf2, 2, // ...0x1bf3 - 0x1c37, 1, // ...0x1c37 - 0x1cd0, 3, // ...0x1cd2 - 0x1cd4, 13, // ...0x1ce0 - 0x1ce2, 7, // ...0x1ce8 - 0x1ced, 1, // ...0x1ced - 0x1cf4, 1, // ...0x1cf4 - 0x1cf8, 2, // ...0x1cf9 - 0x1dc0, 64, // ...0x1dff - 0x20d0, 13, // ...0x20dc - 0x20e1, 1, // ...0x20e1 - 0x20e5, 12, // ...0x20f0 - 0x2cef, 3, // ...0x2cf1 - 0x2d7f, 1, // ...0x2d7f - 0x2de0, 32, // ...0x2dff - 0x302a, 6, // ...0x302f - 0x3099, 2, // ...0x309a - 0xa66f, 1, // ...0xa66f - 0xa674, 10, // ...0xa67d - 0xa69e, 2, // ...0xa69f - 0xa6f0, 2, // ...0xa6f1 - 0xa806, 1, // ...0xa806 - 0xa82c, 1, // ...0xa82c - 0xa8c4, 1, // ...0xa8c4 - 0xa8e0, 18, // ...0xa8f1 - 0xa92b, 3, // ...0xa92d - 0xa953, 1, // ...0xa953 - 0xa9b3, 1, // ...0xa9b3 - 0xa9c0, 1, // ...0xa9c0 - 0xaab0, 1, // ...0xaab0 - 0xaab2, 3, // ...0xaab4 - 0xaab7, 2, // ...0xaab8 - 0xaabe, 2, // ...0xaabf - 0xaac1, 1, // ...0xaac1 - 0xaaf6, 1, // ...0xaaf6 - 0xabed, 1, // ...0xabed - 0xfb1e, 1, // ...0xfb1e - 0xfe20, 16, // ...0xfe2f - 0x101fd, 1, // ...0x101fd - 0x102e0, 1, // ...0x102e0 - 0x10376, 5, // ...0x1037a - 0x10a0d, 1, // ...0x10a0d - 0x10a0f, 1, // ...0x10a0f - 0x10a38, 3, // ...0x10a3a - 0x10a3f, 1, // ...0x10a3f - 0x10ae5, 2, // ...0x10ae6 - 0x10d24, 4, // ...0x10d27 - 0x10eab, 2, // ...0x10eac - 0x10efd, 3, // ...0x10eff - 0x10f46, 11, // ...0x10f50 - 0x10f82, 4, // ...0x10f85 - 0x11046, 1, // ...0x11046 - 0x11070, 1, // ...0x11070 - 0x1107f, 1, // ...0x1107f - 0x110b9, 2, // ...0x110ba - 0x11100, 3, // ...0x11102 - 0x11133, 2, // ...0x11134 - 0x11173, 1, // ...0x11173 - 0x111c0, 1, // ...0x111c0 - 0x111ca, 1, // ...0x111ca - 0x11235, 2, // ...0x11236 - 0x112e9, 2, // ...0x112ea - 0x1133b, 2, // ...0x1133c - 0x1134d, 1, // ...0x1134d - 0x11366, 7, // ...0x1136c - 0x11370, 5, // ...0x11374 - 0x11442, 1, // ...0x11442 - 0x11446, 1, // ...0x11446 - 0x1145e, 1, // ...0x1145e - 0x114c2, 2, // ...0x114c3 - 0x115bf, 2, // ...0x115c0 - 0x1163f, 1, // ...0x1163f - 0x116b6, 2, // ...0x116b7 - 0x1172b, 1, // ...0x1172b - 0x11839, 2, // ...0x1183a - 0x1193d, 2, // ...0x1193e - 0x11943, 1, // ...0x11943 - 0x119e0, 1, // ...0x119e0 - 0x11a34, 1, // ...0x11a34 - 0x11a47, 1, // ...0x11a47 - 0x11a99, 1, // ...0x11a99 - 0x11c3f, 1, // ...0x11c3f - 0x11d42, 1, // ...0x11d42 - 0x11d44, 2, // ...0x11d45 - 0x11d97, 1, // ...0x11d97 - 0x11f41, 2, // ...0x11f42 - 0x16af0, 5, // ...0x16af4 - 0x16b30, 7, // ...0x16b36 - 0x16ff0, 2, // ...0x16ff1 - 0x1bc9e, 1, // ...0x1bc9e - 0x1d165, 5, // ...0x1d169 - 0x1d16d, 6, // ...0x1d172 - 0x1d17b, 8, // ...0x1d182 - 0x1d185, 7, // ...0x1d18b - 0x1d1aa, 4, // ...0x1d1ad - 0x1d242, 3, // ...0x1d244 - 0x1e000, 7, // ...0x1e006 - 0x1e008, 17, // ...0x1e018 - 0x1e01b, 7, // ...0x1e021 - 0x1e023, 2, // ...0x1e024 - 0x1e026, 5, // ...0x1e02a - 0x1e08f, 1, // ...0x1e08f - 0x1e130, 7, // ...0x1e136 - 0x1e2ae, 1, // ...0x1e2ae - 0x1e2ec, 4, // ...0x1e2ef - 0x1e4ec, 4, // ...0x1e4ef - 0x1e8d0, 7, // ...0x1e8d6 - 0x1e944, 7, // ...0x1e94a -}; From 55a025c767262b45ed122850e6b4add2295f0832 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Wed, 5 Jun 2024 16:53:43 -0500 Subject: [PATCH 09/64] refactor(core): devolve regex to js for wasm - new module, core/src/util_regex.hpp - no wasm implementationyet Fixes: #9467 --- core/src/core_icu.h | 4 + core/src/ldml/ldml_transforms.cpp | 176 ++---------------- core/src/ldml/ldml_transforms.hpp | 10 +- core/src/meson.build | 1 + core/src/util_regex.cpp | 223 +++++++++++++++++++++++ core/src/util_regex.hpp | 48 +++++ core/tests/unit/ldml/test_transforms.cpp | 11 +- 7 files changed, 299 insertions(+), 174 deletions(-) create mode 100644 core/src/util_regex.cpp create mode 100644 core/src/util_regex.hpp diff --git a/core/src/core_icu.h b/core/src/core_icu.h index edfc209802..20b13b0456 100644 --- a/core/src/core_icu.h +++ b/core/src/core_icu.h @@ -33,6 +33,10 @@ #include "unicode/utypes.h" #include "unicode/unistr.h" #include "unicode/normalizer2.h" +#include "unicode/uniset.h" +#include "unicode/usetiter.h" +#include "unicode/regex.h" +#include "unicode/utext.h" #include "keyman_core.h" #include "debuglog.h" diff --git a/core/src/ldml/ldml_transforms.cpp b/core/src/ldml/ldml_transforms.cpp index 42953c604e..8e37011bda 100644 --- a/core/src/ldml/ldml_transforms.cpp +++ b/core/src/ldml/ldml_transforms.cpp @@ -1,5 +1,3 @@ -// TEMP -#define KMN_NO_ICU 0 /* Copyright: © SIL International. Description: This is an implementation of the LDML keyboard spec 3.0. @@ -439,20 +437,14 @@ reorder_group::apply(std::u32string &str) const { } transform_entry::transform_entry(const transform_entry &other) - : fFrom(other.fFrom), fTo(other.fTo), fFromPattern(nullptr), fMapFromStrId(other.fMapFromStrId), + : fFrom(other.fFrom), fTo(other.fTo), fFromPattern(other.fFromPattern), fMapFromStrId(other.fMapFromStrId), fMapToStrId(other.fMapToStrId), fMapFromList(other.fMapFromList), fMapToList(other.fMapToList), normalization_disabled(other.normalization_disabled) { - if (other.fFromPattern) { - // clone pattern - fFromPattern.reset(other.fFromPattern->clone()); - } } transform_entry::transform_entry(const std::u32string &from, const std::u32string &to) - : fFrom(from), fTo(to), fFromPattern(nullptr), fMapFromStrId(), fMapToStrId(), fMapFromList(), fMapToList(), normalization_disabled(false) { + : fFrom(from), fTo(to), fFromPattern(from), fMapFromStrId(), fMapToStrId(), fMapFromList(), fMapToList(), normalization_disabled(false) { assert(!fFrom.empty()); - - init(); } transform_entry::transform_entry( @@ -463,7 +455,7 @@ transform_entry::transform_entry( const kmx::kmx_plus &kplus, bool &valid, bool norm_disabled) - : fFrom(from), fTo(to), fFromPattern(nullptr), fMapFromStrId(mapFrom), fMapToStrId(mapTo), normalization_disabled(norm_disabled) { + : fFrom(from), fTo(to), fFromPattern(), fMapFromStrId(mapFrom), fMapToStrId(mapTo), normalization_disabled(norm_disabled) { if (!valid) return; // exit early assert(!fFrom.empty()); // TODO-LDML: should not happen? @@ -471,7 +463,12 @@ transform_entry::transform_entry( assert(kplus.strs != nullptr); assert(kplus.vars != nullptr); assert(kplus.elem != nullptr); - if(!init()) { + std::u32string from2 = fFrom; + if (!normalization_disabled) { + // normalize, including markers, for regex + normalize_nfd_markers(from2, regex_sentinel); + } + if (!fFromPattern.init(from2)) { valid = false; } @@ -508,160 +505,17 @@ transform_entry::transform_entry( } } -bool -transform_entry::init() { - if (fFrom.empty()) { - return false; - } - // TODO-LDML: if we have mapFrom, may need to do other processing. - std::u32string from2 = fFrom; - if (!normalization_disabled) { - // normalize, including markers, for regex - normalize_nfd_markers(from2, regex_sentinel); - } - std::u16string patstr = km::core::kmx::u32string_to_u16string(from2); - UErrorCode status = U_ZERO_ERROR; - /* const */ icu::UnicodeString patustr = icu::UnicodeString(patstr.data(), (int32_t)patstr.length()); - // add '$' to match to end - patustr.append(u'$'); // TODO-LDML: may need to escape some markers. Marker #91 will look like a `[` to the pattern - fFromPattern.reset(icu::RegexPattern::compile(patustr, 0, status)); - return (UASSERT_SUCCESS(status)); -} - size_t transform_entry::apply(const std::u32string &input, std::u32string &output) const { - assert(fFromPattern); - // TODO-LDML: Really? can't go from u32 to UnicodeString? - // TODO-LDML: Also, we could cache the u16 string at the transformGroup level or higher. - UErrorCode status = U_ZERO_ERROR; - const std::u16string matchstr = km::core::kmx::u32string_to_u16string(input); - icu::UnicodeString matchustr = icu::UnicodeString(matchstr.data(), (int32_t)matchstr.length()); - // TODO-LDML: create a new Matcher every time. These could be cached and reset. - std::unique_ptr matcher(fFromPattern->matcher(matchustr, status)); - if (!UASSERT_SUCCESS(status)) { - return 0; // TODO-LDML: return error - } - - if (!matcher->find(status)) { // i.e. matches somewhere, in this case at end of str - return 0; // no match - } - - // TODO-LDML: this is UTF-16 len, not UTF-32 len!! - // TODO-LDML: if we had an underlying UText this would be simpler. - int32_t matchStart = matcher->start(status); - int32_t matchEnd = matcher->end(status); - if (!UASSERT_SUCCESS(status)) { - return 0; // TODO-LDML: return error - } - // extract.. - const icu::UnicodeString substr = matchustr.tempSubStringBetween(matchStart, matchEnd); - // preflight to UTF-32 to get length - UErrorCode substrStatus = U_ZERO_ERROR; // throwaway status - // we need the UTF-32 matchLen for our return. - auto matchLen = substr.toUTF32(nullptr, 0, substrStatus); - - // should have matched something. - assert(matchLen > 0); - - // now, do the replace. - - /** this is the 'to' or other replacement string.*/ - icu::UnicodeString rustr; - if (fMapFromStrId == 0) { - // Normal case: not a map. - // This replace will apply $1, $2 etc. - // Convert the fTo into u16 TODO-LDML (we could cache this?) - const std::u16string rstr = km::core::kmx::u32string_to_u16string(fTo); - rustr = icu::UnicodeString(rstr.data(), (int32_t)rstr.length()); - } else { - // Set map case: mapping from/to - - // we actually need the group(1) string here. - // this is only the content in parenthesis () - icu::UnicodeString group1 = matcher->group(1, status); - if (!UASSERT_SUCCESS(status)) { - // TODO-LDML: could be a malformed from pattern - return 0; // TODO-LDML: return error - } - // now, how long is group1 in UTF-32, hmm? - UErrorCode preflightStatus = U_ZERO_ERROR; // throwaway status - auto group1Len = group1.toUTF32(nullptr, 0, preflightStatus); - char32_t *s = new char32_t[group1Len + 1]; - assert(s != nullptr); // TODO-LDML: OOM - // convert - group1.toUTF32((UChar32 *)s, group1Len + 1, status); - if (!UASSERT_SUCCESS(status)) { - return 0; // TODO-LDML: memory issue - } - std::u32string match32(s, group1Len); // taken from just group1 - // clean up buffer - delete [] s; - - // Now we're ready to do the actual mapping. - - // 1., we need to find the index in the source set. - auto matchIndex = findIndexFrom(match32); - assert(matchIndex != -1L); // TODO-LDML: not matching shouldn't happen, the regex wouldn't have matched. - // we already asserted on load that the from and to sets have the same cardinality. - - // 2. get the target string, convert to utf-16 - // we use the same matchIndex that was just found - const std::u16string rstr = km::core::kmx::u32string_to_u16string(fMapToList.at(matchIndex)); - - // 3. update the UnicodeString for replacement - rustr = icu::UnicodeString(rstr.data(), (int32_t)rstr.length()); - // and we return to the regular code flow. - } - // here we replace the match output. No normalization, yet. - icu::UnicodeString entireOutput = matcher->replaceFirst(rustr, status); - if (!UASSERT_SUCCESS(status)) { - // TODO-LDML: could fail here due to bad input (syntax err) - return 0; - } - // entireOutput includes all of 'input', but modified. Need to substring it. - icu::UnicodeString outu = entireOutput.tempSubString(matchStart); - - // Special case if there's no output, save some allocs - if (outu.length() == 0) { - output.clear(); - } else { - // TODO-LDML: All we are trying to do is to extract the output string. Probably too many steps. - UErrorCode preflightStatus = U_ZERO_ERROR; - // calculate how big the buffer is - auto out32len = outu.toUTF32(nullptr, 0, preflightStatus); // preflightStatus will be an err, because we know the buffer overruns zero bytes - // allocate - std::unique_ptr s(new char32_t[out32len + 1]); - assert(s); - if (!s) { - return 0; // TODO-LDML: allocation failed - } - // convert - outu.toUTF32((UChar32 *)(s.get()), out32len + 1, status); - if (!UASSERT_SUCCESS(status)) { - return 0; // TODO-LDML: memory issue - } - output.assign(s.get(), out32len); - // NOW do a marker-safe normalize - if (!normalization_disabled && !normalize_nfd_markers(output)) { + auto result = fFromPattern.apply(input, output, fTo, fMapFromList, fMapToList); + // NOW do a marker-safe normalize + if (result != 0 && !output.empty() && !normalization_disabled) { + if (!normalize_nfd_markers(output)) { DebugLog("normalize_nfd_markers(output) failed"); - return 0; // TODO-LDML: normalization failed. + return 0; // TODO-LDML: normalization failed. } } - return matchLen; -} - -int32_t transform_entry::findIndexFrom(const std::u32string &match) const { - return findIndex(match, fMapFromList); -} - -int32_t transform_entry::findIndex(const std::u32string &match, const std::deque list) { - int32_t index = 0; - for(auto e = list.begin(); e < list.end(); e++, index++) { - if (match == *e) { - return index; - } - } - return -1; // not found + return result; } any_group::any_group(const transform_group &g) : type(any_group_type::transform), transform(g), reorder() { diff --git a/core/src/ldml/ldml_transforms.hpp b/core/src/ldml/ldml_transforms.hpp index 1b56a76246..90dd5f5e43 100644 --- a/core/src/ldml/ldml_transforms.hpp +++ b/core/src/ldml/ldml_transforms.hpp @@ -16,11 +16,7 @@ #include #include "debuglog.h" -#include "core_icu.h" -#include "unicode/uniset.h" -#include "unicode/usetiter.h" -#include "unicode/regex.h" -#include "unicode/utext.h" +#include "util_regex.hpp" namespace km { namespace core { @@ -111,14 +107,12 @@ public: private: const std::u32string fFrom; const std::u32string fTo; - std::unique_ptr fFromPattern; + km::core::util::km_regex fFromPattern; const KMX_DWORD fMapFromStrId; const KMX_DWORD fMapToStrId; std::deque fMapFromList; std::deque fMapToList; - /** Internal function to setup pattern string @returns true on success */ - bool init(); bool normalization_disabled; /** @returns the index of the item in the fMapFromList list, or -1 */ int32_t findIndexFrom(const std::u32string &match) const; diff --git a/core/src/meson.build b/core/src/meson.build index a4160842eb..b7b1ae78f5 100644 --- a/core/src/meson.build +++ b/core/src/meson.build @@ -83,6 +83,7 @@ kmx_files = files( 'km_core_processevent_api.cpp', 'jsonpp.cpp', 'util_normalize.cpp', + 'util_regex.cpp', 'core_icu.cpp', 'ldml/ldml_processor.cpp', 'ldml/ldml_transforms.cpp', diff --git a/core/src/util_regex.cpp b/core/src/util_regex.cpp new file mode 100644 index 0000000000..f6c0801997 --- /dev/null +++ b/core/src/util_regex.cpp @@ -0,0 +1,223 @@ +/* + Copyright: © SIL International. + Description: Core Regex Utilities - abstract out ICU dependencies + Create Date: 5 Jun 2024 + Authors: Steven R. Loomis +*/ + +#include "util_regex.hpp" + +#include "core_icu.h" +#include "kmx/kmx_xstring.h" + + +namespace km { +namespace core { +namespace util { + +/** find the */ +int32_t km_regex::findIndex(const std::u32string &match, const std::deque &list) { + int32_t index = 0; + for(auto e = list.begin(); e < list.end(); e++, index++) { + if (match == *e) { + return index; + } + } + return -1; // not found +} + +km_regex::km_regex() +#if KMN_NO_ICU +#else + : fPattern(nullptr) +#endif +{ + +} + + +km_regex::km_regex(const km_regex& other) +#if KMN_NO_ICU +#else + : fPattern(nullptr) +#endif +{ +#if KMN_NO_ICU + +#else + if (other.fPattern) { + // clone pattern + fPattern.reset(other.fPattern->clone()); + } +#endif +} + +km_regex::km_regex(const std::u32string &pattern) +#if KMN_NO_ICU +#else + : fPattern(nullptr) +#endif +{ + init(pattern); +} + +km_regex::~km_regex() { + +} + +bool km_regex::valid() const { +#if KMN_NO_ICU +#error todo +#else + // valid if fPattern is present. + return !!fPattern; +#endif +} + +bool km_regex::init(const std::u32string &pattern) { +#if KMN_NO_ICU +#error todo +#else + if (pattern.empty()) { + return false; + } + // TODO-LDML: if we have mapFrom, may need to do other processing. + std::u16string patstr = km::core::kmx::u32string_to_u16string(pattern); + UErrorCode status = U_ZERO_ERROR; + /* const */ icu::UnicodeString patustr = icu::UnicodeString(patstr.data(), (int32_t)patstr.length()); + // add '$' to match to end + patustr.append(u'$'); // TODO-LDML: may need to escape some markers. Marker #91 will look like a `[` to the pattern + fPattern.reset(icu::RegexPattern::compile(patustr, 0, status)); + return (UASSERT_SUCCESS(status)); +#endif +} + +size_t km_regex::apply(const std::u32string &input, std::u32string &output, + const std::u32string &to, + const std::deque &fromList, + const std::deque &toList ) const { +#if KMN_NO_ICU +#error TODO +#else + assert(fPattern); + // TODO-LDML: Really? can't go from u32 to UnicodeString? + // TODO-LDML: Also, we could cache the u16 string at the transformGroup level or higher. + UErrorCode status = U_ZERO_ERROR; + const std::u16string matchstr = km::core::kmx::u32string_to_u16string(input); + icu::UnicodeString matchustr = icu::UnicodeString(matchstr.data(), (int32_t)matchstr.length()); + // TODO-LDML: create a new Matcher every time. These could be cached and reset. + std::unique_ptr matcher(fPattern->matcher(matchustr, status)); + if (!UASSERT_SUCCESS(status)) { + return 0; // TODO-LDML: return error + } + + if (!matcher->find(status)) { // i.e. matches somewhere, in this case at end of str + return 0; // no match + } + + // TODO-LDML: this is UTF-16 len, not UTF-32 len!! + // TODO-LDML: if we had an underlying UText this would be simpler. + int32_t matchStart = matcher->start(status); + int32_t matchEnd = matcher->end(status); + if (!UASSERT_SUCCESS(status)) { + return 0; // TODO-LDML: return error + } + // extract.. + const icu::UnicodeString substr = matchustr.tempSubStringBetween(matchStart, matchEnd); + // preflight to UTF-32 to get length + UErrorCode substrStatus = U_ZERO_ERROR; // throwaway status + // we need the UTF-32 matchLen for our return. + auto matchLen = substr.toUTF32(nullptr, 0, substrStatus); + + // should have matched something. + assert(matchLen > 0); + + + // now, do the replace. + + /** this is the 'to' or other replacement string.*/ + icu::UnicodeString rustr; + if (fromList.empty()) { + // Normal case: not a map. + // This replace will apply $1, $2 etc. + // Convert the fTo into u16 TODO-LDML (we could cache this?) + const std::u16string rstr = km::core::kmx::u32string_to_u16string(to); + rustr = icu::UnicodeString(rstr.data(), (int32_t)rstr.length()); + } else { + // Set map case: mapping from/to + + // we actually need the group(1) string here. + // this is only the content in parenthesis () + icu::UnicodeString group1 = matcher->group(1, status); + if (!UASSERT_SUCCESS(status)) { + // TODO-LDML: could be a malformed from pattern + return 0; // TODO-LDML: return error + } + // now, how long is group1 in UTF-32, hmm? + UErrorCode preflightStatus = U_ZERO_ERROR; // throwaway status + auto group1Len = group1.toUTF32(nullptr, 0, preflightStatus); + char32_t *s = new char32_t[group1Len + 1]; + assert(s != nullptr); // TODO-LDML: OOM + // convert + group1.toUTF32((UChar32 *)s, group1Len + 1, status); + if (!UASSERT_SUCCESS(status)) { + return 0; // TODO-LDML: memory issue + } + std::u32string match32(s, group1Len); // taken from just group1 + // clean up buffer + delete [] s; + + // Now we're ready to do the actual mapping. + + // 1., we need to find the index in the source set. + auto matchIndex = findIndex(match32, fromList); + assert(matchIndex != -1L); // TODO-LDML: not matching shouldn't happen, the regex wouldn't have matched. + // we already asserted on load that the from and to sets have the same cardinality. + + // 2. get the target string, convert to utf-16 + // we use the same matchIndex that was just found + const std::u16string rstr = km::core::kmx::u32string_to_u16string(toList.at(matchIndex)); + + // 3. update the UnicodeString for replacement + rustr = icu::UnicodeString(rstr.data(), (int32_t)rstr.length()); + // and we return to the regular code flow. + } + // here we replace the match output. No normalization, yet. + icu::UnicodeString entireOutput = matcher->replaceFirst(rustr, status); + if (!UASSERT_SUCCESS(status)) { + // TODO-LDML: could fail here due to bad input (syntax err) + return 0; + } + // entireOutput includes all of 'input', but modified. Need to substring it. + icu::UnicodeString outu = entireOutput.tempSubString(matchStart); + + // Special case if there's no output, save some allocs + if (outu.length() == 0) { + output.clear(); + } else { + // TODO-LDML: All we are trying to do is to extract the output string. Probably too many steps. + UErrorCode preflightStatus = U_ZERO_ERROR; + // calculate how big the buffer is + auto out32len = outu.toUTF32(nullptr, 0, preflightStatus); // preflightStatus will be an err, because we know the buffer overruns zero bytes + // allocate + std::unique_ptr s(new char32_t[out32len + 1]); + assert(s); + if (!s) { + return 0; // TODO-LDML: allocation failed + } + // convert + outu.toUTF32((UChar32 *)(s.get()), out32len + 1, status); + if (!UASSERT_SUCCESS(status)) { + return 0; // TODO-LDML: memory issue + } + output.assign(s.get(), out32len); + } + return matchLen; + +#endif +} + + +} +} +} diff --git a/core/src/util_regex.hpp b/core/src/util_regex.hpp new file mode 100644 index 0000000000..8d155815b7 --- /dev/null +++ b/core/src/util_regex.hpp @@ -0,0 +1,48 @@ +/* + Copyright: © SIL International. + Description: Normalization and Regex utilities + Create Date: 23 May 2024 + Authors: Steven R. Loomis +*/ + +#pragma once + +#include "core_icu.h" +#include "keyman_core.h" +#include +#include + +namespace km { +namespace core { +namespace util { + +class km_regex { +public: + km_regex(); + km_regex(const km_regex &other); + km_regex(const std::u32string &pattern); + ~km_regex(); + bool init(const std::u32string &pattern); + + size_t apply( + const std::u32string &input, + std::u32string &output, + const std::u32string &to, + const std::deque &fromList, + const std::deque &toList) const; + + bool valid() const; +private: +#if KMN_NO_ICU + void *stuff; +#else + std::unique_ptr fPattern; +#endif +// utility functions + public: + static int32_t findIndex(const std::u32string &match, const std::deque &list); +}; + +} // namespace util +} // namespace core +} // namespace km diff --git a/core/tests/unit/ldml/test_transforms.cpp b/core/tests/unit/ldml/test_transforms.cpp index 94c53f3845..bf95ba3115 100644 --- a/core/tests/unit/ldml/test_transforms.cpp +++ b/core/tests/unit/ldml/test_transforms.cpp @@ -1,5 +1,6 @@ #include "../../../src/ldml/ldml_markers.hpp" #include "../../../src/ldml/ldml_transforms.hpp" +#include "../../../src/util_regex.hpp" #include "kmx/kmx_plus.h" #include "kmx/kmx_xstring.h" #include "test_color.h" @@ -624,16 +625,16 @@ test_map() { std::cout << __FILE__ << ":" << __LINE__ << " transform_entry::findIndex" << std::endl; { std::deque list; - assert_equal(transform_entry::findIndex(U"Does Not Exist", list), -1); + assert_equal(km::core::util::km_regex::findIndex(U"Does Not Exist", list), -1); list.emplace_back(U"0th"); list.emplace_back(U"First"); list.emplace_back(U"Second"); - assert_equal(transform_entry::findIndex(U"First", list), 1); - assert_equal(transform_entry::findIndex(U"0th", list), 0); - assert_equal(transform_entry::findIndex(U"Second", list), 2); - assert_equal(transform_entry::findIndex(U"Nowhere", list), -1); + assert_equal(km::core::util::km_regex::findIndex(U"First", list), 1); + assert_equal(km::core::util::km_regex::findIndex(U"0th", list), 0); + assert_equal(km::core::util::km_regex::findIndex(U"Second", list), 2); + assert_equal(km::core::util::km_regex::findIndex(U"Nowhere", list), -1); } return EXIT_SUCCESS; From d66d474f64ac4e0d94df05f27b5886f44d5ae871 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 7 Jun 2024 11:56:18 +0700 Subject: [PATCH 10/64] fix(web): add limited Array.from polyfill for lm-worker use Fixes: #11502 Fixes: KEYMAN-WEB-K4 --- .../models/wordbreakers/src/default/index.ts | 2 +- common/web/lm-worker/build-polyfiller.js | 3 ++ .../web/lm-worker/src/polyfills/array.from.js | 47 +++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 common/web/lm-worker/src/polyfills/array.from.js diff --git a/common/models/wordbreakers/src/default/index.ts b/common/models/wordbreakers/src/default/index.ts index 20e159ed19..ef50ab4b31 100644 --- a/common/models/wordbreakers/src/default/index.ts +++ b/common/models/wordbreakers/src/default/index.ts @@ -274,7 +274,7 @@ export class BreakerContext { * @param chunk a chunk of text. Starts and ends at word boundaries. */ function isNonSpace(chunk: string, options?: DefaultWordBreakerOptions): boolean { - return !Array.from(chunk).map((char) => property(char, options)).every(wb => ( + return !chunk.split('').map((char) => property(char, options)).every(wb => ( wb === WordBreakProperty.CR || wb === WordBreakProperty.LF || wb === WordBreakProperty.Newline || diff --git a/common/web/lm-worker/build-polyfiller.js b/common/web/lm-worker/build-polyfiller.js index 20f3ab587f..1978038836 100644 --- a/common/web/lm-worker/build-polyfiller.js +++ b/common/web/lm-worker/build-polyfiller.js @@ -144,6 +144,9 @@ let sourceFileSet = [ // Needed for Android / Chromium browser pre-41. loadPolyfill('../../../node_modules/string.prototype.codepointat/codepointat.js', 'src/polyfills/string.codepointat.js'), // Needed for Android / Chromium browser pre-45. + // Not used in this codebase, but used by some compiled model defaults. + loadPolyfill('src/polyfills/array.from.js', 'src/polyfills/array.from.js'), + // Needed for Android / Chromium browser pre-45. loadPolyfill('src/polyfills/array.fill.js', 'src/polyfills/array.fill.js'), // Needed for Android / Chromium browser pre-45. loadPolyfill('src/polyfills/array.findIndex.js', 'src/polyfills/array.findIndex.js'), diff --git a/common/web/lm-worker/src/polyfills/array.from.js b/common/web/lm-worker/src/polyfills/array.from.js new file mode 100644 index 0000000000..6234d4a90c --- /dev/null +++ b/common/web/lm-worker/src/polyfills/array.from.js @@ -0,0 +1,47 @@ +if(!Array.from) { + function isHighSurrogate(codeUnit) { + if(typeof codeUnit == 'string') { + codeUnit = codeUnit.charCodeAt(0); + } + + return codeUnit >= 0xD800 && codeUnit <= 0xDBFF; + } + + function isLowSurrogate(codeUnit) { + if(typeof codeUnit == 'string') { + codeUnit = codeUnit.charCodeAt(0); + } + + return codeUnit >= 0xDC00 && codeUnit <= 0xDFFF; + } + + Array.from = function (obj) { + if(Array.isArray(obj)) { + // Simple array clone + return obj.slice(); + } else if(typeof obj == 'string') { + // Array.from is surrogate-aware and will not split surrogate pairs. + // We can start with a full split and then remerge the pairs. + var simpleSplit = obj.split(); + + /** @type {string[]} */ + var finalSplit = []; + /** @type {number} */ + var i; + + for(i=0; i < simpleSplit.length; i++) { + // Do we have a surrogate pair? + var a = simpleSplit.shift(); + if(isHighSurrogate(a) && isLowSurrogate(simpleShift[0] || '')) { + // yes, so merge them before pushing. + a = a + simpleSplit.shift(); + } // else: 'no', so just push the current char to the array and continue + + finalSplit.push(a); + } + return finalSplit; + } else { + throw "Unexpected + nonpolyfilled use of Array.from encountered; aborting"; + } + } +} \ No newline at end of file From 891928e10b99d8480ac1e87ca43026cb1f9c3b7b Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 7 Jun 2024 14:04:41 +0700 Subject: [PATCH 11/64] fix(web): array.from polyfill scoping, fixes after extracted test --- .../web/lm-worker/src/polyfills/array.from.js | 91 ++++++++++--------- 1 file changed, 47 insertions(+), 44 deletions(-) diff --git a/common/web/lm-worker/src/polyfills/array.from.js b/common/web/lm-worker/src/polyfills/array.from.js index 6234d4a90c..7e87e66678 100644 --- a/common/web/lm-worker/src/polyfills/array.from.js +++ b/common/web/lm-worker/src/polyfills/array.from.js @@ -1,47 +1,50 @@ -if(!Array.from) { - function isHighSurrogate(codeUnit) { - if(typeof codeUnit == 'string') { - codeUnit = codeUnit.charCodeAt(0); - } - - return codeUnit >= 0xD800 && codeUnit <= 0xDBFF; - } - - function isLowSurrogate(codeUnit) { - if(typeof codeUnit == 'string') { - codeUnit = codeUnit.charCodeAt(0); - } - - return codeUnit >= 0xDC00 && codeUnit <= 0xDFFF; - } - - Array.from = function (obj) { - if(Array.isArray(obj)) { - // Simple array clone - return obj.slice(); - } else if(typeof obj == 'string') { - // Array.from is surrogate-aware and will not split surrogate pairs. - // We can start with a full split and then remerge the pairs. - var simpleSplit = obj.split(); - - /** @type {string[]} */ - var finalSplit = []; - /** @type {number} */ - var i; - - for(i=0; i < simpleSplit.length; i++) { - // Do we have a surrogate pair? - var a = simpleSplit.shift(); - if(isHighSurrogate(a) && isLowSurrogate(simpleShift[0] || '')) { - // yes, so merge them before pushing. - a = a + simpleSplit.shift(); - } // else: 'no', so just push the current char to the array and continue - - finalSplit.push(a); +(function() { + if(!Array.from) { + function isHighSurrogate(codeUnit) { + if(typeof codeUnit == 'string') { + codeUnit = codeUnit.charCodeAt(0); + } + + return codeUnit >= 0xD800 && codeUnit <= 0xDBFF; + } + + function isLowSurrogate(codeUnit) { + if(typeof codeUnit == 'string') { + codeUnit = codeUnit.charCodeAt(0); + } + + return codeUnit >= 0xDC00 && codeUnit <= 0xDFFF; + } + + Array.from = function (obj) { + if(Array.isArray(obj)) { + // Simple array clone + return obj.slice(); + } else if(typeof obj == 'string') { + // Array.from is surrogate-aware and will not split surrogate pairs. + // We can start with a full split and then remerge the pairs. + var simpleSplit = obj.split(''); + + /** @type {string[]} */ + var finalSplit = []; + /** @type {number} */ + var i; + + while(simpleSplit.length > 0) { + // Do we have a surrogate pair? + var a = simpleSplit.shift(); + if(isHighSurrogate(a) && (isLowSurrogate(simpleSplit[0] || ''))) { + // yes, so merge them before pushing. + a = a + simpleSplit.shift(); + console.log(a); + } // else: 'no', so just push the current char to the array and continue + + finalSplit.push(a); + } + return finalSplit; + } else { + throw "Unexpected + nonpolyfilled use of Array.from encountered; aborting"; } - return finalSplit; - } else { - throw "Unexpected + nonpolyfilled use of Array.from encountered; aborting"; } } -} \ No newline at end of file +}()); \ No newline at end of file From 062a867ecda2179c006707638b01f297716ffc10 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 7 Jun 2024 14:24:41 +0700 Subject: [PATCH 12/64] change(android): removes unneeded conditionals --- common/web/lm-worker/src/polyfills/array.from.js | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/common/web/lm-worker/src/polyfills/array.from.js b/common/web/lm-worker/src/polyfills/array.from.js index 7e87e66678..ae6ba84600 100644 --- a/common/web/lm-worker/src/polyfills/array.from.js +++ b/common/web/lm-worker/src/polyfills/array.from.js @@ -1,18 +1,12 @@ (function() { if(!Array.from) { function isHighSurrogate(codeUnit) { - if(typeof codeUnit == 'string') { - codeUnit = codeUnit.charCodeAt(0); - } - + codeUnit = codeUnit.charCodeAt(0); return codeUnit >= 0xD800 && codeUnit <= 0xDBFF; } function isLowSurrogate(codeUnit) { - if(typeof codeUnit == 'string') { - codeUnit = codeUnit.charCodeAt(0); - } - + codeUnit = codeUnit.charCodeAt(0); return codeUnit >= 0xDC00 && codeUnit <= 0xDFFF; } From f473b4ff0d68efef97f131ac5c8952ebc87b0d38 Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Fri, 7 Jun 2024 11:03:01 +0100 Subject: [PATCH 13/64] fix(developer): googletest for compiler messages, plus refactor of CompMsg.cpp to use map --- developer/src/kmcmplib/src/CompMsg.cpp | 20 +++++-------------- developer/src/kmcmplib/src/meson.build | 8 ++++++++ developer/src/kmcmplib/subprojects/.gitignore | 1 + developer/src/kmcmplib/subprojects/gtest.wrap | 16 +++++++++++++++ .../src/kmcmplib/tests/gtest-compmsg-test.cpp | 19 ++++++++++++++++++ developer/src/kmcmplib/tests/meson.build | 13 ++++++++++++ 6 files changed, 62 insertions(+), 15 deletions(-) create mode 100644 developer/src/kmcmplib/subprojects/gtest.wrap create mode 100644 developer/src/kmcmplib/tests/gtest-compmsg-test.cpp diff --git a/developer/src/kmcmplib/src/CompMsg.cpp b/developer/src/kmcmplib/src/CompMsg.cpp index 86a9b4d475..6480458fda 100644 --- a/developer/src/kmcmplib/src/CompMsg.cpp +++ b/developer/src/kmcmplib/src/CompMsg.cpp @@ -1,11 +1,7 @@ #include +#include -struct CompilerError { - KMX_DWORD ErrorCode; - const KMX_CHAR* Text; - }; - -const struct CompilerError CompilerErrors[] = { +std::map CompilerErrorMap = { { CERR_InvalidLayoutLine , "Invalid 'layout' command"}, { CERR_NoVersionLine , "No version line found for file"}, { CERR_InvalidGroupLine , "Invalid 'group' command"}, @@ -150,14 +146,8 @@ const struct CompilerError CompilerErrors[] = { { CWARN_VirtualKeyInOutput , "Virtual keys are not supported in output"}, { 0, nullptr } - }; +}; -KMX_CHAR *GetCompilerErrorString(KMX_DWORD code) -{ - for(int i = 0; CompilerErrors[i].ErrorCode; i++) { - if(CompilerErrors[i].ErrorCode == code) { - return ( KMX_CHAR*) CompilerErrors[i].Text; - } - } - return nullptr; +KMX_CHAR *GetCompilerErrorString(KMX_DWORD code) { + return (KMX_CHAR*) CompilerErrorMap[code]; } diff --git a/developer/src/kmcmplib/src/meson.build b/developer/src/kmcmplib/src/meson.build index e3e4efcd48..61b909a75e 100644 --- a/developer/src/kmcmplib/src/meson.build +++ b/developer/src/kmcmplib/src/meson.build @@ -54,6 +54,10 @@ if cpp_compiler.get_id() == 'emscripten' # emscripten < 3.1.44 does not include `wasmExports` links += ['-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\']'] endif + + # For Google Test + add_global_arguments('-pthread', language: [ 'cpp', 'c' ] ) + endif icu = subproject('icu-for-uset', default_options: [ 'default_library=static', 'cpp_std=c++17', 'warning_level=0', 'werror=false']) @@ -116,3 +120,7 @@ if cpp_compiler.get_id() == 'emscripten' endif endif + +gtest = subproject('gtest') +gtest_dep = gtest.get_variable('gtest_main_dep') +gmock_dep = gtest.get_variable('gmock_dep') diff --git a/developer/src/kmcmplib/subprojects/.gitignore b/developer/src/kmcmplib/subprojects/.gitignore index 99479aa952..37346d9045 100644 --- a/developer/src/kmcmplib/subprojects/.gitignore +++ b/developer/src/kmcmplib/subprojects/.gitignore @@ -2,3 +2,4 @@ /*.zip /*.tgz /packagecache +/googletest-1.14.0 diff --git a/developer/src/kmcmplib/subprojects/gtest.wrap b/developer/src/kmcmplib/subprojects/gtest.wrap new file mode 100644 index 0000000000..adb8a9a6d9 --- /dev/null +++ b/developer/src/kmcmplib/subprojects/gtest.wrap @@ -0,0 +1,16 @@ +[wrap-file] +directory = googletest-1.14.0 +source_url = https://github.com/google/googletest/archive/refs/tags/v1.14.0.tar.gz +source_filename = gtest-1.14.0.tar.gz +source_hash = 8ad598c73ad796e0d8280b082cebd82a630d73e73cd3c70057938a6501bba5d7 +patch_filename = gtest_1.14.0-2_patch.zip +patch_url = https://wrapdb.mesonbuild.com/v2/gtest_1.14.0-2/get_patch +patch_hash = 4ec7f767364386a99f7b2d61678287a73ad6ba0f9998be43b51794c464a63732 +source_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/gtest_1.14.0-2/gtest-1.14.0.tar.gz +wrapdb_version = 1.14.0-2 + +[provide] +gtest = gtest_dep +gtest_main = gtest_main_dep +gmock = gmock_dep +gmock_main = gmock_main_dep diff --git a/developer/src/kmcmplib/tests/gtest-compmsg-test.cpp b/developer/src/kmcmplib/tests/gtest-compmsg-test.cpp new file mode 100644 index 0000000000..94f0f40d8d --- /dev/null +++ b/developer/src/kmcmplib/tests/gtest-compmsg-test.cpp @@ -0,0 +1,19 @@ +#include +#include "..\..\common\include\kmn_compiler_errors.h" +#include "..\..\..\..\common\include\km_types.h" + +KMX_CHAR *GetCompilerErrorString(KMX_DWORD code); + +class CompMsgTest : public testing::Test { + protected: + CompMsgTest() {} + ~CompMsgTest() override {} + void SetUp() override {} + void TearDown() override {} +}; + +TEST_F(CompMsgTest, GetCompilerErrorString) { + EXPECT_EQ(nullptr, GetCompilerErrorString(CERR_None)); + EXPECT_EQ(nullptr, GetCompilerErrorString(0x00004FFF)); // top of range ERROR + EXPECT_EQ("Invalid 'layout' command", GetCompilerErrorString(CERR_InvalidLayoutLine)); +}; \ No newline at end of file diff --git a/developer/src/kmcmplib/tests/meson.build b/developer/src/kmcmplib/tests/meson.build index 1ae32590e2..1095b48095 100644 --- a/developer/src/kmcmplib/tests/meson.build +++ b/developer/src/kmcmplib/tests/meson.build @@ -154,3 +154,16 @@ usetapitest = executable('uset-api-test', 'uset-api-test.cpp', common_test_files ) test('uset-api-test', usetapitest) + +# Google Test + +gtestcompmsgtest = executable('gtest-compmsg-test', 'gtest-compmsg-test.cpp', + cpp_args: defns + flags, + include_directories: inc, + name_suffix: name_suffix, + link_args: links + tests_links, + objects: lib.extract_all_objects(), + dependencies: [ icuuc_dep, gtest_dep, gmock_dep ], + ) + +test('gtest-compmsg-test', gtestcompmsgtest) From c4b01f7715d9b0f4d24a351222430504a2d752ad Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 7 Jun 2024 16:27:45 -0500 Subject: [PATCH 14/64] test(core): detailed unit test for util_regex Fixes: #9467 --- core/tests/unit/ldml/test_transforms.cpp | 101 +++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/core/tests/unit/ldml/test_transforms.cpp b/core/tests/unit/ldml/test_transforms.cpp index bf95ba3115..8175badc93 100644 --- a/core/tests/unit/ldml/test_transforms.cpp +++ b/core/tests/unit/ldml/test_transforms.cpp @@ -7,6 +7,7 @@ #include #include #include +#include "../../../src/util_regex.hpp" // TODO-LDML: normal asserts wern't working, so using some hacks. // #include "ldml_test_utils.hpp" @@ -1136,6 +1137,102 @@ test_normalize() { return EXIT_SUCCESS; } +/** test for the util_regex.hpp functions */ +int +test_util_regex() { + std::cout << "== " << __FUNCTION__ << std::endl; + + { + std::cout << __FILE__ << ":" << __LINE__ << " * util_regex.hpp null tests" << std::endl; + km::core::util::km_regex r; + assert(!r.valid()); // not valid because of an empty string + } + { + std::cout << __FILE__ << ":" << __LINE__ << " * util_regex.hpp simple tests" << std::endl; + km::core::util::km_regex r(U"ion"); + assert(r.valid()); + const std::u32string to(U"ivity"); + const std::deque fromList; + const std::deque toList; + std::u32string output; + auto apply0 = r.apply(U"not present", output, to, fromList, toList); + assert_equal(apply0, 0); // not found + + const std::u32string input(U"action"); + auto apply1 = r.apply(input, output, to, fromList, toList); + assert_equal(apply1, 3); // matched last 3 codepoints + std::u32string expect(U"ivity"); + zassert_string_equal(output, expect) + } + { + std::cout << __FILE__ << ":" << __LINE__ << " * util_regex.hpp wide tests" << std::endl; + km::core::util::km_regex r(U"e𐒻"); + assert(r.valid()); + const std::u32string to(U"𐓏"); + const std::deque fromList; + const std::deque toList; + std::u32string output; + const std::u32string input(U":e𐒻"); + auto apply1 = r.apply(input, output, to, fromList, toList); + assert_equal(apply1, 2); // matched last 3 codepoints + std::u32string expect(U"𐓏"); + zassert_string_equal(output, expect) + } + { + std::cout << __FILE__ << ":" << __LINE__ << " * util_regex.hpp simple map tests" << std::endl; + km::core::util::km_regex r(U"(A|B|C)"); + assert(r.valid()); + const std::u32string to(U"$[1:alpha2]"); // ignored + std::deque fromList; + fromList.emplace_back(U"A"); + fromList.emplace_back(U"B"); + fromList.emplace_back(U"C"); + std::deque toList; + toList.emplace_back(U"N"); + toList.emplace_back(U"O"); + toList.emplace_back(U"P"); + std::u32string output; + auto apply0 = r.apply(U"not present", output, to, fromList, toList); + assert_equal(apply0, 0); // not found + + const std::u32string input(U"WHOA"); + auto apply1 = r.apply(input, output, to, fromList, toList); + assert_equal(apply1, 1); // matched last 1 codepoint + std::u32string expect(U"N"); + zassert_string_equal(output, expect) + } + { + std::cout << __FILE__ << ":" << __LINE__ << " * util_regex.hpp wide map tests" << std::endl; + km::core::util::km_regex r(U"(𐒷|𐒻|𐓏𐓏|x)"); + assert(r.valid()); + const std::u32string to(U"$[1:alpha2]"); // ignored + std::deque fromList; + fromList.emplace_back(U"𐒷"); + fromList.emplace_back(U"𐒻"); + fromList.emplace_back(U"𐓏𐓏"); + fromList.emplace_back(U"x"); + std::deque toList; + toList.emplace_back(U"x"); + toList.emplace_back(U"𐒷"); + toList.emplace_back(U"𐒻"); + toList.emplace_back(U"𐓏"); + std::u32string output; + auto apply0 = r.apply(U"not present", output, to, fromList, toList); + assert_equal(apply0, 0); // not found + + assert_equal(r.apply(U"WHO𐓏𐒷", output, to, fromList, toList), 1); + zassert_string_equal(output, U"x"); + + assert_equal(r.apply(U"WHO𐓏x", output, to, fromList, toList), 1); + zassert_string_equal(output, U"𐓏"); + + assert_equal(r.apply(U"WHO𐓏𐓏", output, to, fromList, toList), 2); // 2 codepoints + zassert_string_equal(output, U"𐒻"); + } + + return EXIT_SUCCESS; +} + int main(int argc, const char *argv[]) { int rc = EXIT_SUCCESS; @@ -1178,6 +1275,10 @@ main(int argc, const char *argv[]) { rc = EXIT_FAILURE; } + if (test_util_regex() != EXIT_SUCCESS) { + rc = EXIT_FAILURE; + } + if (rc == EXIT_FAILURE) { std::cout << "== FAILURE" << std::endl; } else { From acd3cc5847dbd05e44a398ff2641d6f2fb09c05d Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 7 Jun 2024 18:11:20 -0500 Subject: [PATCH 15/64] feat(core): devolve regex to JS - use JS regex code - almost there - some perhaps escaping differences Fixes: #9467 --- core/src/util_regex.cpp | 130 ++++++++++++++++++++++- core/src/util_regex.hpp | 2 +- core/tests/unit/ldml/test_transforms.cpp | 25 +++-- 3 files changed, 138 insertions(+), 19 deletions(-) diff --git a/core/src/util_regex.cpp b/core/src/util_regex.cpp index f6c0801997..3943c87360 100644 --- a/core/src/util_regex.cpp +++ b/core/src/util_regex.cpp @@ -10,6 +10,85 @@ #include "core_icu.h" #include "kmx/kmx_xstring.h" +#ifdef __EMSCRIPTEN__ +#include +#include "utfcodec.hpp" +#include +// JS implementations + + +/** + * RegexMatchLen(pattern, input) + * @param pattern the string, sans trailing $, for the pattern + * @param input the text to match against + * @return the length, in code units, of the matched portion + */ +EM_JS(int, RegexMatchLen, (const char* pattern, const char *input), { + const DEBUG_JS = false; + if (!pattern) return -1; + if (!input) return -1; + const patternstr = Module.UTF8ToString(pattern) + '$'; + const inputstr = Module.UTF8ToString(input); + const re = new RegExp(patternstr); + const result = re.exec(inputstr); + if (DEBUG_JS) console.dir({patternstr,inputstr,re,result}); + if (!result) return 0; // no match + const index = result.index; + // code unit indices + const startIndex = index; + const endIndex = inputstr.length; + const matchedText = inputstr.substring(startIndex); + const matchedCodepoints = [...matchedText]; + if (DEBUG_JS) console.dir({index, startIndex,endIndex,matchedText,matchedCodepoints}); + return matchedCodepoints.length; +}); + +/** + * RegexGroup1(pattern, input) + * @param pattern the string, sans trailing $, for the pattern + * @param input the text to match against + * @return string of group 1 match or null + */ +EM_JS(char*, RegexGroup1, (const char* pattern, const char *input), { + const DEBUG_JS = false; + if (!pattern) return 0; + if (!input) return 0; + const patternstr = Module.UTF8ToString(pattern) + '$'; + const inputstr = Module.UTF8ToString(input); + const re = new RegExp(patternstr); + const result = re.exec(inputstr); + if (!result) return 0; // no match + const g1 = result[1]; + if (DEBUG_JS) console.dir({patternstr,inputstr,re,result,g1}); + if (!g1) return 0; + return stringToNewUTF8(g1); +}); + + +/** + * RegexSubstitute + * @param pattern the string, sans trailing $, for the pattern + * @param input the text to match against + * @param to the replacement text + * @return the entire updated output string + */ +EM_JS(char*, RegexSubstitute, (const char* pattern, const char *input, const char *to), { + const DEBUG_JS = false; + if (!pattern) return -1; + if (!input) return -1; + const patternstr = Module.UTF8ToString(pattern) + '$'; + const inputstr = Module.UTF8ToString(input); + const tostr = Module.UTF8ToString(to); + const re = new RegExp(patternstr); + const output = inputstr.replace(re, tostr); + return stringToNewUTF8(output); +}); + +// pull in the generated table +#include "util_normalize_table.h" + +#endif + namespace km { namespace core { @@ -38,6 +117,7 @@ km_regex::km_regex() km_regex::km_regex(const km_regex& other) #if KMN_NO_ICU + : fPattern(other.fPattern) #else : fPattern(nullptr) #endif @@ -54,6 +134,7 @@ km_regex::km_regex(const km_regex& other) km_regex::km_regex(const std::u32string &pattern) #if KMN_NO_ICU + : fPattern(pattern) #else : fPattern(nullptr) #endif @@ -67,7 +148,7 @@ km_regex::~km_regex() { bool km_regex::valid() const { #if KMN_NO_ICU -#error todo + return (!fPattern.empty()); #else // valid if fPattern is present. return !!fPattern; @@ -76,7 +157,10 @@ bool km_regex::valid() const { bool km_regex::init(const std::u32string &pattern) { #if KMN_NO_ICU -#error todo +// for now- new regex every time. + assert(!pattern.empty()); + fPattern = pattern; + return true; // TODO #else if (pattern.empty()) { return false; @@ -97,7 +181,44 @@ size_t km_regex::apply(const std::u32string &input, std::u32string &output, const std::deque &fromList, const std::deque &toList ) const { #if KMN_NO_ICU -#error TODO + // length in code points of match from end + std::string patstr = convert(fPattern); + std::string instr = convert(input); + + /** code units */ + const auto matchLen = RegexMatchLen(patstr.c_str(), instr.c_str()); + assert(matchLen != -1); // error + if (matchLen == 0) { + // TODO: not correct, just trying to quell unused arg. + return 0; + } + std::u32string rustr; // replacement + if (fromList.empty()) { + rustr = to; + } else { + + char *group1 = RegexGroup1(patstr.c_str(), instr.c_str()); + assert(group1 != nullptr); + const std::string group1str(group1); + const std::u32string match32 = convert(group1str); + free(group1); + auto matchIndex = findIndex(match32, fromList); + assert(matchIndex != 1L); + rustr = toList.at(matchIndex); + } + std::string rstr = convert(rustr); + // now, perform substitution + char *out = RegexSubstitute(patstr.c_str(), instr.c_str(), rstr.c_str()); + assert(out != nullptr); + std::string outstr(out); + free(out); + output = convert(outstr); + /** code units */ + const auto matchStart = input.length() - matchLen; + // remove the unmatched prefix. + output.erase(0, matchStart); + + return matchLen; #else assert(fPattern); // TODO-LDML: Really? can't go from u32 to UnicodeString? @@ -132,7 +253,6 @@ size_t km_regex::apply(const std::u32string &input, std::u32string &output, // should have matched something. assert(matchLen > 0); - // now, do the replace. /** this is the 'to' or other replacement string.*/ @@ -171,7 +291,7 @@ size_t km_regex::apply(const std::u32string &input, std::u32string &output, // 1., we need to find the index in the source set. auto matchIndex = findIndex(match32, fromList); - assert(matchIndex != -1L); // TODO-LDML: not matching shouldn't happen, the regex wouldn't have matched. + assert(matchIndex != -1L); // This indicates that the regex and the fromList are out of sync. // we already asserted on load that the from and to sets have the same cardinality. // 2. get the target string, convert to utf-16 diff --git a/core/src/util_regex.hpp b/core/src/util_regex.hpp index 8d155815b7..b10ddff1cf 100644 --- a/core/src/util_regex.hpp +++ b/core/src/util_regex.hpp @@ -34,7 +34,7 @@ public: bool valid() const; private: #if KMN_NO_ICU - void *stuff; + std::u32string fPattern; // TODO: by value? #else std::unique_ptr fPattern; #endif diff --git a/core/tests/unit/ldml/test_transforms.cpp b/core/tests/unit/ldml/test_transforms.cpp index 8175badc93..fa909516bc 100644 --- a/core/tests/unit/ldml/test_transforms.cpp +++ b/core/tests/unit/ldml/test_transforms.cpp @@ -15,14 +15,14 @@ // #include "debuglog.h" #ifndef zassert_string_equal -#define zassert_string_equal(actual, expected) \ - { \ - if (actual != expected) { \ - std::wcerr << __FILE__ << ":" << __LINE__ << ": " << console_color::fg(console_color::BRIGHT_RED) \ - << "got: " << km::core::kmx::Debug_UnicodeString(actual, 0) << " expected " \ - << km::core::kmx::Debug_UnicodeString(expected, 1) << console_color::reset() << std::endl; \ - return EXIT_FAILURE; \ - } \ +#define zassert_string_equal(actual, expected) \ + { \ + if (actual != expected) { \ + std::wcerr << __FILE__ << ":" << __LINE__ << ": " << console_color::fg(console_color::BRIGHT_RED) << "got: " << actual \ + << " " << km::core::kmx::Debug_UnicodeString(actual, 0) << " expected " << expected << " " \ + << km::core::kmx::Debug_UnicodeString(expected, 1) << console_color::reset() << std::endl; \ + return EXIT_FAILURE; \ + } \ } #endif @@ -1174,7 +1174,7 @@ test_util_regex() { std::u32string output; const std::u32string input(U":e𐒻"); auto apply1 = r.apply(input, output, to, fromList, toList); - assert_equal(apply1, 2); // matched last 3 codepoints + assert_equal(apply1, 2); // matched last 2 codepoints std::u32string expect(U"𐓏"); zassert_string_equal(output, expect) } @@ -1250,6 +1250,9 @@ main(int argc, const char *argv[]) { console_color::enabled = console_color::isaterminal() || arg_color; + if (test_util_regex() != EXIT_SUCCESS) { + rc = EXIT_FAILURE; + } if (test_transforms() != EXIT_SUCCESS) { rc = EXIT_FAILURE; @@ -1275,10 +1278,6 @@ main(int argc, const char *argv[]) { rc = EXIT_FAILURE; } - if (test_util_regex() != EXIT_SUCCESS) { - rc = EXIT_FAILURE; - } - if (rc == EXIT_FAILURE) { std::cout << "== FAILURE" << std::endl; } else { From 0cac134d1979b7fe4e5545b05025b1d520465315 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 10 Jun 2024 08:39:45 +0700 Subject: [PATCH 16/64] fix(web): set new-context rules' device to match that of the active OSK Fixes: #11740 --- web/src/engine/main/src/keymanEngine.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/web/src/engine/main/src/keymanEngine.ts b/web/src/engine/main/src/keymanEngine.ts index 7e100f0f36..890574e284 100644 --- a/web/src/engine/main/src/keymanEngine.ts +++ b/web/src/engine/main/src/keymanEngine.ts @@ -349,6 +349,10 @@ export default class KeymanEngine< this.core.keyboardProcessor.layerStore.handler = this.osk.layerChangeHandler; } this._osk = value; + // As the `new context` ruleset is designed to facilitate OSK layer-change updates + // based on the context being entered, we want the keyboard processor's current + // contextDevice to match that of the active OSK. See #11740. + this.core.keyboardProcessor.contextDevice = value.targetDevice ?? this.config.softDevice; if(value) { // Don't build an OSK if no keyboard is available yet; avoid the extra flash. if(this.contextManager.activeKeyboard) { From be78b58df148ceeb5a6dbbada1ef04beb507d3ef Mon Sep 17 00:00:00 2001 From: Meng-Heng Date: Tue, 11 Jun 2024 14:36:19 +0700 Subject: [PATCH 17/64] docs(developer): context help in keyboard-editor section Fixes: #2131 --- developer/src/tike/xml/help/contexthelp.xml | 97 +++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/developer/src/tike/xml/help/contexthelp.xml b/developer/src/tike/xml/help/contexthelp.xml index 62bd8677ae..134fb619a4 100644 --- a/developer/src/tike/xml/help/contexthelp.xml +++ b/developer/src/tike/xml/help/contexthelp.xml @@ -102,6 +102,18 @@ ======================================================================== -->
+ +

Keyboard Editor page consists of: +

    +
  • Details
    • +
  • Layout
    • +
  • Icon
    • +
  • On-Screen
    • +
  • Touch Layout
    • +
  • Build
    • +
+

+ @@ -139,6 +151,24 @@

This corresponds to the following source line:

store(&message) 'Here is a message about a keyboard'
 + + +

The Keyboard Version documents the version of the keyboard.

+

A keyboard version should be updated whenever there are changes to a keyboard. The good principles to follow are: +

    +
  • Increment the major version number for a + keyboard that has significant new functionality.
    • +
  • Increment the minor version number for changes that impact functionality but not + in a significant manner.
    • +
  • Optionally, use a third number for bug fixes.
    • +
+

+

This corresponds to the following source line:

+
store(&keyboardversion) '1.1.2'
+

Note: there is a difference between &keyboardversion, which documents the keyboard version, and &version, + which determines which version of Keyman a keyboard will run with.

+ +

In this field, enter information about the keyboard for your own reference. These comments will only be visible in the source file, and not to users of your keyboard.

@@ -183,6 +213,41 @@

Tests your KeymanWeb keyboard in an Internet Explorer embedded window

 + +

The Features grid controls which additional file components are included in + the keyboard. Each of the features relates to a system store. Here are the file + components: +

    +
  • Embedded JavaScript
    • +
  • Embedded CSS
    • +
  • Web Help
    • +
  • Include Codes
    • +
  • Desktop On-Screen Keyboard (auto-included if Targets is any)
    • +
  • Touch-Optimised Keyboard (auto-included if Targets is any)
    • +
+ Icon will be automatically included when a new keyboard project is created. +

+ + + +

This will open a selection dialog allowing you to choose a feature to add to + the keyboard project. Adding a feature will add an extra tab to the editor, + and add the corresponding store to the keyboard source

+ + + +

Depending on what is included in the Feature Grid, you can select a feature from + the grid then click on Edit... This will take you to the corresponding tab and + let you make changes. +

+ + + +

Removing a feature will not delete the component file, + but will just remove the store from the keyboard source. +

+ + @@ -197,8 +262,40 @@

The toolbox allows for the changing of the colours, addition of text and shapes. It also allows for the moving of the icon around the canvas and also a preview of the icon is displayed.

 + + +

Click on a colour from the box to apply the colour onto the Keyboard's + icon. You can see the Foreground Color displays the colour you chose. To deselect + the colour, click on the X mark on the corner left of the colour box, or choose + another colour. +

+ + +

This tab allows you to edit the visual representation of your keyboard + layout. The content on this tab is stored in the .kvks file associated with + your keyboard. The visual representation is used only in desktop and desktop + web; however if no touch layout is defined, this layout will be synthesized + into a touch layout automatically. +

+

An On-Screen keyboard is optional but in most keyboards is recommended. + The On-Screen keyboard may not always match the actual layout identically, + because you may choose to hide some of the details of encoding from the + interface presented to the user. +

+

+ This keyboard layout can also be printed or included in HTML or other documentation. + The editor allows you to export the file to HTML, PNG or BMP formats. +

+ + + +

If this option is checked, when the Fill from layout button is clicked, + then keys without corresponding rules in the Layout will be filled with the + base layout character. +

+ From 4116f73ecccf61ffa649a12e812d3a4409f29eaf Mon Sep 17 00:00:00 2001 From: MengHeng <90595388+Meng-Heng@users.noreply.github.com> Date: Wed, 12 Jun 2024 11:50:07 +0700 Subject: [PATCH 18/64] Apply suggestions from code review Co-authored-by: Marc Durdin --- developer/src/tike/xml/help/contexthelp.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer/src/tike/xml/help/contexthelp.xml b/developer/src/tike/xml/help/contexthelp.xml index 134fb619a4..9c7750f7b7 100644 --- a/developer/src/tike/xml/help/contexthelp.xml +++ b/developer/src/tike/xml/help/contexthelp.xml @@ -102,7 +102,7 @@ ======================================================================== --> - +

Keyboard Editor page consists of:

  • Details
    • From 074c8f7dac9f865ec4ec4b825206ef7479a09b88 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 12 Jun 2024 14:52:38 +0700 Subject: [PATCH 19/64] fix(developer): show message if no more platforms to add to touch layout editor Fixes: #11646 Fixes: KEYMAN-DEVELOPER-1X2 --- .../src/tike/xml/layoutbuilder/builder.xsl | 4 ++++ .../xml/layoutbuilder/platform-controls.js | 24 ++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/developer/src/tike/xml/layoutbuilder/builder.xsl b/developer/src/tike/xml/layoutbuilder/builder.xsl index 3b75cd81a3..d43d90c574 100644 --- a/developer/src/tike/xml/layoutbuilder/builder.xsl +++ b/developer/src/tike/xml/layoutbuilder/builder.xsl @@ -271,6 +271,10 @@ +
    +

    All available platforms have already been added.

    +
    +
    diff --git a/developer/src/tike/xml/layoutbuilder/platform-controls.js b/developer/src/tike/xml/layoutbuilder/platform-controls.js index 965513e69c..bccb99717c 100644 --- a/developer/src/tike/xml/layoutbuilder/platform-controls.js +++ b/developer/src/tike/xml/layoutbuilder/platform-controls.js @@ -5,14 +5,20 @@ $(function() { for (var platform in KVKL) { platforms[platform] = 0; } + let nPlatforms = 0; for (platform in platforms) { if (platforms[platform]) { var opt = document.createElement('option'); $(opt).text(platform); $('#selAddPlatform').append(opt); + nPlatforms++; } } - $('#addPlatformDialog').dialog('open') + if(nPlatforms == 0) { + $('#addPlatformDialogNoPlatformsToAdd').dialog('open') + } else { + $('#addPlatformDialog').dialog('open') + } }); $('#btnDelPlatform').click(function () { @@ -69,6 +75,22 @@ $(function() { } }); + // + // Platform dialog -- no platforms to add + // + + $('#addPlatformDialogNoPlatformsToAdd').dialog({ + autoOpen: false, + height: 150, + width: 350, + modal: true, + buttons: { + "OK": function () { + $(this).dialog('close'); + } + } + }); + // // Platform Properties Dialog // From 2d6d007f74fc70c25a3b0ea4a42760b2c5884f45 Mon Sep 17 00:00:00 2001 From: Meng-Heng Date: Wed, 12 Jun 2024 15:35:17 +0700 Subject: [PATCH 20/64] docs(developer): context help in package-editor and put the context help in their tabs Fixes a portion of #2131: --- developer/src/tike/xml/help/contexthelp.xml | 217 ++++++++++++++++---- 1 file changed, 173 insertions(+), 44 deletions(-) diff --git a/developer/src/tike/xml/help/contexthelp.xml b/developer/src/tike/xml/help/contexthelp.xml index 62bd8677ae..cd9d37f76a 100644 --- a/developer/src/tike/xml/help/contexthelp.xml +++ b/developer/src/tike/xml/help/contexthelp.xml @@ -331,33 +331,185 @@

    The debugger can be used without the debug information by clicking on Test without debugger.

    - + +

    Enter the name for your new keyboard. Click on the Browse button to change the location of the new keyboard.
    - + +
    + +

    A keyboard package is most likely to have 7 tabs: +

      +
    • Files
      • +
    • Keyboards
      • +
    • Lexical Models
      • +
    • Details
      • +
    • Shortcuts
      • +
    • Source
      • +
    • Build
      • +
    +

    +     + + + +

    Usually, there is only one keyboard listed in the box, and by clicking on it + will show the keyboard information.

    +     + +

    When font files are added to the package, this dropdown tells the Keyman + apps which font to use when rendering the On Screen Keyboard touch keyboard. +

    +     + +

    When font files are added to the package, this dropdown tells the Keyman + apps for iOS and Android which font to use in edit fields. It only applies + within the Keyman app and apps that support this functionality.

    +     + +

    Each language listed here is a BCP 47 language tag and every keyboard must have a + minimum of one language. When Keyman installs the keyboard package, it will associate the + keyboard with the language(s) you select.

    +     + +

    This button will open up the BCP 47 Tag window to add the keyboard's language tag, script tag, + region tag, and language name.

    +     + +

    Select on a language then click on Remove to delete the language tag.

    +     + +

    Click on Edit... then the BCP 47 Tag window will pop up once again with the language information.

    +     + + + +

    This is the location of the Lexical Model (Predictive text) for the keyboard in the keyboard + package.

    +     + +

    Add button brings up the “Select BCP 47 Tag” + window, thus allowing the input of a language for the Lexical Model (Predictive text).

    +     + +

    By ticking this, it will set the text direction of the Lexical Model (Predictive text) to Right-to-left.

    +     + +

    A short or long text related to the Lexical Model (Predictive text) is encouraged to be added here but it is optional.

    +     + + + +

    Choose from the list to specifies which welcome file is suitable to display when they install the keyboard package. + However, this is optional, most keyboard package uses the default option which is (none).

    +     + +

    Choose from the list if there is a License file to specified to the keyboard package. + However, this is optional, most keyboard package uses the default option which is (none) and we + only accept an open-license keyboard package.

    +     + +

    Enter the name of the author or authors of the keyboard package.

    +     + +

    Enter the copyright details of the keyboard package This information will be displayed with the version, author and message information when the package is installed.

    +     + +

    Enter the contact email address for the keyboard package.

    +     + +

    Package Name is the name that will be displayed when the package is installed. It should be a descriptive name, in any language, but remember that some applications may use a font that does not include the language you are writing the keyboard name in. Don't include a version number, help information or hotkey in the name.

    +     + +

    The version number allows the user to check whether they have the latest version of the keyboard. The format should be 'major.minor[.subversion]'. Each number should be an integer, and you should avoid non-integer version strings. See help for more details.

    +     + +

    The website details for the keyboard package if available.

    +     + +

    Specifies an independant version for the Lexical Model.

    +     + +

    By ticking this, the Lexical Model (Predictive text) version will receive a version bump alongside + the keyboard, even when the Model does not receive an update.

    +     + +

    A keyboard package's description about the language, wonderful community, script, or any related information + that would help users once they install the package.

    +     + +

    If a keyboard package is intended to replace an existing keyboard, or if there are related packages, + then the identifiers for these packages should be listed here.

    +     + +

    Add the Package ID and specify Deprecated or Non-deprecated if a keyboard package is intended to + replace an existing keyboard, or if there are related packages.

    +     + +

    Selects the package and click Edit... to add any changes to the current information.

    +     + +

    A keyboard package's description about the language, community, script, or any related information + that would help users when they see once installing the package.

    +     + + + +

    Enter the path of the start menu to be displayed when the keyboard package is installed.

    +     + +

    This is a list of Start menu entries for the keyboard package.

    +     + +

    The uninstall shortcut will be added automatically to the shortcut menu list when the package is installed.

    +     + +

    This option will allow you to create a folder on the Start menu when the keyboard package is installed.

    +     + +

    To add a new menu item to the list of shortcuts to be displayed in the Start menu folder created when the keyboard package is installed.

    +     + +

    To delete the selected shortcut menu item from the Start menu folder.

    +     + +

    The text to be displayed for the selected file as a menu item in the Start up folder that is created when the package is installed.

    +     + + + + + + +

    Starts the Keyman Developer Web Server for the keyboard package. + This will list the various IP addresses and hostnames that Keyman Developer is listening on.

    +     + +

    A list of available servers to test the keyboard package.

    +     + +

    Starts your default browser with the selected address to allow testing of the keyboard package + directly.

    +     + +

    Specifies the location of the Keyman MSI file. As of Keyman Developer 17, bundled executable package + installers for Keyman for Windows can be created using kmc, but cannot be created within the IDE.

    +     +

    You can include an image file that will be displayed to the left of the install details when the package is installed. This image should be 140 pixels wide and 250 pixels high.

    The readme file can be displayed after the installation of the keyboard package, but can also be accessed from the keyboard folder at a later time. The file must be loaded under the Files tab, add option before being able to be selected from the drop down list.

     - - -

    This option will allow you to create a folder on the Start menu when the keyboard package is installed.

    -     - -

    The uninstall shortcut will be added automatically to the shortcut menu list when the package is installed.

    -     +

    This option allows the user to add new files to the package.

     - -

    To delete the selected shortcut menu item from the Start menu folder.

    -

    This allows for the insertion of a copyright symbol if needed.

     @@ -375,8 +527,6 @@ installed correctly. If you can, try installing your package on several different machines.

     - -

    Displays the output path and filename of the file when the package is compiled.

     @@ -404,9 +554,6 @@

    This option will install the package on the computer. A message will be displayed as to the success of the install.

     - -

    To add a new menu item to the list of shortcuts to be displayed in the Start menu folder created when the keyboard package is installed.

    -

    Opens the source folder of the selected file.

     @@ -432,33 +579,6 @@

    File Type

    Enter the details of the file type being added to the keyboard package.

     - -

    Enter the name of the author or authors of the keyboard package.

    -     - -

    Enter the copyright details of the keyboard package This information will be displayed with the version, author and message information when the package is installed.

    -     - -

    Enter the contact email address for the keyboard package.

    -     - -

    Package Name is the name that will be displayed when the package is installed. It should be a descriptive name, in any language, but remember that some applications may use a font that does not include the language you are writing the keyboard name in. Don't include a version number, help information or hotkey in the name.

    -     - -

    The version number allows the user to check whether they have the latest version of the keyboard. The format should be 'major.minor[.subversion]'. Each number should be an integer, and you should avoid non-integer version strings. See help for more details.

    -     - -

    The website details for the keyboard package if available.

    -     - - -

    The text to be displayed for the selected file as a menu item in the Start up folder that is created when the package is installed.

    -     - - -

    Start Menu Path

    -

    Enter the path of the start menu to be displayed when the keyboard package is installed.

    -

    This will display all the files that have been added to the keyboard package. It allows for the addition and removal of files, the entering of file details and the editing of any of the files listed if the appropriate editor is available.

     @@ -467,12 +587,14 @@
    +

    The Project Manager allows you to manage all the files related to a keyboard layout in a single location.
    +

    The name of the developer of the keyboard. This is either your full name or @@ -560,6 +682,7 @@

    +

    Wordlist tabs have two views: Design, and Code. Changes to one view are reflected @@ -584,6 +707,7 @@

    +

    Editor windows in Keyman Developer supports standard Windows editing keystrokes. @@ -593,6 +717,7 @@

    +

    Attempt to identify the fonts on your system that will support the @@ -605,6 +730,7 @@

    +

    The message window appears at the bottom of the screen, or floating in a toolbar @@ -613,6 +739,7 @@

    +

    The debugger input window is used for typing input to test the keyboard. @@ -689,6 +816,7 @@

    +

    The About dialog displays copyright and registration information for Keyman Developer, @@ -696,6 +824,7 @@

    +

    This dialog lets you check the virtual key code for any key combination (except Window reserved key combinations such as Alt + Tab). You can then insert the virtual key code into the last active edit window at the current cursor position.



    From 9d2d790fa9afd92d4ec934e957604f195e45be90 Mon Sep 17 00:00:00 2001 From: Meng-Heng Date: Thu, 13 Jun 2024 12:19:56 +0700 Subject: [PATCH 21/64] docs(developer): context help in new-project section Fixes: #2131 --- developer/src/tike/xml/help/contexthelp.xml | 25 ++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/developer/src/tike/xml/help/contexthelp.xml b/developer/src/tike/xml/help/contexthelp.xml index 62bd8677ae..a7f66ed12b 100644 --- a/developer/src/tike/xml/help/contexthelp.xml +++ b/developer/src/tike/xml/help/contexthelp.xml @@ -710,4 +710,27 @@

    To close the dialog, click the Close button or press Shift + Esc.
    - \ No newline at end of file + + +
    + +

    + Creates a new Keyman Keyboard, LDML Keyboard, or a Wordlist Lexical Model, + or by importing from another source. Details are written on individual icons; clicking on + them shows a detailed explanation of each keyboard project. +

    +     + +

    + Once decided on a project to create, click OK will take you to the next step. +

    +     + +

    + Cancels creating a project. +

    +     +
    + + + From cf6665e2ce4d2ab19591bdba7cb1dde99fe9133f Mon Sep 17 00:00:00 2001 From: Meng-Heng Date: Thu, 13 Jun 2024 13:13:27 +0700 Subject: [PATCH 22/64] docs(developer): context help for new project parameters in keyman developer Fixes: #2131 --- developer/src/tike/xml/help/contexthelp.xml | 86 +++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/developer/src/tike/xml/help/contexthelp.xml b/developer/src/tike/xml/help/contexthelp.xml index 62bd8677ae..0c338aba98 100644 --- a/developer/src/tike/xml/help/contexthelp.xml +++ b/developer/src/tike/xml/help/contexthelp.xml @@ -710,4 +710,90 @@

    To close the dialog, click the Close button or press Shift + Esc.

    + + +
    + +

    The descriptive name of the keyboard. This will be set in the &Name store in the keyboard, + in the package name, and where appropriate in documentation and metadata.

    +     + +

    A brief or long description about the keyboard's functionality, background, script, language, community, + or any information related to showcase to users.

    +     + +

    The name of the developer of the keyboard. This is either your full name or + the organization you're creating the keyboard for.

    +     + +

    This field should contain the word "Copyright", the copyright symbol "©", and the full name of the rights + owner. Do not put the year of copyright in this field (see Full Copyright). Typically, + you can use the automatically generated default value: "Copyright © Your Full Name or + Your Organization".

    +

    A copyright string for the keyboard. This will be set in the &Copyright store in the keyboard, + in the package metadata, and where appropriate in documentation and metadata.

    +     + +

    Who owns the rights to this keyboard? This field should + contain the word "Copyright", the copyright symbol "©", the first year of copyright, + and the full name of the rights owner. Do include the year of copyright in this field. + Typically, you can use the automatically generated default value: + "Copyright © Current-Year Your-Full Name or Your Organization".

    +     + +

    If this is the first time you've created a keyboard for the language, you should + leave the version as 1.0. Otherwise, your version number must conform to the following + rules: A version string made of major revision number.minor revision number.

    +     + +

    + Specifies the default deployment targets for the keyboard, set in the &Targets store in the keyboard, + and controls the files added to the package initially. This also is reflected in documentation and metadata. +

    +     + +

    + Specifies the base path where the project folder will be created. The project folder name will be the keyboard ID. + If the folder already exists, then you will be prompted before Keyman Developer overwrites files inside it. +

    +     + +

    + Specifies a different path to store the project folder. +

    +     + +

    + The base filename of the keyboard, project and package. This must conform to the Keyman + keyboard identifier rules, using the characters a-z, 0-9 and _ (underscore) only. +

    +     + +

    + A Keyboard Project (.kpj) file is store inside the base path, this cannot be changed to anywhere else. +

    +     + +

    This button will open up the BCP 47 Tag window to add the keyboard's language tag, script tag, + region tag, and language name.

    +     + +

    + Click on Edit... to open up the BCP 47 Tag window with the language information. +

    +     + +

    + Delete the selected language. +

    +     + +

    + Once everything is ready, click OK and enjoy developing the keyboard. +

    +     + +

    Click Cancel to close the dialog and reset the process entirely.

    +     +
    \ No newline at end of file From abb2530ea51e2b6419aa5fa098ceda2432fe46e6 Mon Sep 17 00:00:00 2001 From: Meng-Heng Date: Thu, 13 Jun 2024 13:50:53 +0700 Subject: [PATCH 23/64] docs(developer): context help for Select BCP 47 tag in Keyman Developer Fixes: #2131 --- developer/src/tike/xml/help/contexthelp.xml | 43 +++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/developer/src/tike/xml/help/contexthelp.xml b/developer/src/tike/xml/help/contexthelp.xml index 62bd8677ae..aed5df5f02 100644 --- a/developer/src/tike/xml/help/contexthelp.xml +++ b/developer/src/tike/xml/help/contexthelp.xml @@ -710,4 +710,47 @@

    To close the dialog, click the Close button or press Shift + Esc.

    + + +
    + +

    The only required option is the Language tag, which is an ISO 639-1 or ISO 639-3 code. + ISO 639-1 tags are a two-letter code. ISO 639-3 tags are a three-letter code. First, try to find your + language on the list of two-letter ISO 639-1 codes.

    +

    If you can't find a two-letter code, you'll need to find the closest three-letter code. You can use Glottolog + to search for your language, and it will give you an appropriate code.

    +

    The Language tag is conventionally written in lower case.

    +     + +

    The Script tag allows you to specify the writing system used in your language model or keyboard. + If your language only uses one writing system, omit the Script subtag. he Script subtag is conventionally + written in title case - first letter capitalized.

    +     + +

    The Region tag allows you to specify the region your language or + dialect is spoken in. If your language is only spoken in one region, omit the Region subtag. + Alphabetic region will show up in upper case once the region tag is selected.

    +     + +

    This is the BCP 47 Code forms from the language, script, and region tag which were previously + selected.

    +     + +

    If lefts as default, a language name will be assigned automatically. Feel free to change the name + if you must. +

    +     + +

    Reset the Language name to what it was assigned to.

    +     + +

    Once everything is set, click OK to confirm.

    +     + +

    Exits out of the "Select BCP 47 Tag".

    +     + +

    Learn more about the BCP 47 Tag over at w3.org.

    +     +
    \ No newline at end of file From b93c45951536b49b66b03b6372990ddd474def2b Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Thu, 13 Jun 2024 17:49:46 +0200 Subject: [PATCH 24/64] fix(linux): restart ibus after manual integration test run Previously keyboard input was no longer input after manually running the integration tests. This change fixes this by restarting ibus after the tests. However, we only want to do that if the ibus instance we started for the tests got killed. The cleanup script might get called multiple times, so we need this check. --- linux/ibus-keyman/tests/scripts/test-helper.inc.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/linux/ibus-keyman/tests/scripts/test-helper.inc.sh b/linux/ibus-keyman/tests/scripts/test-helper.inc.sh index 01626a0490..5d895272f0 100755 --- a/linux/ibus-keyman/tests/scripts/test-helper.inc.sh +++ b/linux/ibus-keyman/tests/scripts/test-helper.inc.sh @@ -116,7 +116,7 @@ function _setup_init() { echo > "$CLEANUP_FILE" echo > "$PID_FILE" TEMP_DATA_DIR=$(mktemp --directory) - echo "rm -rf ${TEMP_DATA_DIR} || true" >> "$CLEANUP_FILE" + echo "rm -rf ${TEMP_DATA_DIR} || true # TEMP_DATA_DIR" >> "$CLEANUP_FILE" COMMON_ARCH_DIR= [ -d "${TOP_SRCDIR}"/../../core/build/arch ] && COMMON_ARCH_DIR=${TOP_SRCDIR}/../../core/build/arch @@ -170,7 +170,7 @@ function _setup_display_server() { # mutter-Message: 18:56:15.422: Using Wayland display name 'wayland-1' mutter --wayland --headless --no-x11 --virtual-monitor 1024x768 &> "$TMPFILE" & PID=$! - echo "kill -9 ${PID} || true" >> "$CLEANUP_FILE" + echo "kill -9 ${PID} || true # mutter" >> "$CLEANUP_FILE" echo "${PID} mutter" >> "${PID_FILE}" sleep 1s export WAYLAND_DISPLAY @@ -189,7 +189,7 @@ function _setup_display_server() { break fi done - echo "kill -9 ${PID} || true" >> "$CLEANUP_FILE" + echo "kill -9 ${PID} || true # Xvfb" >> "$CLEANUP_FILE" echo "${PID} Xvfb" >> "${PID_FILE}" while true; do echo "Starting Xephyr..." @@ -201,12 +201,12 @@ function _setup_display_server() { break fi done - echo "kill -9 ${PID} || true" >> "$CLEANUP_FILE" + echo "kill -9 ${PID} || true # Xephyr" >> "$CLEANUP_FILE" echo "${PID} Xephyr" >> "${PID_FILE}" echo "Starting metacity" metacity --display=:${DISP_XEPHYR} &> /dev/null & PID=$! - echo "kill -9 ${PID} || true" >> "$CLEANUP_FILE" + echo "kill -9 ${PID} || true # metacity" >> "$CLEANUP_FILE" echo "${PID} metacity" >> "${PID_FILE}" export DISPLAY=:${DISP_XEPHYR} @@ -250,7 +250,7 @@ function _setup_ibus() { #shellcheck disable=SC2086 ibus-daemon ${ARG_VERBOSE-} --daemonize --panel=disable --address=unix:abstract="${TEMP_DATA_DIR}/test-ibus" ${IBUS_CONFIG-} &> /tmp/ibus-daemon.log PID=$(pgrep -f "${TEMP_DATA_DIR}/test-ibus") - echo "kill -9 ${PID} || true" >> "$CLEANUP_FILE" + echo "if kill -9 ${PID}; then ibus restart || ibus start; fi || true # ibus-daemon" >> "$CLEANUP_FILE" echo "${PID} ibus-daemon" >> "${PID_FILE}" sleep 1s @@ -263,7 +263,7 @@ function _setup_ibus() { #shellcheck disable=SC2086 "${TOP_BINDIR}/src/ibus-engine-keyman" --testing ${ARG_VERBOSE-} &> /tmp/ibus-engine-keyman.log & PID=$! - echo "kill -9 ${PID} || true" >> "$CLEANUP_FILE" + echo "kill -9 ${PID} || true # ibus-engine-keyman" >> "${CLEANUP_FILE}" echo "${PID} ibus-engine-keyman" >> "${PID_FILE}" sleep 1s } From e155ae9c0c5a84a2264135dc3a505849f627fdc2 Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Thu, 13 Jun 2024 16:31:41 +0100 Subject: [PATCH 25/64] fix(developer): add gcc flag to remove __cdecl --- developer/src/kmcmplib/src/meson.build | 1 + 1 file changed, 1 insertion(+) diff --git a/developer/src/kmcmplib/src/meson.build b/developer/src/kmcmplib/src/meson.build index ce9c27b488..b5f13dc991 100644 --- a/developer/src/kmcmplib/src/meson.build +++ b/developer/src/kmcmplib/src/meson.build @@ -14,6 +14,7 @@ if cpp_compiler.get_id() == 'gcc' or cpp_compiler.get_id() == 'clang' '-Wall', '-Wextra' ] + flags += ['-D__cdecl= '] endif if cpp_compiler.get_id() == 'msvc' From 328b7b70c36c32453cc2a671aa69b8ac19ed72ab Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Thu, 13 Jun 2024 16:59:47 +0100 Subject: [PATCH 26/64] fix(developer): add include to kmcompx.h --- developer/src/kmcmplib/include/kmcompx.h | 1 + 1 file changed, 1 insertion(+) diff --git a/developer/src/kmcmplib/include/kmcompx.h b/developer/src/kmcmplib/include/kmcompx.h index a1403ad02f..7e7b270496 100644 --- a/developer/src/kmcmplib/include/kmcompx.h +++ b/developer/src/kmcmplib/include/kmcompx.h @@ -29,6 +29,7 @@ typedef KMX_WCHAR* PKMX_WCHAR ; #ifndef _MSC_VER #include +#include template < typename T, size_t N > size_t _countof( T ( & /*arr*/ )[ N ] ) From 8a86cc373f7b609762b57da520905a46ddf7880d Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Thu, 13 Jun 2024 17:06:36 +0100 Subject: [PATCH 27/64] fix(developer): add include to kmcompxtest.cpp --- developer/src/kmcmplib/tests/kmcompxtest.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/developer/src/kmcmplib/tests/kmcompxtest.cpp b/developer/src/kmcmplib/tests/kmcompxtest.cpp index 454481ff5e..4cf3879257 100644 --- a/developer/src/kmcmplib/tests/kmcompxtest.cpp +++ b/developer/src/kmcmplib/tests/kmcompxtest.cpp @@ -19,6 +19,7 @@ #ifdef _MSC_VER #else #include +#include #endif using namespace std; From 604d538875d67fe1e79edfc464fc544ba5b7c8c6 Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Thu, 13 Jun 2024 17:10:59 +0100 Subject: [PATCH 28/64] fix(developer): add include to kmx_u16.h --- developer/src/kmcmplib/src/kmx_u16.h | 1 + 1 file changed, 1 insertion(+) diff --git a/developer/src/kmcmplib/src/kmx_u16.h b/developer/src/kmcmplib/src/kmx_u16.h index 6a54c0f00d..450c400e2f 100644 --- a/developer/src/kmcmplib/src/kmx_u16.h +++ b/developer/src/kmcmplib/src/kmx_u16.h @@ -4,6 +4,7 @@ #include #include #include +#include #include "kmcompx.h" std::string string_from_wstring(std::wstring const str); From 6585d034c6993097cfc3bddc461d697f1e525711 Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Thu, 13 Jun 2024 17:25:31 +0100 Subject: [PATCH 29/64] fix(developer): add include and std::reverse to kmx_u16.cpp --- developer/src/kmcmplib/src/kmx_u16.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/developer/src/kmcmplib/src/kmx_u16.cpp b/developer/src/kmcmplib/src/kmx_u16.cpp index c73796e9b6..174109e3b2 100644 --- a/developer/src/kmcmplib/src/kmx_u16.cpp +++ b/developer/src/kmcmplib/src/kmx_u16.cpp @@ -10,6 +10,11 @@ #include #include +#ifdef _MSC_VER +#else +#include +#endif + //String <- wstring std::string string_from_wstring(std::wstring const str) { std::wstring_convert, wchar_t> converter; @@ -96,7 +101,11 @@ std::string toHex(int num1) { s += (87 + temp); num = num / 16; } +#ifdef _MSC_VER reverse(s.begin(), s.end()); +#else + std::reverse(s.begin(), s.end()); +#endif return s; } From 421593bd726dafae270664c7336570a163879f09 Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Thu, 13 Jun 2024 17:33:20 +0100 Subject: [PATCH 30/64] fix(developer): rename shadowing local variables --- developer/src/kmcmplib/src/Compiler.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/developer/src/kmcmplib/src/Compiler.cpp b/developer/src/kmcmplib/src/Compiler.cpp index 8d0f11af8f..8d799097fc 100644 --- a/developer/src/kmcmplib/src/Compiler.cpp +++ b/developer/src/kmcmplib/src/Compiler.cpp @@ -1191,11 +1191,11 @@ int GetCompileTargetsFromTargetsStore(const KMX_WCHAR* store) { if(AnyTarget == token) { result |= COMPILETARGETS_KMX | COMPILETARGETS_JS; } - for(auto p: KMXKeymanTargets) { - if(p == token) result |= COMPILETARGETS_KMX; + for(auto target: KMXKeymanTargets) { + if(target == token) result |= COMPILETARGETS_KMX; } - for(auto p: KMWKeymanTargets) { - if(p == token) result |= COMPILETARGETS_JS; + for(auto target: KMWKeymanTargets) { + if(target == token) result |= COMPILETARGETS_JS; } token = u16tok(nullptr, u" ", &ctx); From 225fd8d29a84fc937c621616de2c5eee744a3608 Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Thu, 13 Jun 2024 17:36:48 +0100 Subject: [PATCH 31/64] fix(developer): remove ignored printf flag --- developer/src/kmcmplib/tests/util_callbacks.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/developer/src/kmcmplib/tests/util_callbacks.cpp b/developer/src/kmcmplib/tests/util_callbacks.cpp index e610a54c09..5d265fc906 100644 --- a/developer/src/kmcmplib/tests/util_callbacks.cpp +++ b/developer/src/kmcmplib/tests/util_callbacks.cpp @@ -16,7 +16,7 @@ int msgproc(int line, uint32_t dwMsgCode, const char* szText, void* context) { case CERR_ERROR: t=" error"; break; case CERR_FATAL: t=" fatal"; break; } - printf("line %d %s %04.4x: %s\n", line, t, (unsigned int)dwMsgCode, szText); + printf("line %d %s %4.4x: %s\n", line, t, (unsigned int)dwMsgCode, szText); return 1; } @@ -56,4 +56,4 @@ bool loadfileProc(const char* filename, const char* baseFilename, void* data, in } fclose(fp); return true; -} \ No newline at end of file +} From 7c0a422095d9b75d9bb21cacaf98e24db1a0d86d Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Thu, 13 Jun 2024 17:41:55 +0100 Subject: [PATCH 32/64] fix(developer): correct type of loop iterator --- developer/src/kmcmplib/src/CompileKeyboardBuffer.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/developer/src/kmcmplib/src/CompileKeyboardBuffer.cpp b/developer/src/kmcmplib/src/CompileKeyboardBuffer.cpp index 1b4b23bd2e..b889a4fd7e 100644 --- a/developer/src/kmcmplib/src/CompileKeyboardBuffer.cpp +++ b/developer/src/kmcmplib/src/CompileKeyboardBuffer.cpp @@ -174,7 +174,7 @@ namespace kmcmp { void CopyExtraData(PFILE_KEYBOARD fk) { /* Copy stores */ PFILE_STORE store = fk->dpStoreArray; - for(int i = 0; i < fk->cxStoreArray; i++, store++) { + for(KMX_DWORD i = 0; i < fk->cxStoreArray; i++, store++) { KMCMP_COMPILER_RESULT_EXTRA_STORE extraStore; extraStore.storeType = (store->fIsStore ? STORETYPE_STORE : 0) | @@ -188,11 +188,11 @@ namespace kmcmp { } PFILE_GROUP group = fk->dpGroupArray; - for(int i = 0; i < fk->cxGroupArray; i++, group++) { + for(KMX_DWORD i = 0; i < fk->cxGroupArray; i++, group++) { KMCMP_COMPILER_RESULT_EXTRA_GROUP extraGroup; extraGroup.isReadOnly = group->fReadOnly; extraGroup.name = string_from_u16string(group->szName); fk->extra->groups.push_back(extraGroup); } } -} \ No newline at end of file +} From 263d0e32f2d43ebe2f551fd3deea4e04bbc8aae3 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 13 Jun 2024 12:17:55 -0500 Subject: [PATCH 33/64] chore(core): update comments and remove a raw numeric literal - per review comments Fixes: #9467 Co-authored-by: rc-swag <58423624+rc-swag@users.noreply.github.com> --- core/src/util_normalize.cpp | 6 ++++-- core/src/util_normalize_table_generator.cpp | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/core/src/util_normalize.cpp b/core/src/util_normalize.cpp index 947dbe0f85..bcdfc1bb3f 100644 --- a/core/src/util_normalize.cpp +++ b/core/src/util_normalize.cpp @@ -228,8 +228,10 @@ bool has_nfd_boundary_before(km_core_usv cp) { } /** - * Helper to convert icu::UnicodeString to a UTF-32 km_core_usv buffer, - * nul-terminated + * Helper to convert std::u32string to a UTF-32 km_core_usv buffer, + * nul-terminated. + * Parallel to unicode_string_to_usv() + * @returns new buffer, caller owns storage */ km_core_usv *string_to_usv(const std::u32string& src) { return km::core::kmx::u32dup(src.c_str()); diff --git a/core/src/util_normalize_table_generator.cpp b/core/src/util_normalize_table_generator.cpp index b5895a588b..994f16d8b3 100644 --- a/core/src/util_normalize_table_generator.cpp +++ b/core/src/util_normalize_table_generator.cpp @@ -55,7 +55,7 @@ write_nfd_table() { // collect the raw list of chars that do NOT have a boundary before them. std::vector noBoundary; - for (km_core_usv ch = 0; ch < 0x10FFFF; ch++) { + for (km_core_usv ch = 0; ch < km::core::kmx::Uni_MAX_CODEPOINT; ch++) { bool bb = nfd->hasBoundaryBefore(ch); assert(!(ch == 0 && !bb)); // assert that we can use U+0000 as a terminator if (bb) continue; //only emit nonboundary From 4ed7af0c6ca9ad4e84dd1aee8cb4c0a46cc73eed Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Thu, 13 Jun 2024 18:17:58 +0100 Subject: [PATCH 34/64] fix(developer): cast to int for fread size check --- developer/src/kmcmplib/tests/util_callbacks.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer/src/kmcmplib/tests/util_callbacks.cpp b/developer/src/kmcmplib/tests/util_callbacks.cpp index 5d265fc906..6b0fafe35f 100644 --- a/developer/src/kmcmplib/tests/util_callbacks.cpp +++ b/developer/src/kmcmplib/tests/util_callbacks.cpp @@ -49,7 +49,7 @@ bool loadfileProc(const char* filename, const char* baseFilename, void* data, in } } else { // return data - if(fread(data, 1, *size, fp) != *size) { + if((int)fread(data, 1, *size, fp) != *size) { fclose(fp); return false; } From fe05d058583417341cf8f4b9ec3c4b0ac3589abb Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Thu, 13 Jun 2024 18:22:39 +0100 Subject: [PATCH 35/64] fix(developer): cast to long for ftell size check --- developer/src/kmcmplib/tests/kmcompxtest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer/src/kmcmplib/tests/kmcompxtest.cpp b/developer/src/kmcmplib/tests/kmcompxtest.cpp index 4cf3879257..ecfa0d60f2 100644 --- a/developer/src/kmcmplib/tests/kmcompxtest.cpp +++ b/developer/src/kmcmplib/tests/kmcompxtest.cpp @@ -86,7 +86,7 @@ int main(int argc, char *argv[]) fseek(fp2, 0, SEEK_END); auto sz2 = ftell(fp2); fseek(fp2, 0, SEEK_SET); - if (result.kmxSize != sz2) return __LINE__; // exit code: size of kmx-file in build differs from size of kmx-file in source folder + if ((long)result.kmxSize != sz2) return __LINE__; // exit code: size of kmx-file in build differs from size of kmx-file in source folder char* buf2 = new char[result.kmxSize]; fread(buf2, 1, result.kmxSize, fp2); From f0701c87cd80242f48844394a61af4987c4993e3 Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Thu, 13 Jun 2024 18:25:35 +0100 Subject: [PATCH 36/64] fix(developer): correct iterator type --- developer/src/kmcmplib/tests/kmcompxtest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer/src/kmcmplib/tests/kmcompxtest.cpp b/developer/src/kmcmplib/tests/kmcompxtest.cpp index ecfa0d60f2..c5e01e012f 100644 --- a/developer/src/kmcmplib/tests/kmcompxtest.cpp +++ b/developer/src/kmcmplib/tests/kmcompxtest.cpp @@ -103,7 +103,7 @@ int main(int argc, char *argv[]) std::istringstream(ErrNr) >> std::hex >> error_val; // check if error_val is in Array of Errors; if it is found return 0 (it's not an error) - for (int i = 0; i < error_vec.size() ; i++) { + for (std::vector::size_type i = 0; i < error_vec.size() ; i++) { if (error_vec[i] == error_val) { return 0; // success: CERR_ in Name + Error (specified in CERR_Name) IS found } From 46186c90e561be605a873017e7352578e64a8344 Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Thu, 13 Jun 2024 18:27:13 +0100 Subject: [PATCH 37/64] fix(developer): correct iterator type to KMX_DWORD --- developer/src/kmcmplib/tests/kmcompxtest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer/src/kmcmplib/tests/kmcompxtest.cpp b/developer/src/kmcmplib/tests/kmcompxtest.cpp index c5e01e012f..be9eb46b76 100644 --- a/developer/src/kmcmplib/tests/kmcompxtest.cpp +++ b/developer/src/kmcmplib/tests/kmcompxtest.cpp @@ -157,7 +157,7 @@ bool isDesktopKeyboard(FILE* fp) { PCOMP_STORE s = pfs; - for(int i = 0; i < fk.cxStoreArray; i++, s++) { + for(KMX_DWORD i = 0; i < fk.cxStoreArray; i++, s++) { if(s->dwSystemID != TSS_TARGETS) { continue; } From 272b3b153292f22c458f84db21a6264b11c95c9c Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Thu, 13 Jun 2024 18:31:22 +0100 Subject: [PATCH 38/64] fix(developer) cast to int in mixed int type expression --- developer/src/kmcmplib/src/Compiler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer/src/kmcmplib/src/Compiler.cpp b/developer/src/kmcmplib/src/Compiler.cpp index 8d799097fc..6534ebbfaf 100644 --- a/developer/src/kmcmplib/src/Compiler.cpp +++ b/developer/src/kmcmplib/src/Compiler.cpp @@ -806,7 +806,7 @@ bool resizeStoreArray(PFILE_KEYBOARD fk) { * reallocates the key array in increments of 100 */ bool resizeKeyArray(PFILE_GROUP gp, int increment) { - if((gp->cxKeyArray + increment - 1) % 100 < increment) { + if(((int)gp->cxKeyArray + increment - 1) % 100 < increment) { PFILE_KEY kp = new FILE_KEY[((gp->cxKeyArray + increment)/100 + 1) * 100]; if (!kp) return false; if (gp->dpKeyArray) From 1fa13c2b836cf875940c36811df8ea7fc445a737 Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Thu, 13 Jun 2024 18:41:07 +0100 Subject: [PATCH 39/64] fix(developer): add -Wno-ignored-qualifiers flag to allow casting to override warnings with return type casts --- developer/src/kmcmplib/src/meson.build | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/developer/src/kmcmplib/src/meson.build b/developer/src/kmcmplib/src/meson.build index b5f13dc991..16af213dc3 100644 --- a/developer/src/kmcmplib/src/meson.build +++ b/developer/src/kmcmplib/src/meson.build @@ -12,7 +12,8 @@ lib_links = [] if cpp_compiler.get_id() == 'gcc' or cpp_compiler.get_id() == 'clang' warns += [ '-Wall', - '-Wextra' + '-Wextra', + '-Wno-ignored-qualifiers' ] flags += ['-D__cdecl= '] endif From 1a61a589b35c5fee4c9ad5d967668163350203f8 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 13 Jun 2024 13:07:29 -0500 Subject: [PATCH 40/64] feat(core): devolve regex to js for wasm - assert was wrong! Fixes: #9467 --- core/src/util_regex.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/util_regex.cpp b/core/src/util_regex.cpp index 3943c87360..55bcb04e6d 100644 --- a/core/src/util_regex.cpp +++ b/core/src/util_regex.cpp @@ -203,7 +203,7 @@ size_t km_regex::apply(const std::u32string &input, std::u32string &output, const std::u32string match32 = convert(group1str); free(group1); auto matchIndex = findIndex(match32, fromList); - assert(matchIndex != 1L); + assert(matchIndex != -1); rustr = toList.at(matchIndex); } std::string rstr = convert(rustr); From 781a46341dacc4093191a368b57a49af712046f2 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Thu, 13 Jun 2024 14:09:31 -0400 Subject: [PATCH 41/64] auto: increment master version to 18.0.56 --- HISTORY.md | 6 ++++++ VERSION.md | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index b5d6ea2caa..c5e1f2e780 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,11 @@ # Keyman Version History +## 18.0.55 alpha 2024-06-13 + +* fix(developer): handle missing OSK when importing a Windows keyboard into a touch-only project (#11720) +* fix(developer): verify email addresses in .kps and .keyboard_info (#11735) +* change(web): prep for better asynchronous prediction handling (#10343) + ## 18.0.54 alpha 2024-06-12 * fix(common): remove subpackage entries for older TS version (#11745) diff --git a/VERSION.md b/VERSION.md index 1d6c319a44..b38400af1f 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -18.0.55 \ No newline at end of file +18.0.56 \ No newline at end of file From f1e3396249c862fad3be0063f7a9d0dd8e257106 Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Thu, 13 Jun 2024 20:10:16 +0100 Subject: [PATCH 42/64] fix(developer): add additional error if build cannot be read into buffer --- developer/src/kmcmplib/tests/kmcompxtest.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/developer/src/kmcmplib/tests/kmcompxtest.cpp b/developer/src/kmcmplib/tests/kmcompxtest.cpp index be9eb46b76..aa79481116 100644 --- a/developer/src/kmcmplib/tests/kmcompxtest.cpp +++ b/developer/src/kmcmplib/tests/kmcompxtest.cpp @@ -89,7 +89,8 @@ int main(int argc, char *argv[]) if ((long)result.kmxSize != sz2) return __LINE__; // exit code: size of kmx-file in build differs from size of kmx-file in source folder char* buf2 = new char[result.kmxSize]; - fread(buf2, 1, result.kmxSize, fp2); + auto sz3 = fread(buf2, 1, result.kmxSize, fp2); + if (result.kmxSize != sz3) return __LINE__; // exit code: when not able to read the build into the buffer return memcmp(result.kmx, buf2, result.kmxSize) ? __LINE__ : 0; // exit code: when contents of kmx-file in build differs from contents of kmx-file in source folder // success: when contents of kmx-file in build and source folder are the same } From 6a16837b470302c5d9f068e8493a3fc314965dbc Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Thu, 13 Jun 2024 20:17:36 +0100 Subject: [PATCH 43/64] fix(developer): catch error by reference --- developer/src/kmcmplib/src/Compiler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer/src/kmcmplib/src/Compiler.cpp b/developer/src/kmcmplib/src/Compiler.cpp index 6534ebbfaf..3da3fecacd 100644 --- a/developer/src/kmcmplib/src/Compiler.cpp +++ b/developer/src/kmcmplib/src/Compiler.cpp @@ -3516,7 +3516,7 @@ bool UTF16TempFromUTF8(KMX_BYTE* infile, int sz, KMX_BYTE** tempfile, int *sz16) try { std::wstring_convert, char16_t> converter; result = converter.from_bytes((char*)infile, (char*)infile+sz); - } catch(std::range_error e) { + } catch(std::range_error &e) { AddCompileError(CHINT_NonUnicodeFile); result.resize(sz); for(int i = 0; i < sz; i++) { From a744afc34ff9ff31b8ccd68e717606c6a500366c Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 13 Jun 2024 14:20:58 -0500 Subject: [PATCH 44/64] feat(core): remove ICU from core under wasm Fixes: #9467 --- core/src/ldml/ldml_markers.hpp | 4 ---- core/src/meson.build | 18 +++++++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/core/src/ldml/ldml_markers.hpp b/core/src/ldml/ldml_markers.hpp index cfd98d1b15..bbec2f23cc 100644 --- a/core/src/ldml/ldml_markers.hpp +++ b/core/src/ldml/ldml_markers.hpp @@ -17,10 +17,6 @@ #include "debuglog.h" #include "core_icu.h" -#include "unicode/uniset.h" -#include "unicode/usetiter.h" -#include "unicode/regex.h" -#include "unicode/utext.h" namespace km { namespace core { diff --git a/core/src/meson.build b/core/src/meson.build index b7b1ae78f5..212b56f1c2 100644 --- a/core/src/meson.build +++ b/core/src/meson.build @@ -16,11 +16,6 @@ if cpp_compiler.get_id() == 'msvc' version_res += import('windows').compile_resources('version.rc', args:['/n','/c65001']) endif -if cpp_compiler.get_id() == 'emscripten' - # TODO: why do we need this defn here? - defns += ['-DKM_CORE_LIBRARY'] -endif - # ICU4C is used for repertoire tests and core implementation if target_machine.system() == 'linux' @@ -36,6 +31,15 @@ else icu_i18n = icu4c.get_variable('icui18n_dep') endif +if cpp_compiler.get_id() == 'emscripten' + # TODO: why do we need this defn here? + defns += ['-DKM_CORE_LIBRARY'] + icu_if_not_on_wasm = [] +else + # only include this if NOT on wasm. + icu_if_not_on_wasm = [icu_uc, icu_i18n] +endif + if icu_uc.found() defns += '-DHAVE_ICU4C' endif @@ -142,12 +146,12 @@ lib = library('keymancore', include_directories: inc, pic: true, install: true, - dependencies: [icu_uc, icu_i18n], + dependencies: icu_if_not_on_wasm, ) headerdirs = [ '.', 'keyman' ] # subdirectories of ${prefix}/include to add to header path -keymancore = declare_dependency(link_with: lib, include_directories: inc, dependencies: [icu_uc, icu_i18n]) +keymancore = declare_dependency(link_with: lib, include_directories: inc, dependencies: icu_if_not_on_wasm) pkg = import('pkgconfig') pkg.generate( From 446661ae0371f798b605cdb79fc888c956b97e74 Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Thu, 13 Jun 2024 20:25:08 +0100 Subject: [PATCH 45/64] fix(developer): make explicit cast in mixed int type comparison --- developer/src/kmcmplib/src/NamedCodeConstants.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer/src/kmcmplib/src/NamedCodeConstants.cpp b/developer/src/kmcmplib/src/NamedCodeConstants.cpp index 35273542eb..897faa98d0 100644 --- a/developer/src/kmcmplib/src/NamedCodeConstants.cpp +++ b/developer/src/kmcmplib/src/NamedCodeConstants.cpp @@ -268,7 +268,7 @@ int IsHangulSyllable(const KMX_WCHAR *codename, int *code) if(strchr("GNDRMBSJCKTPH", ch)) { /* Has an initial syllable */ - int isDoubled = towupper(*(codename+1)) == ch; + int isDoubled = (int)towupper(*(codename+1)) == ch; LIndex = -1; for(i = 0; i < HangulLCount; i++) { From 3453319563f078947e58624a10d084328f496cdc Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Thu, 13 Jun 2024 20:35:07 +0100 Subject: [PATCH 46/64] fix(developer): correct test that is always false --- developer/src/kmcmplib/src/kmx_u16.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer/src/kmcmplib/src/kmx_u16.cpp b/developer/src/kmcmplib/src/kmx_u16.cpp index 174109e3b2..c99a27b201 100644 --- a/developer/src/kmcmplib/src/kmx_u16.cpp +++ b/developer/src/kmcmplib/src/kmx_u16.cpp @@ -285,7 +285,7 @@ double u16tof( KMX_WCHAR* str) PKMX_WCHAR q = (PKMX_WCHAR)u16chr(str, '.'); size_t pos_dot = q-str ; - if (pos_dot < 0) + if (q-str < 0) pos_dot = u16len(str); for (size_t i = 0; i < u16len(str); i++) From ecd4944a2bb6b0ce20d0b88e5bcb0153b97ad9a4 Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Thu, 13 Jun 2024 20:50:24 +0100 Subject: [PATCH 47/64] fix(developer): add terminating null to expected strings --- developer/src/kmcmplib/tests/gtest-compiler-test.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/developer/src/kmcmplib/tests/gtest-compiler-test.cpp b/developer/src/kmcmplib/tests/gtest-compiler-test.cpp index 61d6c03aa9..59f385b5c0 100644 --- a/developer/src/kmcmplib/tests/gtest-compiler-test.cpp +++ b/developer/src/kmcmplib/tests/gtest-compiler-test.cpp @@ -223,13 +223,13 @@ TEST_F(CompilerTest, ProcessBeginLine_test) { TEST_F(CompilerTest, ValidateMatchNomatchOutput_test) { EXPECT_EQ(CERR_None, ValidateMatchNomatchOutput(NULL)); EXPECT_EQ(CERR_None, ValidateMatchNomatchOutput((PKMX_WCHAR)u"")); - const KMX_WCHAR context[] = { 'a', 'b', 'c', UC_SENTINEL, CODE_CONTEXT, 'd', 'e', 'f' }; + const KMX_WCHAR context[] = { 'a', 'b', 'c', UC_SENTINEL, CODE_CONTEXT, 'd', 'e', 'f', 0 }; EXPECT_EQ(CERR_ContextAndIndexInvalidInMatchNomatch, ValidateMatchNomatchOutput((PKMX_WCHAR)context)); - const KMX_WCHAR contextex[] = { 'a', 'b', 'c', UC_SENTINEL, CODE_CONTEXTEX, 'd', 'e', 'f' }; + const KMX_WCHAR contextex[] = { 'a', 'b', 'c', UC_SENTINEL, CODE_CONTEXTEX, 'd', 'e', 'f', 0 }; EXPECT_EQ(CERR_ContextAndIndexInvalidInMatchNomatch, ValidateMatchNomatchOutput((PKMX_WCHAR)contextex)); - const KMX_WCHAR index[] = { 'a', 'b', 'c', UC_SENTINEL, CODE_INDEX, 'd', 'e', 'f' }; + const KMX_WCHAR index[] = { 'a', 'b', 'c', UC_SENTINEL, CODE_INDEX, 'd', 'e', 'f', 0 }; EXPECT_EQ(CERR_ContextAndIndexInvalidInMatchNomatch, ValidateMatchNomatchOutput((PKMX_WCHAR)index)); - const KMX_WCHAR sentinel[] = { 'a', 'b', 'c', UC_SENTINEL, 'd', 'e', 'f' }; + const KMX_WCHAR sentinel[] = { 'a', 'b', 'c', UC_SENTINEL, 'd', 'e', 'f', 0 }; EXPECT_EQ(CERR_None, ValidateMatchNomatchOutput((PKMX_WCHAR)sentinel)); }; From 0b8347b06748bd682a2a3902852d962142f99ba6 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 14 Jun 2024 10:06:10 +0700 Subject: [PATCH 48/64] fix(web): fix id of longpress keys with modifier set in touch layout Fixes: #11782 --- web/src/engine/osk/src/input/gestures/browser/oskSubKey.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/web/src/engine/osk/src/input/gestures/browser/oskSubKey.ts b/web/src/engine/osk/src/input/gestures/browser/oskSubKey.ts index abf7116af7..7640c3b20e 100644 --- a/web/src/engine/osk/src/input/gestures/browser/oskSubKey.ts +++ b/web/src/engine/osk/src/input/gestures/browser/oskSubKey.ts @@ -16,8 +16,7 @@ export default class OSKSubKey extends OSKKey { } getId(): string { - // Create (temporarily) unique ID by prefixing 'popup-' to actual key ID - return 'popup-'+this.layer+'-'+this.spec['id']; + return 'popup-'+this.spec.elementID; } construct(osk: VisualKeyboard, baseKey: KeyElement, width: number, topMargin: boolean): HTMLDivElement { From b1c908dee93e022bedae044fd6949a5eab9491ac Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 14 Jun 2024 13:53:51 +0700 Subject: [PATCH 49/64] fix(web): prevent desktop OSK crash when addKeyboards is called before engine init Fixes: #11785 Fixes: KEYMAN-WEB-KC --- .../engine/osk/src/views/floatingOskView.ts | 13 +++ web/src/test/manual/web/index.html | 1 + web/src/test/manual/web/issue11785/index.html | 109 ++++++++++++++++++ 3 files changed, 123 insertions(+) create mode 100644 web/src/test/manual/web/issue11785/index.html diff --git a/web/src/engine/osk/src/views/floatingOskView.ts b/web/src/engine/osk/src/views/floatingOskView.ts index 11b26073c4..ec1bc0877a 100644 --- a/web/src/engine/osk/src/views/floatingOskView.ts +++ b/web/src/engine/osk/src/views/floatingOskView.ts @@ -51,6 +51,7 @@ export default class FloatingOSKView extends OSKView { // Add header element to OSK only for desktop browsers this.titleBar = new TitleBar(this.titleDragHandler); + // this.titleBar.on('help', () => { this.legacyEvents.callEvent('helpclick', {}); }); @@ -91,6 +92,12 @@ export default class FloatingOSKView extends OSKView { listenerSpy.on('listenerremoved', onListenedEvent); } + if(this.activeKeyboard) { + // If the keyboard was loaded during OSK init, we may need to set the + // title in place now, as it wasn't possible at the standard time. + this.postKeyboardAdjustments(); + } + this.loadPersistedLayout(); } @@ -119,6 +126,12 @@ export default class FloatingOSKView extends OSKView { } protected postKeyboardAdjustments() { + // It is possible for this to be called during OSK initialization, + // when `this.titleBar` has not yet been initialized. + if(!this.titleBar) { + return; + } + // Add header element to OSK only for desktop browsers this.enableMoveResizeHandlers(); if(this.activeKeyboard) { diff --git a/web/src/test/manual/web/index.html b/web/src/test/manual/web/index.html index 218eaa3a45..8ee99a5ac3 100644 --- a/web/src/test/manual/web/index.html +++ b/web/src/test/manual/web/index.html @@ -75,6 +75,7 @@

    Test text selection (#9073)

    Test key-cap scaling / font load interactions (#10506)

    Test page interaction + engine-initialization race condition handling (#10743)

    +

    Test OSK loading with early add-keyboard calls (#11785)

    Other

    Keystroke processing regression test engine.

    diff --git a/web/src/test/manual/web/issue11785/index.html b/web/src/test/manual/web/issue11785/index.html new file mode 100644 index 0000000000..bd82343ca9 --- /dev/null +++ b/web/src/test/manual/web/issue11785/index.html @@ -0,0 +1,109 @@ + + + + + + + + + + + + + + + KeymanWeb Test Page - Keyboard Quick-Load + + + + + + + + + + + + + + + + + + + + +

    KeymanWeb Test Page - Keyboard Quick-Load

    + +
    + +
    + +

    Type in your language in this text area:

    + + +

    or in this input field:

    + + + +

    Add a keyboard by keyboard name:

    + + + +

    Add a keyboard by BCP-47 language code:

    + + + +

    Add a keyboard by language name(s):

    + + + +

    Return to testing home page

    +
    + + +
    +

    --End of Document--

    + + + + + From 966ea478c75ec5016ef2e118b9cd8513b4189636 Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Fri, 14 Jun 2024 10:09:36 +0100 Subject: [PATCH 50/64] fix(developer): change GetCompilerErrorString to return const --- developer/src/kmcmplib/src/CompMsg.cpp | 2 +- developer/src/kmcmplib/src/CompMsg.h | 2 +- developer/src/kmcmplib/src/Compiler.cpp | 2 +- developer/src/kmcmplib/tests/gtest-compmsg-test.cpp | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/developer/src/kmcmplib/src/CompMsg.cpp b/developer/src/kmcmplib/src/CompMsg.cpp index 6480458fda..132631090a 100644 --- a/developer/src/kmcmplib/src/CompMsg.cpp +++ b/developer/src/kmcmplib/src/CompMsg.cpp @@ -148,6 +148,6 @@ std::map CompilerErrorMap = { { 0, nullptr } }; -KMX_CHAR *GetCompilerErrorString(KMX_DWORD code) { +const KMX_CHAR *GetCompilerErrorString(KMX_DWORD code) { return (KMX_CHAR*) CompilerErrorMap[code]; } diff --git a/developer/src/kmcmplib/src/CompMsg.h b/developer/src/kmcmplib/src/CompMsg.h index 5517ea8811..e9a15f82fe 100644 --- a/developer/src/kmcmplib/src/CompMsg.h +++ b/developer/src/kmcmplib/src/CompMsg.h @@ -2,4 +2,4 @@ #include "km_types.h" #include -KMX_CHAR *GetCompilerErrorString(KMX_DWORD code) ; +const KMX_CHAR *GetCompilerErrorString(KMX_DWORD code) ; diff --git a/developer/src/kmcmplib/src/Compiler.cpp b/developer/src/kmcmplib/src/Compiler.cpp index 8d0f11af8f..4125e993d2 100644 --- a/developer/src/kmcmplib/src/Compiler.cpp +++ b/developer/src/kmcmplib/src/Compiler.cpp @@ -276,7 +276,7 @@ KMX_BOOL kmcmp::AddCompileWarning(PKMX_CHAR buf) KMX_BOOL AddCompileError(KMX_DWORD msg) { KMX_CHAR szText[COMPILE_ERROR_MAX_LEN]; - KMX_CHAR* szTextp = NULL; + const KMX_CHAR* szTextp = NULL; if (msg & CERR_FATAL) { diff --git a/developer/src/kmcmplib/tests/gtest-compmsg-test.cpp b/developer/src/kmcmplib/tests/gtest-compmsg-test.cpp index 94f0f40d8d..afcf6e86b3 100644 --- a/developer/src/kmcmplib/tests/gtest-compmsg-test.cpp +++ b/developer/src/kmcmplib/tests/gtest-compmsg-test.cpp @@ -2,7 +2,7 @@ #include "..\..\common\include\kmn_compiler_errors.h" #include "..\..\..\..\common\include\km_types.h" -KMX_CHAR *GetCompilerErrorString(KMX_DWORD code); +const KMX_CHAR *GetCompilerErrorString(KMX_DWORD code); class CompMsgTest : public testing::Test { protected: From 8c9813c4888ea4fd50e8232126530db0947f9020 Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Fri, 14 Jun 2024 10:18:04 +0100 Subject: [PATCH 51/64] fix(developer): remove unnecessary cast in GetCompilerErrorString return --- developer/src/kmcmplib/src/CompMsg.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer/src/kmcmplib/src/CompMsg.cpp b/developer/src/kmcmplib/src/CompMsg.cpp index 132631090a..70ea4d7b81 100644 --- a/developer/src/kmcmplib/src/CompMsg.cpp +++ b/developer/src/kmcmplib/src/CompMsg.cpp @@ -149,5 +149,5 @@ std::map CompilerErrorMap = { }; const KMX_CHAR *GetCompilerErrorString(KMX_DWORD code) { - return (KMX_CHAR*) CompilerErrorMap[code]; + return CompilerErrorMap[code]; } From 20c5b7938a18e215d2ab5a4b87092d0c6da8fa10 Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Fri, 14 Jun 2024 11:12:45 +0100 Subject: [PATCH 52/64] fix(developer): use const local variable for int type cxKeyArray --- developer/src/kmcmplib/src/Compiler.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/developer/src/kmcmplib/src/Compiler.cpp b/developer/src/kmcmplib/src/Compiler.cpp index 3da3fecacd..ff50fe9f9a 100644 --- a/developer/src/kmcmplib/src/Compiler.cpp +++ b/developer/src/kmcmplib/src/Compiler.cpp @@ -806,12 +806,13 @@ bool resizeStoreArray(PFILE_KEYBOARD fk) { * reallocates the key array in increments of 100 */ bool resizeKeyArray(PFILE_GROUP gp, int increment) { - if(((int)gp->cxKeyArray + increment - 1) % 100 < increment) { - PFILE_KEY kp = new FILE_KEY[((gp->cxKeyArray + increment)/100 + 1) * 100]; + const int cxKeyArray = (int)gp->cxKeyArray; + if((cxKeyArray + increment - 1) % 100 < increment) { + PFILE_KEY kp = new FILE_KEY[((cxKeyArray + increment)/100 + 1) * 100]; if (!kp) return false; if (gp->dpKeyArray) { - memcpy(kp, gp->dpKeyArray, gp->cxKeyArray * sizeof(FILE_KEY)); + memcpy(kp, gp->dpKeyArray, cxKeyArray * sizeof(FILE_KEY)); delete[] gp->dpKeyArray; } From 822bd5f55486694b0055a7c44e1c75238bd223f6 Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Fri, 14 Jun 2024 11:31:24 +0100 Subject: [PATCH 53/64] fix(developer): changed cast to follow expected integer promotion --- developer/src/kmcmplib/src/NamedCodeConstants.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer/src/kmcmplib/src/NamedCodeConstants.cpp b/developer/src/kmcmplib/src/NamedCodeConstants.cpp index 897faa98d0..6067a1ae50 100644 --- a/developer/src/kmcmplib/src/NamedCodeConstants.cpp +++ b/developer/src/kmcmplib/src/NamedCodeConstants.cpp @@ -268,7 +268,7 @@ int IsHangulSyllable(const KMX_WCHAR *codename, int *code) if(strchr("GNDRMBSJCKTPH", ch)) { /* Has an initial syllable */ - int isDoubled = (int)towupper(*(codename+1)) == ch; + int isDoubled = towupper(*(codename+1)) == (wint_t)ch; LIndex = -1; for(i = 0; i < HangulLCount; i++) { From e51a516a78a8cd7f7e10a020038295e05d019c12 Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Fri, 14 Jun 2024 11:44:52 +0100 Subject: [PATCH 54/64] fix(developer): user tertiary operator to keep pos_dot variable positive --- developer/src/kmcmplib/src/kmx_u16.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/developer/src/kmcmplib/src/kmx_u16.cpp b/developer/src/kmcmplib/src/kmx_u16.cpp index c99a27b201..b8a030296c 100644 --- a/developer/src/kmcmplib/src/kmx_u16.cpp +++ b/developer/src/kmcmplib/src/kmx_u16.cpp @@ -283,10 +283,7 @@ double u16tof( KMX_WCHAR* str) char digit; PKMX_WCHAR q = (PKMX_WCHAR)u16chr(str, '.'); - size_t pos_dot = q-str ; - - if (q-str < 0) - pos_dot = u16len(str); + size_t pos_dot = (q-str < 0) ? u16len(str) : q-str; for (size_t i = 0; i < u16len(str); i++) { From 721e38d52e11a878de5ef6ef570604780d603517 Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Fri, 14 Jun 2024 11:54:36 +0100 Subject: [PATCH 55/64] fix(developer): change cast in kmcmp_CompileKeyboard to follow expected integer promotion --- developer/src/kmcmplib/tests/kmcompxtest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/developer/src/kmcmplib/tests/kmcompxtest.cpp b/developer/src/kmcmplib/tests/kmcompxtest.cpp index aa79481116..3800005921 100644 --- a/developer/src/kmcmplib/tests/kmcompxtest.cpp +++ b/developer/src/kmcmplib/tests/kmcompxtest.cpp @@ -86,11 +86,11 @@ int main(int argc, char *argv[]) fseek(fp2, 0, SEEK_END); auto sz2 = ftell(fp2); fseek(fp2, 0, SEEK_SET); - if ((long)result.kmxSize != sz2) return __LINE__; // exit code: size of kmx-file in build differs from size of kmx-file in source folder + if (result.kmxSize != (size_t)sz2) return __LINE__; // exit code: size of kmx-file in build differs from size of kmx-file in source folder char* buf2 = new char[result.kmxSize]; auto sz3 = fread(buf2, 1, result.kmxSize, fp2); - if (result.kmxSize != sz3) return __LINE__; // exit code: when not able to read the build into the buffer + if (result.kmxSize != sz3) return __LINE__; // exit code: when not able to read the build into the buffer return memcmp(result.kmx, buf2, result.kmxSize) ? __LINE__ : 0; // exit code: when contents of kmx-file in build differs from contents of kmx-file in source folder // success: when contents of kmx-file in build and source folder are the same } From 15fd72970674f06b58d1a414b246f244fbe32e21 Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Fri, 14 Jun 2024 12:00:22 +0100 Subject: [PATCH 56/64] fix(developer): change cast to follow expected integer promotion in loadfileProc --- developer/src/kmcmplib/tests/util_callbacks.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer/src/kmcmplib/tests/util_callbacks.cpp b/developer/src/kmcmplib/tests/util_callbacks.cpp index 6b0fafe35f..fdfd28b375 100644 --- a/developer/src/kmcmplib/tests/util_callbacks.cpp +++ b/developer/src/kmcmplib/tests/util_callbacks.cpp @@ -49,7 +49,7 @@ bool loadfileProc(const char* filename, const char* baseFilename, void* data, in } } else { // return data - if((int)fread(data, 1, *size, fp) != *size) { + if(fread(data, 1, *size, fp) != (size_t)(*size)) { fclose(fp); return false; } From 407cfa8f3357b37bf862cfd0de561345d6049761 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Fri, 14 Jun 2024 15:09:13 +0200 Subject: [PATCH 57/64] chore(linux): address code review comments Co-authored-by: Marc Durdin --- linux/ibus-keyman/tests/scripts/test-helper.inc.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/linux/ibus-keyman/tests/scripts/test-helper.inc.sh b/linux/ibus-keyman/tests/scripts/test-helper.inc.sh index 5d895272f0..4d04c30d54 100755 --- a/linux/ibus-keyman/tests/scripts/test-helper.inc.sh +++ b/linux/ibus-keyman/tests/scripts/test-helper.inc.sh @@ -116,7 +116,7 @@ function _setup_init() { echo > "$CLEANUP_FILE" echo > "$PID_FILE" TEMP_DATA_DIR=$(mktemp --directory) - echo "rm -rf ${TEMP_DATA_DIR} || true # TEMP_DATA_DIR" >> "$CLEANUP_FILE" + echo "rm -rf \"${TEMP_DATA_DIR}\" || true # TEMP_DATA_DIR" >> "${CLEANUP_FILE}" COMMON_ARCH_DIR= [ -d "${TOP_SRCDIR}"/../../core/build/arch ] && COMMON_ARCH_DIR=${TOP_SRCDIR}/../../core/build/arch @@ -250,7 +250,7 @@ function _setup_ibus() { #shellcheck disable=SC2086 ibus-daemon ${ARG_VERBOSE-} --daemonize --panel=disable --address=unix:abstract="${TEMP_DATA_DIR}/test-ibus" ${IBUS_CONFIG-} &> /tmp/ibus-daemon.log PID=$(pgrep -f "${TEMP_DATA_DIR}/test-ibus") - echo "if kill -9 ${PID}; then ibus restart || ibus start; fi || true # ibus-daemon" >> "$CLEANUP_FILE" + echo "if kill -9 ${PID}; then ibus restart || ibus start; fi # ibus-daemon" >> "${CLEANUP_FILE}" echo "${PID} ibus-daemon" >> "${PID_FILE}" sleep 1s From 1e67022a44d1523d5bb8fab69b005d51df3bd683 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Fri, 14 Jun 2024 15:37:43 +0200 Subject: [PATCH 58/64] change(linux): restart ibus only when manually running tests If we try to restart ibus when we run the tests as part of a build we run into a timeout in the test teardown. --- linux/ibus-keyman/tests/scripts/run-tests.sh | 2 +- .../ibus-keyman/tests/scripts/test-helper.inc.sh | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/linux/ibus-keyman/tests/scripts/run-tests.sh b/linux/ibus-keyman/tests/scripts/run-tests.sh index d06d8898b1..012dee996f 100755 --- a/linux/ibus-keyman/tests/scripts/run-tests.sh +++ b/linux/ibus-keyman/tests/scripts/run-tests.sh @@ -63,7 +63,7 @@ function run_tests() { G_TEST_BUILDDIR="$(dirname "$0")/../../../build/$(arch)/${CONFIG}/tests" - setup "$DISPLAY_SERVER" "$ENV_FILE" "$CLEANUP_FILE" "$PID_FILE" + setup "$DISPLAY_SERVER" "$ENV_FILE" "$CLEANUP_FILE" "$PID_FILE" --standalone echo "# NOTE: When the tests fail check /tmp/ibus-engine-keyman.log and /tmp/ibus-daemon.log!" echo "" diff --git a/linux/ibus-keyman/tests/scripts/test-helper.inc.sh b/linux/ibus-keyman/tests/scripts/test-helper.inc.sh index 4d04c30d54..a32fef8689 100755 --- a/linux/ibus-keyman/tests/scripts/test-helper.inc.sh +++ b/linux/ibus-keyman/tests/scripts/test-helper.inc.sh @@ -241,16 +241,23 @@ function _setup_schema_and_gsettings() { } function _setup_ibus() { - local ENV_FILE CLEANUP_FILE PID_FILE PID + local ENV_FILE CLEANUP_FILE PID_FILE PID STANDALONE ENV_FILE=$1 CLEANUP_FILE=$2 PID_FILE=$3 + STANDALONE=${4:-} echo "Starting ibus-daemon..." #shellcheck disable=SC2086 ibus-daemon ${ARG_VERBOSE-} --daemonize --panel=disable --address=unix:abstract="${TEMP_DATA_DIR}/test-ibus" ${IBUS_CONFIG-} &> /tmp/ibus-daemon.log PID=$(pgrep -f "${TEMP_DATA_DIR}/test-ibus") - echo "if kill -9 ${PID}; then ibus restart || ibus start; fi # ibus-daemon" >> "${CLEANUP_FILE}" + if [[ "${STANDALONE}" == "--standalone" ]]; then + # manual test run + echo "if kill -9 ${PID}; then ibus restart || ibus start; fi # ibus-daemon" >> "${CLEANUP_FILE}" + else + # test run as part of the build + echo "kill -9 ${PID} || true" >> "${CLEANUP_FILE}" + fi echo "${PID} ibus-daemon" >> "${PID_FILE}" sleep 1s @@ -269,11 +276,12 @@ function _setup_ibus() { } function setup() { - local DISPLAY_SERVER ENV_FILE CLEANUP_FILE PID_FILE TESTBASEDIR TESTDIR + local DISPLAY_SERVER ENV_FILE CLEANUP_FILE PID_FILE TESTBASEDIR TESTDIR STANDALONE DISPLAY_SERVER=$1 ENV_FILE=$2 CLEANUP_FILE=$3 PID_FILE=$4 + STANDALONE=${5:-} _setup_init "${ENV_FILE}" "${CLEANUP_FILE}" "${PID_FILE}" @@ -287,7 +295,7 @@ function setup() { _setup_test_dbus_server "${ENV_FILE}" "${CLEANUP_FILE}" _setup_display_server "${ENV_FILE}" "${CLEANUP_FILE}" "${PID_FILE}" "${DISPLAY_SERVER}" _setup_schema_and_gsettings "${ENV_FILE}" - _setup_ibus "${ENV_FILE}" "${CLEANUP_FILE}" "${PID_FILE}" + _setup_ibus "${ENV_FILE}" "${CLEANUP_FILE}" "${PID_FILE}" "${STANDALONE}" } function setup_display_server_only() { From ca34550a8d3c3e88ede1b579d7ad678c4cfcf181 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 14 Jun 2024 08:55:15 -0500 Subject: [PATCH 59/64] feat(core): update util_regex per review comments - remove some TODOs that were obsolete - copy/clarify/expand comments between the ICU and non-ICU sides Fixes: #9467 --- core/src/util_regex.cpp | 45 ++++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/core/src/util_regex.cpp b/core/src/util_regex.cpp index 55bcb04e6d..f8982fdf65 100644 --- a/core/src/util_regex.cpp +++ b/core/src/util_regex.cpp @@ -157,20 +157,19 @@ bool km_regex::valid() const { bool km_regex::init(const std::u32string &pattern) { #if KMN_NO_ICU -// for now- new regex every time. +// The current implementation makes a new regex every time, so we always return true. assert(!pattern.empty()); fPattern = pattern; - return true; // TODO + return true; #else if (pattern.empty()) { return false; } - // TODO-LDML: if we have mapFrom, may need to do other processing. std::u16string patstr = km::core::kmx::u32string_to_u16string(pattern); UErrorCode status = U_ZERO_ERROR; /* const */ icu::UnicodeString patustr = icu::UnicodeString(patstr.data(), (int32_t)patstr.length()); // add '$' to match to end - patustr.append(u'$'); // TODO-LDML: may need to escape some markers. Marker #91 will look like a `[` to the pattern + patustr.append(u'$'); fPattern.reset(icu::RegexPattern::compile(patustr, 0, status)); return (UASSERT_SUCCESS(status)); #endif @@ -189,30 +188,41 @@ size_t km_regex::apply(const std::u32string &input, std::u32string &output, const auto matchLen = RegexMatchLen(patstr.c_str(), instr.c_str()); assert(matchLen != -1); // error if (matchLen == 0) { - // TODO: not correct, just trying to quell unused arg. - return 0; + return 0; // Normal case return: no match } std::u32string rustr; // replacement if (fromList.empty()) { + // Normal case: not a map. + // This replace will apply $1, $2 etc. rustr = to; } else { - + // we actually need the group(1) string here. + // this is only the content in parenthesis () char *group1 = RegexGroup1(patstr.c_str(), instr.c_str()); assert(group1 != nullptr); const std::string group1str(group1); const std::u32string match32 = convert(group1str); free(group1); + // Now we're ready to do the actual mapping. + + // 1., we need to find the index in the source set. auto matchIndex = findIndex(match32, fromList); - assert(matchIndex != -1); + assert(matchIndex != -1L); // This indicates that the regex and the fromList are out of sync. + // we already asserted on load that the from and to sets have the same cardinality. + + // 2. get the target string + // we use the same matchIndex that was just found + // 3. update the UnicodeString for replacement rustr = toList.at(matchIndex); } std::string rstr = convert(rustr); - // now, perform substitution + // here we replace the match output. char *out = RegexSubstitute(patstr.c_str(), instr.c_str(), rstr.c_str()); assert(out != nullptr); std::string outstr(out); free(out); output = convert(outstr); + // output includes all of 'input', but modified. Need to substring it. /** code units */ const auto matchStart = input.length() - matchLen; // remove the unmatched prefix. @@ -221,23 +231,21 @@ size_t km_regex::apply(const std::u32string &input, std::u32string &output, return matchLen; #else assert(fPattern); - // TODO-LDML: Really? can't go from u32 to UnicodeString? - // TODO-LDML: Also, we could cache the u16 string at the transformGroup level or higher. + // TODO-LDML: This entire section may have too many conversions and copies. Could be optimized. UErrorCode status = U_ZERO_ERROR; const std::u16string matchstr = km::core::kmx::u32string_to_u16string(input); icu::UnicodeString matchustr = icu::UnicodeString(matchstr.data(), (int32_t)matchstr.length()); // TODO-LDML: create a new Matcher every time. These could be cached and reset. std::unique_ptr matcher(fPattern->matcher(matchustr, status)); if (!UASSERT_SUCCESS(status)) { - return 0; // TODO-LDML: return error + return 0; } if (!matcher->find(status)) { // i.e. matches somewhere, in this case at end of str - return 0; // no match + return 0; // Normal case return: no match } - // TODO-LDML: this is UTF-16 len, not UTF-32 len!! - // TODO-LDML: if we had an underlying UText this would be simpler. + // Note: this is UTF-16 len, not UTF-32 len. int32_t matchStart = matcher->start(status); int32_t matchEnd = matcher->end(status); if (!UASSERT_SUCCESS(status)) { @@ -253,6 +261,7 @@ size_t km_regex::apply(const std::u32string &input, std::u32string &output, // should have matched something. assert(matchLen > 0); + // now, do the replace. /** this is the 'to' or other replacement string.*/ @@ -302,7 +311,7 @@ size_t km_regex::apply(const std::u32string &input, std::u32string &output, rustr = icu::UnicodeString(rstr.data(), (int32_t)rstr.length()); // and we return to the regular code flow. } - // here we replace the match output. No normalization, yet. + // here we replace the match output. icu::UnicodeString entireOutput = matcher->replaceFirst(rustr, status); if (!UASSERT_SUCCESS(status)) { // TODO-LDML: could fail here due to bad input (syntax err) @@ -323,12 +332,12 @@ size_t km_regex::apply(const std::u32string &input, std::u32string &output, std::unique_ptr s(new char32_t[out32len + 1]); assert(s); if (!s) { - return 0; // TODO-LDML: allocation failed + return 0; } // convert outu.toUTF32((UChar32 *)(s.get()), out32len + 1, status); if (!UASSERT_SUCCESS(status)) { - return 0; // TODO-LDML: memory issue + return 0; } output.assign(s.get(), out32len); } From ab835547a3fb75bcbb02d2b4ee8ba32d6a107fcc Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Fri, 14 Jun 2024 14:04:35 -0400 Subject: [PATCH 60/64] auto: increment master version to 18.0.57 --- HISTORY.md | 15 +++++++++++++++ VERSION.md | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index c5e1f2e780..bbf1a66cac 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,20 @@ # Keyman Version History +## 18.0.56 alpha 2024-06-14 + +* feat(core): devolve normalization to js (#11541) +* fix(developer): show message if no more platforms to add to touch layout editor (#11759) +* docs(developer): context help in package-editor and put the existing context help in their own tab comments (#11760) +* docs(developer): context help in keyboard-editor section (#11754) +* docs(developer): context help in new-project section (#11767) +* docs(developer): context help for new-project-parameters in keyman developer (#11769) +* docs(developer): context help for Select BCP 47 tag in Keyman Developer (#11770) +* change(common): update esbuild to 0.18.9 (#11693) +* change(web): more prep for better async prediction handling (#10347) +* fix(web): set new-context rules' device to match that of the active OSK (#11743) +* chore(linux): Update debian changelog (#11671) +* fix(web): add limited Array.from polyfill for lm-worker use (#11732) + ## 18.0.55 alpha 2024-06-13 * fix(developer): handle missing OSK when importing a Windows keyboard into a touch-only project (#11720) diff --git a/VERSION.md b/VERSION.md index b38400af1f..847efab442 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -18.0.56 \ No newline at end of file +18.0.57 \ No newline at end of file From db53ac13435635ba0ba5ef6d1da88776f38d21e2 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Sun, 16 Jun 2024 16:42:38 +1000 Subject: [PATCH 61/64] fix(core): serialize tests for core/wasm on mac agents Mitigates: #11794 --- core/commands.inc.sh | 9 ++++++++- docs/build/macos.md | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/core/commands.inc.sh b/core/commands.inc.sh index 9f1365cdda..f2c49d88ca 100644 --- a/core/commands.inc.sh +++ b/core/commands.inc.sh @@ -91,7 +91,14 @@ do_test() { if [[ $target =~ ^(x86|x64)$ ]]; then cmd //C build.bat $target $BUILDER_CONFIGURATION test $testparams else - meson test -C "$MESON_PATH" $testparams + if [[ $target == wasm ]] && [[ $BUILDER_OS == mac ]]; then + # 11794 -- parallel tests failing on some mac build agents; temporary + # mitigation until we diagnose root cause + meson test -j 1 -C "$MESON_PATH" $testparams + else + meson test -C "$MESON_PATH" $testparams + fi + fi builder_finish_action success test:$target } diff --git a/docs/build/macos.md b/docs/build/macos.md index 673739379e..26ece4a6d6 100644 --- a/docs/build/macos.md +++ b/docs/build/macos.md @@ -77,10 +77,10 @@ PATH="$HOMEBREW_PREFIX/opt/coreutils/libexec/gnubin:$PATH" ## KeymanWeb Dependencies -* node.js 18+, emscripten 3.1.46 or later, openjdk 8 +* node.js 22+, emscripten 3.1.46 or later ```shell -brew install node emscripten openjdk@8 +brew install node emscripten ``` Note: if you install emscripten with brew on macOS, only emscripten binaries are From ef03237c2c572664337af8f955c7d971474a3ab0 Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Mon, 17 Jun 2024 10:28:04 +0100 Subject: [PATCH 62/64] fix(developer): minor changes from review --- developer/src/kmcmplib/src/Compiler.cpp | 2 +- developer/src/kmcmplib/src/kmx_u16.cpp | 8 -------- developer/src/kmcmplib/tests/kmcompxtest.cpp | 2 +- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/developer/src/kmcmplib/src/Compiler.cpp b/developer/src/kmcmplib/src/Compiler.cpp index ff50fe9f9a..1ff5b3036c 100644 --- a/developer/src/kmcmplib/src/Compiler.cpp +++ b/developer/src/kmcmplib/src/Compiler.cpp @@ -3517,7 +3517,7 @@ bool UTF16TempFromUTF8(KMX_BYTE* infile, int sz, KMX_BYTE** tempfile, int *sz16) try { std::wstring_convert, char16_t> converter; result = converter.from_bytes((char*)infile, (char*)infile+sz); - } catch(std::range_error &e) { + } catch(std::range_error& e) { AddCompileError(CHINT_NonUnicodeFile); result.resize(sz); for(int i = 0; i < sz; i++) { diff --git a/developer/src/kmcmplib/src/kmx_u16.cpp b/developer/src/kmcmplib/src/kmx_u16.cpp index b8a030296c..7255196e08 100644 --- a/developer/src/kmcmplib/src/kmx_u16.cpp +++ b/developer/src/kmcmplib/src/kmx_u16.cpp @@ -9,11 +9,7 @@ #include #include #include - -#ifdef _MSC_VER -#else #include -#endif //String <- wstring std::string string_from_wstring(std::wstring const str) { @@ -101,11 +97,7 @@ std::string toHex(int num1) { s += (87 + temp); num = num / 16; } -#ifdef _MSC_VER - reverse(s.begin(), s.end()); -#else std::reverse(s.begin(), s.end()); -#endif return s; } diff --git a/developer/src/kmcmplib/tests/kmcompxtest.cpp b/developer/src/kmcmplib/tests/kmcompxtest.cpp index 3800005921..4e4b91e830 100644 --- a/developer/src/kmcmplib/tests/kmcompxtest.cpp +++ b/developer/src/kmcmplib/tests/kmcompxtest.cpp @@ -104,7 +104,7 @@ int main(int argc, char *argv[]) std::istringstream(ErrNr) >> std::hex >> error_val; // check if error_val is in Array of Errors; if it is found return 0 (it's not an error) - for (std::vector::size_type i = 0; i < error_vec.size() ; i++) { + for (size_t i = 0; i < error_vec.size() ; i++) { if (error_vec[i] == error_val) { return 0; // success: CERR_ in Name + Error (specified in CERR_Name) IS found } From 0806d5bd38885bc037749d1d32af7661e0383c84 Mon Sep 17 00:00:00 2001 From: "Dr Mark C. Sinclair" Date: Mon, 17 Jun 2024 10:39:14 +0100 Subject: [PATCH 63/64] fix(developer): correct backslashes in gtest-compmsg-test.cpp --- developer/src/kmcmplib/tests/gtest-compmsg-test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/developer/src/kmcmplib/tests/gtest-compmsg-test.cpp b/developer/src/kmcmplib/tests/gtest-compmsg-test.cpp index afcf6e86b3..9fea4670c3 100644 --- a/developer/src/kmcmplib/tests/gtest-compmsg-test.cpp +++ b/developer/src/kmcmplib/tests/gtest-compmsg-test.cpp @@ -1,6 +1,6 @@ #include -#include "..\..\common\include\kmn_compiler_errors.h" -#include "..\..\..\..\common\include\km_types.h" +#include "../../common/include/kmn_compiler_errors.h" +#include "../../../../common/include/km_types.h" const KMX_CHAR *GetCompilerErrorString(KMX_DWORD code); From 5c98e86b588edc40c1b3547389d428c06e68d817 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Mon, 17 Jun 2024 14:03:54 -0400 Subject: [PATCH 64/64] auto: increment master version to 18.0.58 --- HISTORY.md | 11 +++++++++++ VERSION.md | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index bbf1a66cac..89517888c2 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,16 @@ # Keyman Version History +## 18.0.57 alpha 2024-06-17 + +* fix(web): fix id of longpress keys with modifier set in touch layout (#11783) +* fix(web): prevent desktop OSK crash when addKeyboards is called before engine init (#11786) +* fix(core): serialize tests for core/wasm on mac agents (#11795) +* fix(developer): refactor kmcmplib compiler messages to use map (#11738) +* fix(developer): make native compilation of kmcmplib under Linux possible (#11779) +* feat(core): devolve regex to javascript (#11777) +* feat(core): remove ICU from core under wasm (#11778) +* fix(linux): restart ibus after manual integration test run (#11775) + ## 18.0.56 alpha 2024-06-14 * feat(core): devolve normalization to js (#11541) diff --git a/VERSION.md b/VERSION.md index 847efab442..43a4d909f5 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -18.0.57 \ No newline at end of file +18.0.58 \ No newline at end of file