diff --git a/core/src/kmx/kmx_plus.cpp b/core/src/kmx/kmx_plus.cpp index b02fec77d8..a0da603d95 100644 --- a/core/src/kmx/kmx_plus.cpp +++ b/core/src/kmx/kmx_plus.cpp @@ -1072,16 +1072,16 @@ COMP_KMXPLUS_USET_Helper::setUset(const COMP_KMXPLUS_USET *newUset) { return is_valid; } -USet::USet(const COMP_KMXPLUS_USET_RANGE *newRange, size_t newCount) { +SimpleUSet::SimpleUSet(const COMP_KMXPLUS_USET_RANGE *newRange, size_t newCount) { for (size_t i = 0; i < newCount; i++) { ranges.emplace_back(newRange[i].start, newRange[i].end); } } -USet::USet() { +SimpleUSet::SimpleUSet() { } -bool USet::contains(km_kbp_usv ch) const { +bool SimpleUSet::contains(km_kbp_usv ch) const { for (const auto &range : ranges) { if (range.start <= ch && range.end >= ch) { return true; @@ -1091,7 +1091,7 @@ bool USet::contains(km_kbp_usv ch) const { } bool -USet::valid() const { +SimpleUSet::valid() const { // double check for (const auto &range : ranges) { if (!Uni_IsValid(range.start, range.end)) { @@ -1103,7 +1103,7 @@ USet::valid() const { } void -USet::dump() const { +SimpleUSet::dump() const { DebugLog(" - USet size=%d", ranges.size()); for (const auto &range : ranges) { if (range.start == range.end) { @@ -1114,14 +1114,14 @@ USet::dump() const { } } -USet +SimpleUSet COMP_KMXPLUS_USET_Helper::getUset(KMXPLUS_USET i) const { if (!valid() || i >= uset->usetCount) { assert(false); - return USet(nullptr, 0); // empty set + return SimpleUSet(nullptr, 0); // empty set } auto &set = usets[i]; - return USet(getRange(set.range), set.count); + return SimpleUSet(getRange(set.range), set.count); } const COMP_KMXPLUS_USET_RANGE * diff --git a/core/src/kmx/kmx_plus.h b/core/src/kmx/kmx_plus.h index aa96494043..d84ee34aed 100644 --- a/core/src/kmx/kmx_plus.h +++ b/core/src/kmx/kmx_plus.h @@ -699,14 +699,16 @@ struct COMP_KMXPLUS_USET_RANGE { }; /** - * represents one of the uset elements + * represents one of the uset elements. + * TODO-LDML: replace this with a real icu::UnicodeSet? or at least + * a function producing the same? */ -class USet { +class SimpleUSet { public: /** construct a set over the specified range. Data is copied. */ - USet(const COMP_KMXPLUS_USET_RANGE* newStart, size_t newCount); + SimpleUSet(const COMP_KMXPLUS_USET_RANGE* newStart, size_t newCount); /** empty set */ - USet(); + SimpleUSet(); /** true if the uset contains this char */ bool contains(km_kbp_usv ch) const; /** debugging */ @@ -726,7 +728,7 @@ public: bool setUset(const COMP_KMXPLUS_USET *newUset); inline bool valid() const { return is_valid; } - USet getUset(KMXPLUS_USET list) const; + SimpleUSet getUset(KMXPLUS_USET list) const; const COMP_KMXPLUS_USET_RANGE *getRange(KMX_DWORD index) const; private: diff --git a/core/src/kmx/kmx_xstring.h b/core/src/kmx/kmx_xstring.h index 9b479dac7a..028cde8e99 100644 --- a/core/src/kmx/kmx_xstring.h +++ b/core/src/kmx/kmx_xstring.h @@ -172,6 +172,19 @@ u16string_to_u32string(const std::u16string &source) { return out; } +inline std::u16string +u32string_to_u16string(const std::u32string &source) { + std::u16string out; + char16_single ch; + for (auto c : source) { + const auto len = Utf32CharToUtf16(c, ch); + for (auto i = 0; i < len; i++) { + out.push_back(ch.ch[i]); + } + } + return out; +} + inline bool Uni_IsEndOfPlaneNonCharacter(km_kbp_usv ch) { return (((ch) & Uni_FFFE_NONCHARACTER) == Uni_FFFE_NONCHARACTER); // matches FFFF or FFFE } diff --git a/core/src/ldml/ldml_transforms.cpp b/core/src/ldml/ldml_transforms.cpp index 821f48e0dd..2d6afe8ff0 100644 --- a/core/src/ldml/ldml_transforms.cpp +++ b/core/src/ldml/ldml_transforms.cpp @@ -11,7 +11,6 @@ #include #include "kmx/kmx_xstring.h" - #ifndef assert #define assert(x) // TODO-LDML #endif @@ -36,7 +35,7 @@ namespace ldml { #define DebugTran(msg, ...) #endif -element::element(const USet &new_u, KMX_DWORD new_flags) +element::element(const SimpleUSet &new_u, KMX_DWORD new_flags) : chr(), uset(new_u), flags((new_flags & ~LDML_ELEM_FLAGS_TYPE) | LDML_ELEM_FLAGS_TYPE_USET) { } @@ -191,7 +190,7 @@ element_list::load(const kmx::kmx_plus &kplus, kmx::KMXPLUS_ELEM id) { km_kbp_usv ch = e.element; emplace_back(ch, flags); // char } else if (type == LDML_ELEM_FLAGS_TYPE_USET) { - // need to load a USet + // need to load a SimpleUSet auto u = kplus.usetHelper.getUset(e.element); if (!u.valid()) { DebugLog("Error, invalid UnicodeSet at element %d", (int)i); @@ -393,35 +392,96 @@ reorder_group::apply(std::u32string &str) const { return applied; } +transform_entry::transform_entry(const transform_entry &other) : + fFrom(other.fFrom), fTo(other.fTo), fFromPattern(nullptr) { + if (other.fFromPattern) { + // clone pattern + fFromPattern.reset(other.fFromPattern->clone()); + } +} + transform_entry::transform_entry(const std::u32string &from, const std::u32string &to) : fFrom(from), fTo(to) { + assert(!fFrom.empty()); // TODO-LDML: should not happen? + + if (!fFrom.empty()) { + const std::u16string patstr = km::kbp::kmx::u32string_to_u16string(fFrom); + UErrorCode status = U_ZERO_ERROR; + /* const */ icu::UnicodeString patustr = icu::UnicodeString(patstr.data(), (int32_t)patstr.length()); + // add '$' to match to end + patustr.append(u'$'); + fFromPattern.reset(icu::RegexPattern::compile(patustr, 0, status)); + assert(U_SUCCESS(status)); // TODO-LDML: may be best to propagate status up ^^ + } } size_t -transform_entry::match(const std::u32string &input) const { - if (input.length() < fFrom.length()) { - // TODO-LDML: regex - // Too small, can't match. - return 0; - } - // string at end - auto substr = input.substr(input.length() - fFrom.length(), fFrom.length()); - if (substr != fFrom) { - // end of string doesn't match - return 0; - } - // match length == fFrom.length - return substr.length(); -} +transform_entry::apply(const std::u32string &input, std::u32string &output) const { + assert(fFromPattern); + // TODO-LDML: Really? can't go from u32 to UnicodeString? + // TODO-LDML: Also, we could cache the u16 string at the transformGroup level or higher. + UErrorCode status = U_ZERO_ERROR; + const std::u16string matchstr = km::kbp::kmx::u32string_to_u16string(input); + icu::UnicodeString matchustr = icu::UnicodeString(matchstr.data(), (int32_t)matchstr.length()); + // TODO-LDML: create a new Matcher every time. These could be cached and reset. + std::unique_ptr matcher(fFromPattern->matcher(matchustr, status)); + assert(U_SUCCESS(status)); -std::u32string -transform_entry::apply(const std::u32string & /*input*/, size_t /*matchLen*/) const { - // TODO-LDML: regex - // For now, we just return the 'to' string literally. - return fTo; + if (!matcher->find(status)) { // i.e. matches somewhere, in this case at end of str + return 0; // no match + } + + // TODO-LDML: this is UTF-16 len, not UTF-32 len!! + // TODO-LDML: if we had an underlying UText this would be simpler. + int32_t matchStart = matcher->start(status); + int32_t matchEnd = matcher->end(status); + assert(U_SUCCESS(status)); + // extract.. + const icu::UnicodeString substr = matchustr.tempSubStringBetween(matchStart, matchEnd); + // preflight to UTF-32 to get length + UErrorCode substrStatus = U_ZERO_ERROR; + auto matchLen = substr.toUTF32(nullptr, 0, substrStatus); + assert(matchLen > 0); + if (matchLen == 0) { + return 0; + } + // Now, we have a matchLen. + + // now, do the replace. + // Convert the fTo into u16 TODO-LDML (we could cache this?) + const std::u16string rstr = km::kbp::kmx::u32string_to_u16string(fTo); + icu::UnicodeString rustr = icu::UnicodeString(rstr.data(), (int32_t)rstr.length()); + // This replace will apply $1, $2 etc. TODO-LDML it will NOT handle mapFrom or mapTo. + icu::UnicodeString entireOutput = matcher->replaceFirst(rustr, status); + assert(U_SUCCESS(status)); + // entireOutput includes all of 'input', but modified. Need to substring it. + icu::UnicodeString outu = entireOutput.tempSubString(matchStart); + + // Special case if there's no output + if (outu.length() == 0) { + output.clear(); + } else { + // TODO-LDML: All we are trying to do is to extract the output string. Probably too many steps. + UErrorCode preflightStatus = U_ZERO_ERROR; + // calculate how big the buffer is + auto out32len = outu.toUTF32(nullptr, 0, preflightStatus); // preflightStatus will be an err, because we know the buffer overruns zero bytes + // allocate + char32_t *s = new char32_t[out32len + 1]; + assert(s != nullptr); + // convert + outu.toUTF32((UChar32 *)s, out32len + 1, status); + assert(U_SUCCESS(status)); + output.assign(s, out32len); + // now, build a u32string + std::u32string out32(s, out32len); + // clean up buffer + delete [] s; + } + return matchLen; } any_group::any_group(const transform_group &g) : type(any_group_type::transform), transform(g), reorder() { } + any_group::any_group(const reorder_group &g) : type(any_group_type::reorder), transform(), reorder(g) { } @@ -444,17 +504,18 @@ transform_group::transform_group() { /** * return the first transform match in this group */ -const transform_entry * -transform_group::match(const std::u32string &input, size_t &subMatched) const { +size_t +transform_group::apply(const std::u32string &input, std::u32string &output) const { + size_t subMatched = 0; for (auto transform = begin(); (subMatched == 0) && (transform < end()); transform++) { // TODO-LDML: non regex implementation // is the match area too short? - subMatched = transform->match(input); + subMatched = transform->apply(input, output); if (subMatched != 0) { - return &(*transform); // return alias to transform + return subMatched; // matched. break out. } } - return nullptr; + return 0; // no match } /** @@ -507,20 +568,14 @@ transforms::apply(const std::u32string &input, std::u32string &output) { // TODO-LDML: reorders // Assume it's a non reorder group /** Length of match within this group*/ - size_t subMatched = 0; // find the first match in this group (if present) // TODO-LDML: check if reorder if (group->type == any_group_type::transform) { - auto entry = group->transform.match(updatedInput, subMatched); - - if (entry != nullptr) { - // now apply the found transform - - // update subOutput (string) and subMatched - // the returned string must replace the last "subMatched" chars of the string. - std::u32string subOutput = entry->apply(updatedInput, subMatched); + std::u32string subOutput; + size_t subMatched = group->transform.apply(updatedInput, subOutput); + if (subMatched != 0) { // remove the matched part of the updatedInput updatedInput.resize(updatedInput.length() - subMatched); // chop of the subMatched part at end updatedInput.append(subOutput); // subOutput could be empty such as in backspace transform diff --git a/core/src/ldml/ldml_transforms.hpp b/core/src/ldml/ldml_transforms.hpp index 6296399f2b..cbe1b03dce 100644 --- a/core/src/ldml/ldml_transforms.hpp +++ b/core/src/ldml/ldml_transforms.hpp @@ -14,11 +14,23 @@ #include #include +#if !defined(HAVE_ICU4C) +#error icu4c is required for this code +#endif + +#define U_FALLTHROUGH +#include "unicode/utypes.h" +#include "unicode/uniset.h" +#include "unicode/usetiter.h" +#include "unicode/unistr.h" +#include "unicode/regex.h" +#include "unicode/utext.h" + namespace km { namespace kbp { namespace ldml { -using km::kbp::kmx::USet; +using km::kbp::kmx::SimpleUSet; /** * Type of a group @@ -33,12 +45,11 @@ enum any_group_type { */ class element { public: - /** construct from a USet */ - element(const USet &u, KMX_DWORD flags); + /** construct from a SimpleUSet */ + element(const SimpleUSet &u, KMX_DWORD flags); /** construct from a single char */ element(km_kbp_usv ch, KMX_DWORD flags); - - /** @returns true if a USet type */ + /** @returns true if a SimpleUSet type */ bool is_uset() const; /** @returns true if prebase bit set*/ bool is_prebase() const; @@ -58,7 +69,7 @@ public: private: // TODO-LDML: support multi-char strings? const km_kbp_usv chr; - const USet uset; + const SimpleUSet uset; const KMX_DWORD flags; }; @@ -67,6 +78,7 @@ private: */ class transform_entry { public: + transform_entry(const transform_entry &other); transform_entry( const std::u32string &from, const std::u32string &to @@ -74,18 +86,17 @@ public: ); /** - * @returns length if it's a match + * If matching, apply the match to the output string + * @param input input string to match + * @param output output string + * @returns length of 'input' which was matched */ - size_t match(const std::u32string &input) const; - - /** - * @returns output string - */ - std::u32string apply(const std::u32string &input, size_t matchLen) const; + size_t apply(const std::u32string &input, std::u32string &output) const; private: - const std::u32string fFrom; // TODO-LDML: regex + const std::u32string fFrom; const std::u32string fTo; + std::unique_ptr fFromPattern; }; /** @@ -101,12 +112,12 @@ public: transform_group(); /** - * Find the first match in the group + * Find the first match in the group and apply it. * @param input input string to match - * @param subMatched on output, the matched length - * @returns alias to transform_entry or nullptr + * @param output output string + * @returns length of 'input' which was matched */ - const transform_entry *match(const std::u32string &input, size_t &subMatched) const; + size_t apply(const std::u32string &input, std::u32string &output) const; }; /** a single char, categorized according to reorder rules*/ @@ -228,7 +239,7 @@ public: size_t apply(const std::u32string &input, std::u32string &output); /** - * For tests - TODO-LDML only supports reorder + * For tests * @return true if str was altered */ bool apply(std::u32string &str); diff --git a/core/src/meson.build b/core/src/meson.build index 97b9291646..87fd9181dd 100644 --- a/core/src/meson.build +++ b/core/src/meson.build @@ -21,6 +21,26 @@ if cpp_compiler.get_id() == 'emscripten' defns += ['-DKMN_KBP'] endif +# ICU4C is used for repertoire tests and core implementation + +if target_machine.system() == 'linux' + # use pkg-config when targetting linux + pkgconfig = 'pkg-config' + icu_uc = dependency('icu-uc', required: true) + icu_i18n = dependency('icu-i18n', required: true) +else + # load ICU from wrap + icu4c = subproject('icu-minimal', default_options: [ 'default_library=static', 'cpp_std=c++17', 'warning_level=0', +'werror=false']) # TODO-LDML: options: static, no data (these are in the meson build files) + icu_uc = icu4c.get_variable('icuuc_dep') + icu_i18n = icu4c.get_variable('icui18n_dep') +endif + +if icu_uc.found() + defns += '-DHAVE_ICU4C' +endif + + kmx_files = files( 'option.cpp', 'keyboard.cpp', @@ -85,11 +105,13 @@ lib = library('kmnkbp0', version: lib_version, include_directories: inc, pic: true, - install: true) + install: true, + dependencies: [icu_uc, icu_i18n], + ) headerdirs = [ '.', 'keyman' ] # subdirectories of ${prefix}/include to add to header path -kmnkbp = declare_dependency(link_with: lib, include_directories: inc) +kmnkbp = declare_dependency(link_with: lib, include_directories: inc, dependencies: [icu_uc, icu_i18n]) pkg = import('pkgconfig') pkg.generate( diff --git a/core/subprojects/icu-minimal.wrap b/core/subprojects/icu-minimal.wrap index fe509b48e0..3c85940c60 100644 --- a/core/subprojects/icu-minimal.wrap +++ b/core/subprojects/icu-minimal.wrap @@ -10,5 +10,4 @@ patch_directory = icu [provide] icu-uc = icuuc_dep -# TODO-LDML: not including i18n at present -#icu-i18n = icui18n_dep +icu-i18n = icui18n_dep diff --git a/core/subprojects/packagefiles/icu/meson.build b/core/subprojects/packagefiles/icu/meson.build index 32bc209139..128fc9fbf9 100644 --- a/core/subprojects/packagefiles/icu/meson.build +++ b/core/subprojects/packagefiles/icu/meson.build @@ -26,9 +26,8 @@ endif # per icudefs.mk.in: # "U_ATTRIBUTE_DEPRECATED is defined to hide warnings about deprecated API warnings." -add_project_arguments('-DU_ATTRIBUTE_DEPRECATED=', '-DUCONFIG_USE_LOCAL=1', language: 'c') - -add_project_arguments('-DU_NOEXCEPT=', language: 'c') +add_project_arguments('-DU_NOEXCEPT=', '-DU_ATTRIBUTE_DEPRECATED=', '-DUCONFIG_USE_LOCAL=1', language: 'c') +add_project_arguments('-DU_NOEXCEPT=', '-DU_ATTRIBUTE_DEPRECATED=', '-DUCONFIG_USE_LOCAL=1', language: 'cpp') uconfig = configuration_data() @@ -42,8 +41,12 @@ uconfig.set('UCONFIG_NO_IDNA', 1) uconfig.set('UCONFIG_NO_COLLATION', 1) uconfig.set('UCONFIG_NO_FORMATTING', 1) uconfig.set('UCONFIG_NO_TRANSLITERATION', 1) -uconfig.set('UCONFIG_NO_REGULAR_EXPRESSIONS', 1) # TODO-LDML: Will probably want this for transforms #7375 +uconfig.set('UCONFIG_NO_REGULAR_EXPRESSIONS', 0) # want these for transforms #7375 uconfig.set('UCONFIG_NO_SERVICE', 1) +uconfig.set('U_OVERRIDE_CXX_ALLOCATION', 1) +uconfig.set('UCONFIG_NO_CONVERSION', 1) +uconfig.set('UCONFIG_USE_WINDOWS_LCID_MAPPING_API', 0) + uconfig_local = configure_file( diff --git a/core/subprojects/packagefiles/icu/source/common/meson.build b/core/subprojects/packagefiles/icu/source/common/meson.build index 4653d88383..4694499a86 100644 --- a/core/subprojects/packagefiles/icu/source/common/meson.build +++ b/core/subprojects/packagefiles/icu/source/common/meson.build @@ -75,10 +75,10 @@ sources = files( 'stringpiece.cpp', 'stringtriebuilder.cpp', 'uarrsort.cpp', - 'ubidi.cpp', + # 'ubidi.cpp', 'ubidi_props.cpp', - 'ubidiln.cpp', - 'ubidiwrt.cpp', + # 'ubidiln.cpp', + # 'ubidiwrt.cpp', 'ubrk.cpp', 'ucase.cpp', 'ucasemap.cpp', @@ -91,29 +91,29 @@ sources = files( 'uchriter.cpp', 'ucln_cmn.cpp', 'ucmndata.cpp', - 'ucnv.cpp', - 'ucnv2022.cpp', - 'ucnv_bld.cpp', - 'ucnv_cb.cpp', - 'ucnv_cnv.cpp', - 'ucnv_ct.cpp', - 'ucnv_err.cpp', - 'ucnv_ext.cpp', - 'ucnv_io.cpp', - 'ucnv_lmb.cpp', - 'ucnv_set.cpp', - 'ucnv_u16.cpp', - 'ucnv_u32.cpp', - 'ucnv_u7.cpp', - 'ucnv_u8.cpp', - 'ucnvbocu.cpp', - 'ucnvdisp.cpp', - 'ucnvhz.cpp', - 'ucnvisci.cpp', - 'ucnvlat1.cpp', - 'ucnvmbcs.cpp', - 'ucnvscsu.cpp', - 'ucnvsel.cpp', + # 'ucnv.cpp', + # 'ucnv2022.cpp', + # 'ucnv_bld.cpp', + # 'ucnv_cb.cpp', + # 'ucnv_cnv.cpp', + # 'ucnv_ct.cpp', + # 'ucnv_err.cpp', + # 'ucnv_ext.cpp', + # 'ucnv_io.cpp', + # 'ucnv_lmb.cpp', + # 'ucnv_set.cpp', + # 'ucnv_u16.cpp', + # 'ucnv_u32.cpp', + # 'ucnv_u7.cpp', + # 'ucnv_u8.cpp', + # 'ucnvbocu.cpp', + # 'ucnvdisp.cpp', + # 'ucnvhz.cpp', + # 'ucnvisci.cpp', + # 'ucnvlat1.cpp', + # 'ucnvmbcs.cpp', + # 'ucnvscsu.cpp', + # 'ucnvsel.cpp', 'ucol_swp.cpp', 'ucptrie.cpp', 'ucurr.cpp', @@ -287,8 +287,8 @@ headers = files( 'unicode/stringpiece.h', 'unicode/stringtriebuilder.h', 'unicode/symtable.h', - 'unicode/ubidi.h', - 'unicode/ubiditransform.h', + # 'unicode/ubidi.h', + # 'unicode/ubiditransform.h', 'unicode/ubrk.h', 'unicode/ucasemap.h', 'unicode/ucat.h', diff --git a/core/subprojects/packagefiles/icu/source/i18n/meson.build b/core/subprojects/packagefiles/icu/source/i18n/meson.build index 787afe3390..b2e64e63be 100644 --- a/core/subprojects/packagefiles/icu/source/i18n/meson.build +++ b/core/subprojects/packagefiles/icu/source/i18n/meson.build @@ -52,7 +52,7 @@ sources = files( 'dayperiodrules.cpp', 'dcfmtsym.cpp', 'decContext.cpp', - 'decNumber.cpp', + # 'decNumber.cpp', 'decimfmt.cpp', 'displayoptions.cpp', 'double-conversion-bignum-dtoa.cpp', diff --git a/core/subprojects/packagefiles/icu/source/meson.build b/core/subprojects/packagefiles/icu/source/meson.build index 7d08a1e3cb..a304720dfb 100644 --- a/core/subprojects/packagefiles/icu/source/meson.build +++ b/core/subprojects/packagefiles/icu/source/meson.build @@ -9,8 +9,7 @@ endif subdir('stubdata') subdir('common') -# TODO-LDML: Not used now, will need for regex -# subdir('i18n') +subdir('i18n') ## Note: The following subdirs are not used by Keyman (at present) diff --git a/core/tests/unit/kmnkbd/meson.build b/core/tests/unit/kmnkbd/meson.build index 049770fa2f..9d3ee825e8 100644 --- a/core/tests/unit/kmnkbd/meson.build +++ b/core/tests/unit/kmnkbd/meson.build @@ -37,6 +37,7 @@ foreach t : tests cpp_args: local_defns + defns + warns, include_directories: [inc, libsrc], link_args: links + tests_flags, + dependencies: [icu_uc, icu_i18n], objects: lib.extract_all_objects(recursive: false)) test(t[0], bin, args: ['--color', test_path]) diff --git a/core/tests/unit/kmnkbd/test_kmx_xstring.cpp b/core/tests/unit/kmnkbd/test_kmx_xstring.cpp index e3c4077622..92404b1bc8 100644 --- a/core/tests/unit/kmnkbd/test_kmx_xstring.cpp +++ b/core/tests/unit/kmnkbd/test_kmx_xstring.cpp @@ -1329,6 +1329,35 @@ test_u16string_to_u32string() { } } + +void +test_u32string_to_u16string() { + std::cout << "== " << __FUNCTION__ << std::endl; + // normal cases + { + const auto str = u32string_to_u16string(U""); + assert_equal(str.length(), 0); + } + { + const auto str = u32string_to_u16string(U"e"); + assert_equal(str.length(), 1); + assert_equal(str.at(0), 0x0065); + } + { + const auto str = u32string_to_u16string(U"🙀"); + assert_equal(str.length(), 2); + assert_equal(str.at(0), 0xD83D); + assert_equal(str.at(1), 0xDE40); + } + { + const auto str = u32string_to_u16string(U"Ω🙀"); + assert_equal(str.length(), 3); + assert_equal(str.at(0), u'Ω'); + assert_equal(str.at(1), 0xD83D); + assert_equal(str.at(2), 0xDE40); + } +} + void test_is_valid() { std::cout << "== " << __FUNCTION__ << std::endl; // valid diff --git a/core/tests/unit/kmx/meson.build b/core/tests/unit/kmx/meson.build index df22473e48..e12ed3a995 100644 --- a/core/tests/unit/kmx/meson.build +++ b/core/tests/unit/kmx/meson.build @@ -31,6 +31,7 @@ kmx = executable('kmx', cpp_args: defns + warns, include_directories: [inc, libsrc, '../../kmx_test_source'], link_args: links + tests_flags, + dependencies: [icu_uc, icu_i18n], objects: [lib.extract_all_objects(recursive: false), kmx_test_source_lib.extract_all_objects(recursive: false)]) tests = [ @@ -178,6 +179,7 @@ key_e = executable('key_list', ['kmx_key_list.cpp', '../emscripten_filesystem.cp cpp_args: defns + warns, include_directories: [inc, libsrc], link_args: links + tests_flags, + dependencies: [icu_uc, icu_i18n], objects: lib.extract_all_objects(recursive: false)) test_kbd = 'kmx_key_list' @@ -196,6 +198,7 @@ imx_e = executable('imx_list', ['kmx_imx.cpp', '../emscripten_filesystem.cpp'], cpp_args: defns + warns, include_directories: [inc, libsrc], link_args: links + tests_flags, + dependencies: [icu_uc, icu_i18n], objects: lib.extract_all_objects(recursive: false)) test_kbd = 'kmx_imsample' @@ -213,6 +216,7 @@ external_e = executable('ext_event', ['kmx_external_event.cpp', '../emscripten_f cpp_args: defns + warns, include_directories: [inc, libsrc], link_args: links + tests_flags, + dependencies: [icu_uc, icu_i18n], objects: lib.extract_all_objects(recursive: false)) test_kbd = 'k_033___caps_always_off' diff --git a/core/tests/unit/ldml/keyboards/k_003_transform.xml b/core/tests/unit/ldml/keyboards/k_003_transform.xml index aa90e79a3c..565378400a 100644 --- a/core/tests/unit/ldml/keyboards/k_003_transform.xml +++ b/core/tests/unit/ldml/keyboards/k_003_transform.xml @@ -45,7 +45,7 @@ from https://github.com/unicode-org/cldr/blob/keyboard-preview/docs/ldml/tr35-ke - + diff --git a/core/tests/unit/ldml/keyboards/k_007_transform_rgx-test.xml b/core/tests/unit/ldml/keyboards/k_007_transform_rgx-test.xml new file mode 100644 index 0000000000..6e4cebf9f1 --- /dev/null +++ b/core/tests/unit/ldml/keyboards/k_007_transform_rgx-test.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/core/tests/unit/ldml/keyboards/k_007_transform_rgx.xml b/core/tests/unit/ldml/keyboards/k_007_transform_rgx.xml new file mode 100644 index 0000000000..3f0bdcdd9b --- /dev/null +++ b/core/tests/unit/ldml/keyboards/k_007_transform_rgx.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/core/tests/unit/ldml/keyboards/meson.build b/core/tests/unit/ldml/keyboards/meson.build index 54ab22f2e8..b09585a77d 100644 --- a/core/tests/unit/ldml/keyboards/meson.build +++ b/core/tests/unit/ldml/keyboards/meson.build @@ -32,6 +32,7 @@ tests_without_testdata = [ # These tests have a k_001_tiny-test.xml file as well. tests_with_testdata = [ 'k_001_tiny', + 'k_007_transform_rgx', 'k_020_fr', # TODO-LDML: move to cldr above (fix vkey) 'k_200_reorder_nod_Lana', 'k_210_marker', diff --git a/core/tests/unit/ldml/ldml_test_source.cpp b/core/tests/unit/ldml/ldml_test_source.cpp index da65e62a27..673f692f1e 100644 --- a/core/tests/unit/ldml/ldml_test_source.cpp +++ b/core/tests/unit/ldml/ldml_test_source.cpp @@ -25,7 +25,6 @@ #include "ldml/keyboardprocessor_ldml.h" #include "ldml/ldml_processor.hpp" - #include "path.hpp" #include "state.hpp" #include "utfcodec.hpp" @@ -36,6 +35,7 @@ #if defined(HAVE_ICU4C) // TODO-LDML: Needed this for some compiler warnings #define U_FALLTHROUGH +#include "unicode/utypes.h" #include "unicode/uniset.h" #include "unicode/usetiter.h" #else diff --git a/core/tests/unit/ldml/meson.build b/core/tests/unit/ldml/meson.build index 679acbe599..c02d64bf18 100644 --- a/core/tests/unit/ldml/meson.build +++ b/core/tests/unit/ldml/meson.build @@ -4,25 +4,6 @@ # Authors: Marc Durdin # -# ICU4C is used for repertoire tests - - -if target_machine.system() == 'linux' - # use pkg-config when targetting linux - pkgconfig = 'pkg-config' - icu_uc = dependency('icu-uc', required: true) -else - # load ICU from wrap - # Requires meson of about 0.57+ - icu4c = subproject('icu-minimal', default_options: [ 'default_library=static', 'cpp_std=c++17', 'warning_level=0', -'werror=false']) # TODO-LDML: options: static, no data (these are in the meson build files) - icu_uc = icu4c.get_variable('icuuc_dep') -endif - -if icu_uc.found() - defns += '-DHAVE_ICU4C' -endif - # TODO -- why are these differing from the standard.meson.build flags? if cpp_compiler.get_id() == 'gcc' or cpp_compiler.get_id() == 'clang' or cpp_compiler.get_id() == 'emscripten' warns = [ @@ -75,8 +56,9 @@ ldml = executable('ldml', cpp_args: defns + warns, include_directories: [inc, libsrc, '../../../../developer/src/ext/json'], link_args: links + tests_flags, + dependencies: [icu_uc, icu_i18n], + # link_with: [lib], objects: lib.extract_all_objects(recursive: false), - dependencies: [icu_uc], ) # Run tests on all keyboards (`tests` defined in keyboards/meson.build) @@ -103,6 +85,7 @@ e = executable('test_kmx_plus', 'test_kmx_plus.cpp', cpp_args: defns + warns, include_directories: [inc, libsrc, '../../../../developer/src/ext/json'], link_args: links + tests_flags, + dependencies: [icu_uc, icu_i18n], objects: lib.extract_all_objects(recursive: false)) test('test_kmx_plus', e, suite: 'ldml') @@ -110,5 +93,6 @@ t = executable('test_transforms', 'test_transforms.cpp', cpp_args: defns + warns, include_directories: [inc, libsrc, '../../../../developer/src/ext/json'], link_args: links + tests_flags, + dependencies: [icu_uc, icu_i18n], objects: lib.extract_all_objects(recursive: false)) test('test_transforms', t, suite: 'ldml') diff --git a/core/tests/unit/ldml/test_kmx_plus.cpp b/core/tests/unit/ldml/test_kmx_plus.cpp index 74961f797e..7f158dd1c3 100644 --- a/core/tests/unit/ldml/test_kmx_plus.cpp +++ b/core/tests/unit/ldml/test_kmx_plus.cpp @@ -152,12 +152,12 @@ int test_uset() { {0x127, 0x127} // [ħ] }; - USet u0(&r[0], 2); + SimpleUSet u0(&r[0], 2); assert_equal(u0.contains(0x62), true); // b assert_equal(u0.contains(0x41), false); // A assert_equal(u0.contains(0x127), true); // ħ - USet uempty; + SimpleUSet uempty; assert_equal(uempty.contains(0x62), false); assert_equal(uempty.contains(0x127), false); diff --git a/core/tests/unit/ldml/test_transforms.cpp b/core/tests/unit/ldml/test_transforms.cpp index 7097bcb64b..db51b19ac3 100644 --- a/core/tests/unit/ldml/test_transforms.cpp +++ b/core/tests/unit/ldml/test_transforms.cpp @@ -40,7 +40,7 @@ test_transforms() { std::cout << __FILE__ << ":" << __LINE__ << " - basic " << std::endl; { // start with one - transform_entry te(std::u32string(U"e^"), std::u32string(U"E")); // keep it simple + transform_entry te(std::u32string(U"e\\^"), std::u32string(U"E")); // keep it simple // OK now make a group do it transforms tr; transform_group st; @@ -166,7 +166,7 @@ test_reorder_standalone() { COMP_KMXPLUS_USET_RANGE(0x1A75, 0x1A79)}; const COMP_KMXPLUS_USET_USET usets[] = {{0, 1, 0xFFFFFFFF}}; const COMP_KMXPLUS_USET_USET &toneMarksUset = usets[0]; - const USet toneMarks(&ranges[toneMarksUset.range], toneMarksUset.count); + const SimpleUSet toneMarks(&ranges[toneMarksUset.range], toneMarksUset.count); // validate that the range [1A75, 1A79] matches assert_equal(toneMarks.contains(0x1A76), true); assert_equal(toneMarks.contains(0x1A60), false); diff --git a/windows/src/engine/keyman32/keyman32.vcxproj b/windows/src/engine/keyman32/keyman32.vcxproj index 670df3c1d4..8688133787 100644 --- a/windows/src/engine/keyman32/keyman32.vcxproj +++ b/windows/src/engine/keyman32/keyman32.vcxproj @@ -55,11 +55,11 @@ - $(ProjectDir)..\..\..\..\core\build\x86\$(Configuration)\src;$(ProjectDir)..\..\..\..\core\build\rust\x86\$(Configuration);$(LibraryPath) + $(ProjectDir)..\..\..\..\core\build\x86\$(Configuration)\src;$(ProjectDir)..\..\..\..\core\build\x86\$(Configuration)\subprojects\icu\source\common;$(ProjectDir)..\..\..\..\core\build\x86\$(Configuration)\subprojects\icu\source\i18n;$(ProjectDir)..\..\..\..\core\build\rust\x86\$(Configuration);$(LibraryPath) $(ProjectDir)..\..\..\..\common\include;$(ProjectDir)..\..\..\..\core\include;$(ProjectDir)..\..\..\..\core\build\x86\$(Configuration)\include;$(IncludePath) - $(ProjectDir)..\..\..\..\core\build\x86\$(Configuration)\src;$(ProjectDir)..\..\..\..\core\build\rust\x86\$(Configuration);$(LibraryPath) + $(ProjectDir)..\..\..\..\core\build\x86\$(Configuration)\src;$(ProjectDir)..\..\..\..\core\build\x86\$(Configuration)\subprojects\icu\source\common;$(ProjectDir)..\..\..\..\core\build\x86\$(Configuration)\subprojects\icu\source\i18n;$(ProjectDir)..\..\..\..\core\build\rust\x86\$(Configuration);$(LibraryPath) $(ProjectDir)..\..\..\..\common\include;$(ProjectDir)..\..\..\..\core\include;$(ProjectDir)..\..\..\..\core\build\x86\$(Configuration)\include;$(IncludePath) @@ -96,7 +96,7 @@ /verbose:lib /section:.SHARDATA,rws %(AdditionalOptions) - libkmnkbp0.a;psapi.lib;rpcrt4.lib;version.lib;setupapi.lib;iphlpapi.lib;imm32.lib;crypt32.lib;wintrust.lib;imagehlp.lib;ws2_32.lib;%(AdditionalDependencies) + libicuuc.a;libicuin.a;libkmnkbp0.a;psapi.lib;rpcrt4.lib;version.lib;setupapi.lib;iphlpapi.lib;imm32.lib;crypt32.lib;wintrust.lib;imagehlp.lib;ws2_32.lib;%(AdditionalDependencies) true keyman32.def true @@ -152,7 +152,7 @@ /verbose:lib /section:.SHARDATA,rws %(AdditionalOptions) - libkmnkbp0.a;psapi.lib;rpcrt4.lib;version.lib;setupapi.lib;iphlpapi.lib;imm32.lib;crypt32.lib;wintrust.lib;imagehlp.lib;ws2_32.lib;%(AdditionalDependencies) + libicuuc.a;libicuin.a;libkmnkbp0.a;psapi.lib;rpcrt4.lib;version.lib;setupapi.lib;iphlpapi.lib;imm32.lib;crypt32.lib;wintrust.lib;imagehlp.lib;ws2_32.lib;%(AdditionalDependencies) true keyman32.def true diff --git a/windows/src/engine/keyman64/keyman64.vcxproj b/windows/src/engine/keyman64/keyman64.vcxproj index 15546caef5..89b7ffa2e1 100644 --- a/windows/src/engine/keyman64/keyman64.vcxproj +++ b/windows/src/engine/keyman64/keyman64.vcxproj @@ -54,11 +54,11 @@ - $(ProjectDir)..\..\..\..\core\build\x64\$(Configuration)\src;$(ProjectDir)..\..\..\..\core\build\rust\x64\$(Configuration);$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64) + $(ProjectDir)..\..\..\..\core\build\x64\$(Configuration)\src;$(ProjectDir)..\..\..\..\core\build\x64\$(Configuration)\subprojects\icu\source\common;$(ProjectDir)..\..\..\..\core\build\x64\$(Configuration)\subprojects\icu\source\i18n;$(ProjectDir)..\..\..\..\core\build\rust\x64\$(Configuration);$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64) $(ProjectDir)..\..\..\..\common\include;$(ProjectDir)..\..\..\..\core\build\x64\$(Configuration)\include;$(ProjectDir)..\..\..\..\core\include;$(IncludePath) - $(ProjectDir)..\..\..\..\core\build\x64\$(Configuration)\src;$(ProjectDir)..\..\..\..\core\build\rust\x64\$(Configuration);$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64) + $(ProjectDir)..\..\..\..\core\build\x64\$(Configuration)\src;$(ProjectDir)..\..\..\..\core\build\x64\$(Configuration)\subprojects\icu\source\common;$(ProjectDir)..\..\..\..\core\build\x64\$(Configuration)\subprojects\icu\source\i18n;$(ProjectDir)..\..\..\..\core\build\rust\x64\$(Configuration);$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64) $(ProjectDir)..\..\..\..\common\include;$(ProjectDir)..\..\..\..\core\build\x64\$(Configuration)\include;$(ProjectDir)..\..\..\..\core\include;$(IncludePath) @@ -96,7 +96,7 @@ /verbose:lib /section:.SHARDATA,rws %(AdditionalOptions) - libkmnkbp0.a;psapi.lib;version.lib;setupapi.lib;iphlpapi.lib;imm32.lib;crypt32.lib;wintrust.lib;imagehlp.lib;ws2_32.lib;libcmt.lib;%(AdditionalDependencies) + libicuuc.a;libicuin.a;libkmnkbp0.a;psapi.lib;version.lib;setupapi.lib;iphlpapi.lib;imm32.lib;crypt32.lib;wintrust.lib;imagehlp.lib;ws2_32.lib;libcmt.lib;%(AdditionalDependencies) true C:\Program Files\Microsoft SDKs\Windows\v7.0\Lib\x64;$(VCInstallDir)lib\amd64;$(VCInstallDir)lib;%(AdditionalLibraryDirectories) false @@ -152,7 +152,7 @@ /verbose:lib /section:.SHARDATA,rws %(AdditionalOptions) - libkmnkbp0.a;psapi.lib;version.lib;setupapi.lib;iphlpapi.lib;imm32.lib;crypt32.lib;wintrust.lib;imagehlp.lib;ws2_32.lib;libcmt.lib;%(AdditionalDependencies) + libicuuc.a;libicuin.a;libkmnkbp0.a;psapi.lib;version.lib;setupapi.lib;iphlpapi.lib;imm32.lib;crypt32.lib;wintrust.lib;imagehlp.lib;ws2_32.lib;libcmt.lib;%(AdditionalDependencies) true C:\Program Files\Microsoft SDKs\Windows\v7.0\Lib\x64;$(VCInstallDir)lib\amd64;$(VCInstallDir)lib;%(AdditionalLibraryDirectories) false