Merge pull request #9440 from keymanapp/feat/core/9121-regex-epic-ldml

This commit is contained in:
Steven R. Loomis 2023-08-16 07:12:54 -05:00 committed by GitHub
commit 233c0d71d3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 321 additions and 142 deletions

View file

@ -1072,16 +1072,16 @@ COMP_KMXPLUS_USET_Helper::setUset(const COMP_KMXPLUS_USET *newUset) {
return is_valid;
}
USet::USet(const COMP_KMXPLUS_USET_RANGE *newRange, size_t newCount) {
SimpleUSet::SimpleUSet(const COMP_KMXPLUS_USET_RANGE *newRange, size_t newCount) {
for (size_t i = 0; i < newCount; i++) {
ranges.emplace_back(newRange[i].start, newRange[i].end);
}
}
USet::USet() {
SimpleUSet::SimpleUSet() {
}
bool USet::contains(km_kbp_usv ch) const {
bool SimpleUSet::contains(km_kbp_usv ch) const {
for (const auto &range : ranges) {
if (range.start <= ch && range.end >= ch) {
return true;
@ -1091,7 +1091,7 @@ bool USet::contains(km_kbp_usv ch) const {
}
bool
USet::valid() const {
SimpleUSet::valid() const {
// double check
for (const auto &range : ranges) {
if (!Uni_IsValid(range.start, range.end)) {
@ -1103,7 +1103,7 @@ USet::valid() const {
}
void
USet::dump() const {
SimpleUSet::dump() const {
DebugLog(" - USet size=%d", ranges.size());
for (const auto &range : ranges) {
if (range.start == range.end) {
@ -1114,14 +1114,14 @@ USet::dump() const {
}
}
USet
SimpleUSet
COMP_KMXPLUS_USET_Helper::getUset(KMXPLUS_USET i) const {
if (!valid() || i >= uset->usetCount) {
assert(false);
return USet(nullptr, 0); // empty set
return SimpleUSet(nullptr, 0); // empty set
}
auto &set = usets[i];
return USet(getRange(set.range), set.count);
return SimpleUSet(getRange(set.range), set.count);
}
const COMP_KMXPLUS_USET_RANGE *

View file

@ -699,14 +699,16 @@ struct COMP_KMXPLUS_USET_RANGE {
};
/**
* represents one of the uset elements
* represents one of the uset elements.
* TODO-LDML: replace this with a real icu::UnicodeSet? or at least
* a function producing the same?
*/
class USet {
class SimpleUSet {
public:
/** construct a set over the specified range. Data is copied. */
USet(const COMP_KMXPLUS_USET_RANGE* newStart, size_t newCount);
SimpleUSet(const COMP_KMXPLUS_USET_RANGE* newStart, size_t newCount);
/** empty set */
USet();
SimpleUSet();
/** true if the uset contains this char */
bool contains(km_kbp_usv ch) const;
/** debugging */
@ -726,7 +728,7 @@ public:
bool setUset(const COMP_KMXPLUS_USET *newUset);
inline bool valid() const { return is_valid; }
USet getUset(KMXPLUS_USET list) const;
SimpleUSet getUset(KMXPLUS_USET list) const;
const COMP_KMXPLUS_USET_RANGE *getRange(KMX_DWORD index) const;
private:

View file

@ -172,6 +172,19 @@ u16string_to_u32string(const std::u16string &source) {
return out;
}
inline std::u16string
u32string_to_u16string(const std::u32string &source) {
std::u16string out;
char16_single ch;
for (auto c : source) {
const auto len = Utf32CharToUtf16(c, ch);
for (auto i = 0; i < len; i++) {
out.push_back(ch.ch[i]);
}
}
return out;
}
inline bool Uni_IsEndOfPlaneNonCharacter(km_kbp_usv ch) {
return (((ch) & Uni_FFFE_NONCHARACTER) == Uni_FFFE_NONCHARACTER); // matches FFFF or FFFE
}

View file

@ -11,7 +11,6 @@
#include <string>
#include "kmx/kmx_xstring.h"
#ifndef assert
#define assert(x) // TODO-LDML
#endif
@ -36,7 +35,7 @@ namespace ldml {
#define DebugTran(msg, ...)
#endif
element::element(const USet &new_u, KMX_DWORD new_flags)
element::element(const SimpleUSet &new_u, KMX_DWORD new_flags)
: chr(), uset(new_u), flags((new_flags & ~LDML_ELEM_FLAGS_TYPE) | LDML_ELEM_FLAGS_TYPE_USET) {
}
@ -191,7 +190,7 @@ element_list::load(const kmx::kmx_plus &kplus, kmx::KMXPLUS_ELEM id) {
km_kbp_usv ch = e.element;
emplace_back(ch, flags); // char
} else if (type == LDML_ELEM_FLAGS_TYPE_USET) {
// need to load a USet
// need to load a SimpleUSet
auto u = kplus.usetHelper.getUset(e.element);
if (!u.valid()) {
DebugLog("Error, invalid UnicodeSet at element %d", (int)i);
@ -393,35 +392,96 @@ reorder_group::apply(std::u32string &str) const {
return applied;
}
transform_entry::transform_entry(const transform_entry &other) :
fFrom(other.fFrom), fTo(other.fTo), fFromPattern(nullptr) {
if (other.fFromPattern) {
// clone pattern
fFromPattern.reset(other.fFromPattern->clone());
}
}
transform_entry::transform_entry(const std::u32string &from, const std::u32string &to) : fFrom(from), fTo(to) {
assert(!fFrom.empty()); // TODO-LDML: should not happen?
if (!fFrom.empty()) {
const std::u16string patstr = km::kbp::kmx::u32string_to_u16string(fFrom);
UErrorCode status = U_ZERO_ERROR;
/* const */ icu::UnicodeString patustr = icu::UnicodeString(patstr.data(), (int32_t)patstr.length());
// add '$' to match to end
patustr.append(u'$');
fFromPattern.reset(icu::RegexPattern::compile(patustr, 0, status));
assert(U_SUCCESS(status)); // TODO-LDML: may be best to propagate status up ^^
}
}
size_t
transform_entry::match(const std::u32string &input) const {
if (input.length() < fFrom.length()) {
// TODO-LDML: regex
// Too small, can't match.
return 0;
}
// string at end
auto substr = input.substr(input.length() - fFrom.length(), fFrom.length());
if (substr != fFrom) {
// end of string doesn't match
return 0;
}
// match length == fFrom.length
return substr.length();
}
transform_entry::apply(const std::u32string &input, std::u32string &output) const {
assert(fFromPattern);
// TODO-LDML: Really? can't go from u32 to UnicodeString?
// TODO-LDML: Also, we could cache the u16 string at the transformGroup level or higher.
UErrorCode status = U_ZERO_ERROR;
const std::u16string matchstr = km::kbp::kmx::u32string_to_u16string(input);
icu::UnicodeString matchustr = icu::UnicodeString(matchstr.data(), (int32_t)matchstr.length());
// TODO-LDML: create a new Matcher every time. These could be cached and reset.
std::unique_ptr<icu::RegexMatcher> matcher(fFromPattern->matcher(matchustr, status));
assert(U_SUCCESS(status));
std::u32string
transform_entry::apply(const std::u32string & /*input*/, size_t /*matchLen*/) const {
// TODO-LDML: regex
// For now, we just return the 'to' string literally.
return fTo;
if (!matcher->find(status)) { // i.e. matches somewhere, in this case at end of str
return 0; // no match
}
// TODO-LDML: this is UTF-16 len, not UTF-32 len!!
// TODO-LDML: if we had an underlying UText this would be simpler.
int32_t matchStart = matcher->start(status);
int32_t matchEnd = matcher->end(status);
assert(U_SUCCESS(status));
// extract..
const icu::UnicodeString substr = matchustr.tempSubStringBetween(matchStart, matchEnd);
// preflight to UTF-32 to get length
UErrorCode substrStatus = U_ZERO_ERROR;
auto matchLen = substr.toUTF32(nullptr, 0, substrStatus);
assert(matchLen > 0);
if (matchLen == 0) {
return 0;
}
// Now, we have a matchLen.
// now, do the replace.
// Convert the fTo into u16 TODO-LDML (we could cache this?)
const std::u16string rstr = km::kbp::kmx::u32string_to_u16string(fTo);
icu::UnicodeString rustr = icu::UnicodeString(rstr.data(), (int32_t)rstr.length());
// This replace will apply $1, $2 etc. TODO-LDML it will NOT handle mapFrom or mapTo.
icu::UnicodeString entireOutput = matcher->replaceFirst(rustr, status);
assert(U_SUCCESS(status));
// entireOutput includes all of 'input', but modified. Need to substring it.
icu::UnicodeString outu = entireOutput.tempSubString(matchStart);
// Special case if there's no output
if (outu.length() == 0) {
output.clear();
} else {
// TODO-LDML: All we are trying to do is to extract the output string. Probably too many steps.
UErrorCode preflightStatus = U_ZERO_ERROR;
// calculate how big the buffer is
auto out32len = outu.toUTF32(nullptr, 0, preflightStatus); // preflightStatus will be an err, because we know the buffer overruns zero bytes
// allocate
char32_t *s = new char32_t[out32len + 1];
assert(s != nullptr);
// convert
outu.toUTF32((UChar32 *)s, out32len + 1, status);
assert(U_SUCCESS(status));
output.assign(s, out32len);
// now, build a u32string
std::u32string out32(s, out32len);
// clean up buffer
delete [] s;
}
return matchLen;
}
any_group::any_group(const transform_group &g) : type(any_group_type::transform), transform(g), reorder() {
}
any_group::any_group(const reorder_group &g) : type(any_group_type::reorder), transform(), reorder(g) {
}
@ -444,17 +504,18 @@ transform_group::transform_group() {
/**
* return the first transform match in this group
*/
const transform_entry *
transform_group::match(const std::u32string &input, size_t &subMatched) const {
size_t
transform_group::apply(const std::u32string &input, std::u32string &output) const {
size_t subMatched = 0;
for (auto transform = begin(); (subMatched == 0) && (transform < end()); transform++) {
// TODO-LDML: non regex implementation
// is the match area too short?
subMatched = transform->match(input);
subMatched = transform->apply(input, output);
if (subMatched != 0) {
return &(*transform); // return alias to transform
return subMatched; // matched. break out.
}
}
return nullptr;
return 0; // no match
}
/**
@ -507,20 +568,14 @@ transforms::apply(const std::u32string &input, std::u32string &output) {
// TODO-LDML: reorders
// Assume it's a non reorder group
/** Length of match within this group*/
size_t subMatched = 0;
// find the first match in this group (if present)
// TODO-LDML: check if reorder
if (group->type == any_group_type::transform) {
auto entry = group->transform.match(updatedInput, subMatched);
if (entry != nullptr) {
// now apply the found transform
// update subOutput (string) and subMatched
// the returned string must replace the last "subMatched" chars of the string.
std::u32string subOutput = entry->apply(updatedInput, subMatched);
std::u32string subOutput;
size_t subMatched = group->transform.apply(updatedInput, subOutput);
if (subMatched != 0) {
// remove the matched part of the updatedInput
updatedInput.resize(updatedInput.length() - subMatched); // chop of the subMatched part at end
updatedInput.append(subOutput); // subOutput could be empty such as in backspace transform

View file

@ -14,11 +14,23 @@
#include <unordered_map>
#include <utility>
#if !defined(HAVE_ICU4C)
#error icu4c is required for this code
#endif
#define U_FALLTHROUGH
#include "unicode/utypes.h"
#include "unicode/uniset.h"
#include "unicode/usetiter.h"
#include "unicode/unistr.h"
#include "unicode/regex.h"
#include "unicode/utext.h"
namespace km {
namespace kbp {
namespace ldml {
using km::kbp::kmx::USet;
using km::kbp::kmx::SimpleUSet;
/**
* Type of a group
@ -33,12 +45,11 @@ enum any_group_type {
*/
class element {
public:
/** construct from a USet */
element(const USet &u, KMX_DWORD flags);
/** construct from a SimpleUSet */
element(const SimpleUSet &u, KMX_DWORD flags);
/** construct from a single char */
element(km_kbp_usv ch, KMX_DWORD flags);
/** @returns true if a USet type */
/** @returns true if a SimpleUSet type */
bool is_uset() const;
/** @returns true if prebase bit set*/
bool is_prebase() const;
@ -58,7 +69,7 @@ public:
private:
// TODO-LDML: support multi-char strings?
const km_kbp_usv chr;
const USet uset;
const SimpleUSet uset;
const KMX_DWORD flags;
};
@ -67,6 +78,7 @@ private:
*/
class transform_entry {
public:
transform_entry(const transform_entry &other);
transform_entry(
const std::u32string &from,
const std::u32string &to
@ -74,18 +86,17 @@ public:
);
/**
* @returns length if it's a match
* If matching, apply the match to the output string
* @param input input string to match
* @param output output string
* @returns length of 'input' which was matched
*/
size_t match(const std::u32string &input) const;
/**
* @returns output string
*/
std::u32string apply(const std::u32string &input, size_t matchLen) const;
size_t apply(const std::u32string &input, std::u32string &output) const;
private:
const std::u32string fFrom; // TODO-LDML: regex
const std::u32string fFrom;
const std::u32string fTo;
std::unique_ptr<icu::RegexPattern> fFromPattern;
};
/**
@ -101,12 +112,12 @@ public:
transform_group();
/**
* Find the first match in the group
* Find the first match in the group and apply it.
* @param input input string to match
* @param subMatched on output, the matched length
* @returns alias to transform_entry or nullptr
* @param output output string
* @returns length of 'input' which was matched
*/
const transform_entry *match(const std::u32string &input, size_t &subMatched) const;
size_t apply(const std::u32string &input, std::u32string &output) const;
};
/** a single char, categorized according to reorder rules*/
@ -228,7 +239,7 @@ public:
size_t apply(const std::u32string &input, std::u32string &output);
/**
* For tests - TODO-LDML only supports reorder
* For tests
* @return true if str was altered
*/
bool apply(std::u32string &str);

View file

@ -21,6 +21,26 @@ if cpp_compiler.get_id() == 'emscripten'
defns += ['-DKMN_KBP']
endif
# ICU4C is used for repertoire tests and core implementation
if target_machine.system() == 'linux'
# use pkg-config when targetting linux
pkgconfig = 'pkg-config'
icu_uc = dependency('icu-uc', required: true)
icu_i18n = dependency('icu-i18n', required: true)
else
# load ICU from wrap
icu4c = subproject('icu-minimal', default_options: [ 'default_library=static', 'cpp_std=c++17', 'warning_level=0',
'werror=false']) # TODO-LDML: options: static, no data (these are in the meson build files)
icu_uc = icu4c.get_variable('icuuc_dep')
icu_i18n = icu4c.get_variable('icui18n_dep')
endif
if icu_uc.found()
defns += '-DHAVE_ICU4C'
endif
kmx_files = files(
'option.cpp',
'keyboard.cpp',
@ -85,11 +105,13 @@ lib = library('kmnkbp0',
version: lib_version,
include_directories: inc,
pic: true,
install: true)
install: true,
dependencies: [icu_uc, icu_i18n],
)
headerdirs = [ '.', 'keyman' ] # subdirectories of ${prefix}/include to add to header path
kmnkbp = declare_dependency(link_with: lib, include_directories: inc)
kmnkbp = declare_dependency(link_with: lib, include_directories: inc, dependencies: [icu_uc, icu_i18n])
pkg = import('pkgconfig')
pkg.generate(

View file

@ -10,5 +10,4 @@ patch_directory = icu
[provide]
icu-uc = icuuc_dep
# TODO-LDML: not including i18n at present
#icu-i18n = icui18n_dep
icu-i18n = icui18n_dep

View file

@ -26,9 +26,8 @@ endif
# per icudefs.mk.in:
# "U_ATTRIBUTE_DEPRECATED is defined to hide warnings about deprecated API warnings."
add_project_arguments('-DU_ATTRIBUTE_DEPRECATED=', '-DUCONFIG_USE_LOCAL=1', language: 'c')
add_project_arguments('-DU_NOEXCEPT=', language: 'c')
add_project_arguments('-DU_NOEXCEPT=', '-DU_ATTRIBUTE_DEPRECATED=', '-DUCONFIG_USE_LOCAL=1', language: 'c')
add_project_arguments('-DU_NOEXCEPT=', '-DU_ATTRIBUTE_DEPRECATED=', '-DUCONFIG_USE_LOCAL=1', language: 'cpp')
uconfig = configuration_data()
@ -42,8 +41,12 @@ uconfig.set('UCONFIG_NO_IDNA', 1)
uconfig.set('UCONFIG_NO_COLLATION', 1)
uconfig.set('UCONFIG_NO_FORMATTING', 1)
uconfig.set('UCONFIG_NO_TRANSLITERATION', 1)
uconfig.set('UCONFIG_NO_REGULAR_EXPRESSIONS', 1) # TODO-LDML: Will probably want this for transforms #7375
uconfig.set('UCONFIG_NO_REGULAR_EXPRESSIONS', 0) # want these for transforms #7375
uconfig.set('UCONFIG_NO_SERVICE', 1)
uconfig.set('U_OVERRIDE_CXX_ALLOCATION', 1)
uconfig.set('UCONFIG_NO_CONVERSION', 1)
uconfig.set('UCONFIG_USE_WINDOWS_LCID_MAPPING_API', 0)
uconfig_local = configure_file(

View file

@ -75,10 +75,10 @@ sources = files(
'stringpiece.cpp',
'stringtriebuilder.cpp',
'uarrsort.cpp',
'ubidi.cpp',
# 'ubidi.cpp',
'ubidi_props.cpp',
'ubidiln.cpp',
'ubidiwrt.cpp',
# 'ubidiln.cpp',
# 'ubidiwrt.cpp',
'ubrk.cpp',
'ucase.cpp',
'ucasemap.cpp',
@ -91,29 +91,29 @@ sources = files(
'uchriter.cpp',
'ucln_cmn.cpp',
'ucmndata.cpp',
'ucnv.cpp',
'ucnv2022.cpp',
'ucnv_bld.cpp',
'ucnv_cb.cpp',
'ucnv_cnv.cpp',
'ucnv_ct.cpp',
'ucnv_err.cpp',
'ucnv_ext.cpp',
'ucnv_io.cpp',
'ucnv_lmb.cpp',
'ucnv_set.cpp',
'ucnv_u16.cpp',
'ucnv_u32.cpp',
'ucnv_u7.cpp',
'ucnv_u8.cpp',
'ucnvbocu.cpp',
'ucnvdisp.cpp',
'ucnvhz.cpp',
'ucnvisci.cpp',
'ucnvlat1.cpp',
'ucnvmbcs.cpp',
'ucnvscsu.cpp',
'ucnvsel.cpp',
# 'ucnv.cpp',
# 'ucnv2022.cpp',
# 'ucnv_bld.cpp',
# 'ucnv_cb.cpp',
# 'ucnv_cnv.cpp',
# 'ucnv_ct.cpp',
# 'ucnv_err.cpp',
# 'ucnv_ext.cpp',
# 'ucnv_io.cpp',
# 'ucnv_lmb.cpp',
# 'ucnv_set.cpp',
# 'ucnv_u16.cpp',
# 'ucnv_u32.cpp',
# 'ucnv_u7.cpp',
# 'ucnv_u8.cpp',
# 'ucnvbocu.cpp',
# 'ucnvdisp.cpp',
# 'ucnvhz.cpp',
# 'ucnvisci.cpp',
# 'ucnvlat1.cpp',
# 'ucnvmbcs.cpp',
# 'ucnvscsu.cpp',
# 'ucnvsel.cpp',
'ucol_swp.cpp',
'ucptrie.cpp',
'ucurr.cpp',
@ -287,8 +287,8 @@ headers = files(
'unicode/stringpiece.h',
'unicode/stringtriebuilder.h',
'unicode/symtable.h',
'unicode/ubidi.h',
'unicode/ubiditransform.h',
# 'unicode/ubidi.h',
# 'unicode/ubiditransform.h',
'unicode/ubrk.h',
'unicode/ucasemap.h',
'unicode/ucat.h',

View file

@ -52,7 +52,7 @@ sources = files(
'dayperiodrules.cpp',
'dcfmtsym.cpp',
'decContext.cpp',
'decNumber.cpp',
# 'decNumber.cpp',
'decimfmt.cpp',
'displayoptions.cpp',
'double-conversion-bignum-dtoa.cpp',

View file

@ -9,8 +9,7 @@ endif
subdir('stubdata')
subdir('common')
# TODO-LDML: Not used now, will need for regex
# subdir('i18n')
subdir('i18n')
## Note: The following subdirs are not used by Keyman (at present)

View file

@ -37,6 +37,7 @@ foreach t : tests
cpp_args: local_defns + defns + warns,
include_directories: [inc, libsrc],
link_args: links + tests_flags,
dependencies: [icu_uc, icu_i18n],
objects: lib.extract_all_objects(recursive: false))
test(t[0], bin, args: ['--color', test_path])

View file

@ -1329,6 +1329,35 @@ test_u16string_to_u32string() {
}
}
void
test_u32string_to_u16string() {
std::cout << "== " << __FUNCTION__ << std::endl;
// normal cases
{
const auto str = u32string_to_u16string(U"");
assert_equal(str.length(), 0);
}
{
const auto str = u32string_to_u16string(U"e");
assert_equal(str.length(), 1);
assert_equal(str.at(0), 0x0065);
}
{
const auto str = u32string_to_u16string(U"🙀");
assert_equal(str.length(), 2);
assert_equal(str.at(0), 0xD83D);
assert_equal(str.at(1), 0xDE40);
}
{
const auto str = u32string_to_u16string(U"Ω🙀");
assert_equal(str.length(), 3);
assert_equal(str.at(0), u'Ω');
assert_equal(str.at(1), 0xD83D);
assert_equal(str.at(2), 0xDE40);
}
}
void test_is_valid() {
std::cout << "== " << __FUNCTION__ << std::endl;
// valid

View file

@ -31,6 +31,7 @@ kmx = executable('kmx',
cpp_args: defns + warns,
include_directories: [inc, libsrc, '../../kmx_test_source'],
link_args: links + tests_flags,
dependencies: [icu_uc, icu_i18n],
objects: [lib.extract_all_objects(recursive: false), kmx_test_source_lib.extract_all_objects(recursive: false)])
tests = [
@ -178,6 +179,7 @@ key_e = executable('key_list', ['kmx_key_list.cpp', '../emscripten_filesystem.cp
cpp_args: defns + warns,
include_directories: [inc, libsrc],
link_args: links + tests_flags,
dependencies: [icu_uc, icu_i18n],
objects: lib.extract_all_objects(recursive: false))
test_kbd = 'kmx_key_list'
@ -196,6 +198,7 @@ imx_e = executable('imx_list', ['kmx_imx.cpp', '../emscripten_filesystem.cpp'],
cpp_args: defns + warns,
include_directories: [inc, libsrc],
link_args: links + tests_flags,
dependencies: [icu_uc, icu_i18n],
objects: lib.extract_all_objects(recursive: false))
test_kbd = 'kmx_imsample'
@ -213,6 +216,7 @@ external_e = executable('ext_event', ['kmx_external_event.cpp', '../emscripten_f
cpp_args: defns + warns,
include_directories: [inc, libsrc],
link_args: links + tests_flags,
dependencies: [icu_uc, icu_i18n],
objects: lib.extract_all_objects(recursive: false))
test_kbd = 'k_033___caps_always_off'

View file

@ -45,7 +45,7 @@ from https://github.com/unicode-org/cldr/blob/keyboard-preview/docs/ldml/tr35-ke
<transforms type="simple">
<transformGroup>
<transform from="^${e}" to="ê"/>
<transform from="\^${e}" to="ê"/> <!-- TODO-LDML: ^ should not need escape here ideally -->
</transformGroup>
<transformGroup>
<!-- testing -->

View file

@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE keyboardTest SYSTEM "../../../../../resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboardTest.dtd">
<keyboardTest conformsTo="techpreview">
<info keyboard="k_007_transform_rgx.xml" author="Team Keyboard" name="marker" />
<tests name="regex-tests">
<test name="regex-test-basic">
<startContext to="" />
<keystroke key="a" />
<keystroke key="grave" />
<keystroke key="e" />
<check result="e`a" />
</test>
</tests>
</keyboardTest>

View file

@ -0,0 +1,42 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
from https://github.com/unicode-org/cldr/blob/keyboard-preview/docs/ldml/tr35-keyboards.md#element-transform
-->
<!DOCTYPE keyboard SYSTEM "../../../../../resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboard.dtd">
<keyboard locale="en" conformsTo="techpreview">
<info author="srl295" indicator="🙀" layout="qwerty" normalization="NFC" />
<names>
<name value="k_007_transform_rgx" />
<name value="Regex Transform Test" />
</names>
<keys>
<key id="grave" to="`"/>
</keys>
<layers form="us">
<layer modifier="none" id="base">
<row keys="grave 1 2 3 4 5 6 7 8 9 0" />
<row keys="q w e r t y u i o p" />
<row keys="a s d f g h j k l" />
<row keys="z x c v b n m" />
<row keys="space" />
</layer>
<layer modifier="shift" id="shift">
<row keys="grave 1 2 3 4 5 6 7 8 9 0" />
<row keys="Q W E R T Y U I O P" />
<row keys="A S D F G H J K L" />
<row keys="Z X C V B N M" />
<row keys="space" />
</layer>
</layers>
<transforms type="simple">
<transformGroup>
<transform from="([abc])`([def])" to="$2`$1"/> <!-- flip the order: a`e -> e`a -->
</transformGroup>
</transforms>
</keyboard>

View file

@ -32,6 +32,7 @@ tests_without_testdata = [
# These tests have a k_001_tiny-test.xml file as well.
tests_with_testdata = [
'k_001_tiny',
'k_007_transform_rgx',
'k_020_fr', # TODO-LDML: move to cldr above (fix vkey)
'k_200_reorder_nod_Lana',
'k_210_marker',

View file

@ -25,7 +25,6 @@
#include "ldml/keyboardprocessor_ldml.h"
#include "ldml/ldml_processor.hpp"
#include "path.hpp"
#include "state.hpp"
#include "utfcodec.hpp"
@ -36,6 +35,7 @@
#if defined(HAVE_ICU4C)
// TODO-LDML: Needed this for some compiler warnings
#define U_FALLTHROUGH
#include "unicode/utypes.h"
#include "unicode/uniset.h"
#include "unicode/usetiter.h"
#else

View file

@ -4,25 +4,6 @@
# Authors: Marc Durdin
#
# ICU4C is used for repertoire tests
if target_machine.system() == 'linux'
# use pkg-config when targetting linux
pkgconfig = 'pkg-config'
icu_uc = dependency('icu-uc', required: true)
else
# load ICU from wrap
# Requires meson of about 0.57+
icu4c = subproject('icu-minimal', default_options: [ 'default_library=static', 'cpp_std=c++17', 'warning_level=0',
'werror=false']) # TODO-LDML: options: static, no data (these are in the meson build files)
icu_uc = icu4c.get_variable('icuuc_dep')
endif
if icu_uc.found()
defns += '-DHAVE_ICU4C'
endif
# TODO -- why are these differing from the standard.meson.build flags?
if cpp_compiler.get_id() == 'gcc' or cpp_compiler.get_id() == 'clang' or cpp_compiler.get_id() == 'emscripten'
warns = [
@ -75,8 +56,9 @@ ldml = executable('ldml',
cpp_args: defns + warns,
include_directories: [inc, libsrc, '../../../../developer/src/ext/json'],
link_args: links + tests_flags,
dependencies: [icu_uc, icu_i18n],
# link_with: [lib],
objects: lib.extract_all_objects(recursive: false),
dependencies: [icu_uc],
)
# Run tests on all keyboards (`tests` defined in keyboards/meson.build)
@ -103,6 +85,7 @@ e = executable('test_kmx_plus', 'test_kmx_plus.cpp',
cpp_args: defns + warns,
include_directories: [inc, libsrc, '../../../../developer/src/ext/json'],
link_args: links + tests_flags,
dependencies: [icu_uc, icu_i18n],
objects: lib.extract_all_objects(recursive: false))
test('test_kmx_plus', e, suite: 'ldml')
@ -110,5 +93,6 @@ t = executable('test_transforms', 'test_transforms.cpp',
cpp_args: defns + warns,
include_directories: [inc, libsrc, '../../../../developer/src/ext/json'],
link_args: links + tests_flags,
dependencies: [icu_uc, icu_i18n],
objects: lib.extract_all_objects(recursive: false))
test('test_transforms', t, suite: 'ldml')

View file

@ -152,12 +152,12 @@ int test_uset() {
{0x127, 0x127} // [ħ]
};
USet u0(&r[0], 2);
SimpleUSet u0(&r[0], 2);
assert_equal(u0.contains(0x62), true); // b
assert_equal(u0.contains(0x41), false); // A
assert_equal(u0.contains(0x127), true); // ħ
USet uempty;
SimpleUSet uempty;
assert_equal(uempty.contains(0x62), false);
assert_equal(uempty.contains(0x127), false);

View file

@ -40,7 +40,7 @@ test_transforms() {
std::cout << __FILE__ << ":" << __LINE__ << " - basic " << std::endl;
{
// start with one
transform_entry te(std::u32string(U"e^"), std::u32string(U"E")); // keep it simple
transform_entry te(std::u32string(U"e\\^"), std::u32string(U"E")); // keep it simple
// OK now make a group do it
transforms tr;
transform_group st;
@ -166,7 +166,7 @@ test_reorder_standalone() {
COMP_KMXPLUS_USET_RANGE(0x1A75, 0x1A79)};
const COMP_KMXPLUS_USET_USET usets[] = {{0, 1, 0xFFFFFFFF}};
const COMP_KMXPLUS_USET_USET &toneMarksUset = usets[0];
const USet toneMarks(&ranges[toneMarksUset.range], toneMarksUset.count);
const SimpleUSet toneMarks(&ranges[toneMarksUset.range], toneMarksUset.count);
// validate that the range [1A75, 1A79] matches
assert_equal(toneMarks.contains(0x1A76), true);
assert_equal(toneMarks.contains(0x1A60), false);

View file

@ -55,11 +55,11 @@
<CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" />
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LibraryPath>$(ProjectDir)..\..\..\..\core\build\x86\$(Configuration)\src;$(ProjectDir)..\..\..\..\core\build\rust\x86\$(Configuration);$(LibraryPath)</LibraryPath>
<LibraryPath>$(ProjectDir)..\..\..\..\core\build\x86\$(Configuration)\src;$(ProjectDir)..\..\..\..\core\build\x86\$(Configuration)\subprojects\icu\source\common;$(ProjectDir)..\..\..\..\core\build\x86\$(Configuration)\subprojects\icu\source\i18n;$(ProjectDir)..\..\..\..\core\build\rust\x86\$(Configuration);$(LibraryPath)</LibraryPath>
<IncludePath>$(ProjectDir)..\..\..\..\common\include;$(ProjectDir)..\..\..\..\core\include;$(ProjectDir)..\..\..\..\core\build\x86\$(Configuration)\include;$(IncludePath)</IncludePath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LibraryPath>$(ProjectDir)..\..\..\..\core\build\x86\$(Configuration)\src;$(ProjectDir)..\..\..\..\core\build\rust\x86\$(Configuration);$(LibraryPath)</LibraryPath>
<LibraryPath>$(ProjectDir)..\..\..\..\core\build\x86\$(Configuration)\src;$(ProjectDir)..\..\..\..\core\build\x86\$(Configuration)\subprojects\icu\source\common;$(ProjectDir)..\..\..\..\core\build\x86\$(Configuration)\subprojects\icu\source\i18n;$(ProjectDir)..\..\..\..\core\build\rust\x86\$(Configuration);$(LibraryPath)</LibraryPath>
<IncludePath>$(ProjectDir)..\..\..\..\common\include;$(ProjectDir)..\..\..\..\core\include;$(ProjectDir)..\..\..\..\core\build\x86\$(Configuration)\include;$(IncludePath)</IncludePath>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
@ -96,7 +96,7 @@
</ResourceCompile>
<Link>
<AdditionalOptions>/verbose:lib /section:.SHARDATA,rws %(AdditionalOptions)</AdditionalOptions>
<AdditionalDependencies>libkmnkbp0.a;psapi.lib;rpcrt4.lib;version.lib;setupapi.lib;iphlpapi.lib;imm32.lib;crypt32.lib;wintrust.lib;imagehlp.lib;ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>libicuuc.a;libicuin.a;libkmnkbp0.a;psapi.lib;rpcrt4.lib;version.lib;setupapi.lib;iphlpapi.lib;imm32.lib;crypt32.lib;wintrust.lib;imagehlp.lib;ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
<SuppressStartupBanner>true</SuppressStartupBanner>
<ModuleDefinitionFile>keyman32.def</ModuleDefinitionFile>
<GenerateDebugInformation>true</GenerateDebugInformation>
@ -152,7 +152,7 @@
</ResourceCompile>
<Link>
<AdditionalOptions>/verbose:lib /section:.SHARDATA,rws %(AdditionalOptions)</AdditionalOptions>
<AdditionalDependencies>libkmnkbp0.a;psapi.lib;rpcrt4.lib;version.lib;setupapi.lib;iphlpapi.lib;imm32.lib;crypt32.lib;wintrust.lib;imagehlp.lib;ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>libicuuc.a;libicuin.a;libkmnkbp0.a;psapi.lib;rpcrt4.lib;version.lib;setupapi.lib;iphlpapi.lib;imm32.lib;crypt32.lib;wintrust.lib;imagehlp.lib;ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
<SuppressStartupBanner>true</SuppressStartupBanner>
<ModuleDefinitionFile>keyman32.def</ModuleDefinitionFile>
<GenerateDebugInformation>true</GenerateDebugInformation>

View file

@ -54,11 +54,11 @@
<CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LibraryPath>$(ProjectDir)..\..\..\..\core\build\x64\$(Configuration)\src;$(ProjectDir)..\..\..\..\core\build\rust\x64\$(Configuration);$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64)</LibraryPath>
<LibraryPath>$(ProjectDir)..\..\..\..\core\build\x64\$(Configuration)\src;$(ProjectDir)..\..\..\..\core\build\x64\$(Configuration)\subprojects\icu\source\common;$(ProjectDir)..\..\..\..\core\build\x64\$(Configuration)\subprojects\icu\source\i18n;$(ProjectDir)..\..\..\..\core\build\rust\x64\$(Configuration);$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64)</LibraryPath>
<IncludePath>$(ProjectDir)..\..\..\..\common\include;$(ProjectDir)..\..\..\..\core\build\x64\$(Configuration)\include;$(ProjectDir)..\..\..\..\core\include;$(IncludePath)</IncludePath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LibraryPath>$(ProjectDir)..\..\..\..\core\build\x64\$(Configuration)\src;$(ProjectDir)..\..\..\..\core\build\rust\x64\$(Configuration);$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64)</LibraryPath>
<LibraryPath>$(ProjectDir)..\..\..\..\core\build\x64\$(Configuration)\src;$(ProjectDir)..\..\..\..\core\build\x64\$(Configuration)\subprojects\icu\source\common;$(ProjectDir)..\..\..\..\core\build\x64\$(Configuration)\subprojects\icu\source\i18n;$(ProjectDir)..\..\..\..\core\build\rust\x64\$(Configuration);$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64)</LibraryPath>
<IncludePath>$(ProjectDir)..\..\..\..\common\include;$(ProjectDir)..\..\..\..\core\build\x64\$(Configuration)\include;$(ProjectDir)..\..\..\..\core\include;$(IncludePath)</IncludePath>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
@ -96,7 +96,7 @@
</ResourceCompile>
<Link>
<AdditionalOptions>/verbose:lib /section:.SHARDATA,rws %(AdditionalOptions)</AdditionalOptions>
<AdditionalDependencies>libkmnkbp0.a;psapi.lib;version.lib;setupapi.lib;iphlpapi.lib;imm32.lib;crypt32.lib;wintrust.lib;imagehlp.lib;ws2_32.lib;libcmt.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>libicuuc.a;libicuin.a;libkmnkbp0.a;psapi.lib;version.lib;setupapi.lib;iphlpapi.lib;imm32.lib;crypt32.lib;wintrust.lib;imagehlp.lib;ws2_32.lib;libcmt.lib;%(AdditionalDependencies)</AdditionalDependencies>
<SuppressStartupBanner>true</SuppressStartupBanner>
<AdditionalLibraryDirectories>C:\Program Files\Microsoft SDKs\Windows\v7.0\Lib\x64;$(VCInstallDir)lib\amd64;$(VCInstallDir)lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
<IgnoreAllDefaultLibraries>false</IgnoreAllDefaultLibraries>
@ -152,7 +152,7 @@
</ResourceCompile>
<Link>
<AdditionalOptions>/verbose:lib /section:.SHARDATA,rws %(AdditionalOptions)</AdditionalOptions>
<AdditionalDependencies>libkmnkbp0.a;psapi.lib;version.lib;setupapi.lib;iphlpapi.lib;imm32.lib;crypt32.lib;wintrust.lib;imagehlp.lib;ws2_32.lib;libcmt.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>libicuuc.a;libicuin.a;libkmnkbp0.a;psapi.lib;version.lib;setupapi.lib;iphlpapi.lib;imm32.lib;crypt32.lib;wintrust.lib;imagehlp.lib;ws2_32.lib;libcmt.lib;%(AdditionalDependencies)</AdditionalDependencies>
<SuppressStartupBanner>true</SuppressStartupBanner>
<AdditionalLibraryDirectories>C:\Program Files\Microsoft SDKs\Windows\v7.0\Lib\x64;$(VCInstallDir)lib\amd64;$(VCInstallDir)lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
<IgnoreAllDefaultLibraries>false</IgnoreAllDefaultLibraries>