mirror of
https://github.com/keymanapp/keyman.git
synced 2026-08-09 02:15:32 +00:00
Merge pull request #9962 from keymanapp/feat/core/9707-tertiary-reorder-epic-ldml
feat(core): ldml tertiary reordering 🙀
This commit is contained in:
Steven R. Loomis
GitHub
commit
6f4def8b4b
8 changed files with 364 additions and 60 deletions
|
|
@ -45,16 +45,16 @@ element::is_uset() const {
|
|||
return (flags & LDML_ELEM_FLAGS_TYPE) == LDML_ELEM_FLAGS_TYPE_USET;
|
||||
}
|
||||
|
||||
signed char
|
||||
reorder_weight
|
||||
element::get_order() const {
|
||||
unsigned char uorder = ((flags & LDML_ELEM_FLAGS_ORDER_MASK) >> LDML_ELEM_FLAGS_ORDER_BITSHIFT);
|
||||
return (signed char)uorder;
|
||||
return (reorder_weight)uorder; // unsigned to signed
|
||||
}
|
||||
|
||||
signed char
|
||||
reorder_weight
|
||||
element::get_tertiary() const {
|
||||
unsigned char uorder = ((flags & LDML_ELEM_FLAGS_TERTIARY_MASK) >> LDML_ELEM_FLAGS_TERTIARY_BITSHIFT);
|
||||
return (signed char)uorder;
|
||||
return (reorder_weight)uorder; // unsigned to signed
|
||||
}
|
||||
|
||||
bool
|
||||
|
|
@ -93,19 +93,19 @@ element::dump() const {
|
|||
|
||||
int
|
||||
reorder_sort_key::compare(const reorder_sort_key &other) const {
|
||||
int primaryResult = (int)primary - (int)other.primary;
|
||||
int secondaryResult = (int)secondary - (int)other.secondary;
|
||||
int tertiaryResult = (int)tertiary - (int)other.tertiary;
|
||||
int quaternaryResult = (int)quaternary - (int)other.quaternary;
|
||||
auto primaryResult = primary - other.primary;
|
||||
auto secondaryResult = secondary - other.secondary;
|
||||
auto tertiaryResult = tertiary - other.tertiary;
|
||||
auto quaternaryResult = quaternary - other.quaternary;
|
||||
|
||||
if (primaryResult) {
|
||||
return primaryResult;
|
||||
return (int)primaryResult;
|
||||
} else if (secondaryResult) {
|
||||
return secondaryResult;
|
||||
return (int)secondaryResult;
|
||||
} else if (tertiaryResult) {
|
||||
return tertiaryResult;
|
||||
return (int)tertiaryResult;
|
||||
} else if (quaternaryResult) {
|
||||
return quaternaryResult;
|
||||
return (int)quaternaryResult;
|
||||
} else {
|
||||
// We don't expect to get here. quaternaryResult is the string index, which
|
||||
// should be unequal.
|
||||
|
|
@ -131,14 +131,14 @@ reorder_sort_key::from(const std::u32string &str) {
|
|||
// construct a 'baseline' sort key, that is, in the absence of
|
||||
// any match rules.
|
||||
std::deque<reorder_sort_key> keylist;
|
||||
auto s = str.begin(); // str iterator
|
||||
size_t c = 0; // str index
|
||||
auto s = str.begin(); // str iterator
|
||||
reorder_weight c = 0; // str index
|
||||
for (auto e = str.begin(); e < str.end(); e++, s++, c++) {
|
||||
// primary weight: 0
|
||||
// seconary weight: c (the string index)
|
||||
// tertiary weight: 0
|
||||
// quaternary weight: c (the index again)
|
||||
keylist.emplace_back(reorder_sort_key{*s, 0, c, 0, c});
|
||||
keylist.emplace_back(reorder_sort_key{*s, 0, c, 0, c, false});
|
||||
}
|
||||
return keylist;
|
||||
}
|
||||
|
|
@ -146,7 +146,9 @@ reorder_sort_key::from(const std::u32string &str) {
|
|||
void
|
||||
reorder_sort_key::dump() const {
|
||||
// for debugging…
|
||||
DebugLog("- U+%04X\t(%d, %d, %d, %d)", ch, (int)primary, (int)secondary, (int)tertiary, (int)quaternary);
|
||||
DebugLog(
|
||||
"- U+%04X\t(%d, %d, %d, %d) %c", ch, (int)primary, (int)secondary, (int)tertiary, (int)quaternary,
|
||||
is_tertiary_base ? 'T' : ' ');
|
||||
}
|
||||
|
||||
size_t
|
||||
|
|
@ -213,9 +215,14 @@ std::deque<reorder_sort_key> &
|
|||
element_list::update_sort_key(size_t offset, std::deque<reorder_sort_key> &key) const {
|
||||
/** string index */
|
||||
size_t c = 0;
|
||||
bool have_last_base = false;
|
||||
reorder_weight last_base_primary = -1;
|
||||
reorder_weight last_base_secondary = -1;
|
||||
for (auto e = begin(); e < end(); e++, c++) {
|
||||
/** position in the key */
|
||||
auto n = offset + c;
|
||||
/** update this key */
|
||||
auto &k = key.at(offset + c);
|
||||
auto &k = key.at(n);
|
||||
// we double check that the character matches. otherwise something
|
||||
// has really gone awry, because we shouldn't be here if this element list doesn't apply.
|
||||
if (!e->matches(k.ch)) {
|
||||
|
|
@ -225,9 +232,30 @@ element_list::update_sort_key(size_t offset, std::deque<reorder_sort_key> &key)
|
|||
assert(e->matches(k.ch)); // double check that this element matches
|
||||
}
|
||||
// we only update primary and tertiary weights
|
||||
k.primary = e->get_order();
|
||||
// TODO-LDML: need more detailed tertiary work
|
||||
k.tertiary = e->get_tertiary();
|
||||
k.primary = e->get_order();
|
||||
k.tertiary = e->get_tertiary();
|
||||
k.is_tertiary_base = e->is_tertiary_base();
|
||||
|
||||
if (k.tertiary != 0 && n > 0) {
|
||||
// search backwards for a base
|
||||
auto n2 = n;
|
||||
// TODO-LDML: odd loop here because n2 is signed.
|
||||
do {
|
||||
n2--;
|
||||
auto &k2 = key.at(n2);
|
||||
if (k2.is_tertiary_base) {
|
||||
last_base_primary = k2.primary;
|
||||
last_base_secondary = k2.secondary;
|
||||
have_last_base = true;
|
||||
}
|
||||
} while (!have_last_base && n2 > 0);
|
||||
// we may not have found the base. but the common case is that the base is found.
|
||||
if (have_last_base) {
|
||||
// copy the primary and secondary from the last_base
|
||||
k.primary = last_base_primary;
|
||||
k.secondary = last_base_secondary;
|
||||
}
|
||||
}
|
||||
#if KMXPLUS_DEBUG_TRANSFORM
|
||||
DebugTran("Updating at +%d", c);
|
||||
k.dump();
|
||||
|
|
@ -271,6 +299,29 @@ reorder_entry::match_end(std::u32string &str, size_t offset, size_t len) const {
|
|||
return match_len;
|
||||
}
|
||||
|
||||
int
|
||||
reorder_entry::compare(const reorder_entry &other) const {
|
||||
if (this == &other) {
|
||||
return 0;
|
||||
} else if (elements.size() < other.elements.size()) {
|
||||
return -1;
|
||||
} else if (elements.size() > other.elements.size()) {
|
||||
return 1;
|
||||
} else {
|
||||
return 0; // punt
|
||||
}
|
||||
}
|
||||
|
||||
bool
|
||||
reorder_entry::operator<(const reorder_entry &other) const {
|
||||
return (compare(other) < 0);
|
||||
}
|
||||
|
||||
bool
|
||||
reorder_entry::operator>(const reorder_entry &other) const {
|
||||
return (compare(other) > 0);
|
||||
}
|
||||
|
||||
bool
|
||||
reorder_group::apply(std::u32string &str) const {
|
||||
/** did we apply anything */
|
||||
|
|
@ -349,7 +400,9 @@ reorder_group::apply(std::u32string &str) const {
|
|||
/** pointer to the beginning of the current run. */
|
||||
std::deque<reorder_sort_key>::iterator run_start = sort_keys.begin();
|
||||
for(auto e = run_start; e != sort_keys.end(); e++) {
|
||||
if ((e->primary == 0) && (e != run_start)) { // it's a base
|
||||
// find the actual beginning base: primary weight = 0 and tertiary = 0.
|
||||
// (tertiary chars will have primary=0 BUT will have tertiary nonzero.)
|
||||
if ((e->primary == 0) && (e->tertiary == 0) && (e != run_start)) {
|
||||
auto run_end = e - 1;
|
||||
DebugTran("Sorting subrange quaternary=[%d..]", run_start->quaternary);
|
||||
std::sort(run_start, run_end); // reversed because it's a reverse iterator…?
|
||||
|
|
@ -365,7 +418,7 @@ reorder_group::apply(std::u32string &str) const {
|
|||
// recombine into a string by pulling out the 'ch' value
|
||||
// that's in each sortkey element.
|
||||
std::u32string newSuffix;
|
||||
size_t q = sort_keys.begin()->quaternary; // start with the first quaternary
|
||||
signed char q = sort_keys.begin()->quaternary; // start with the first quaternary
|
||||
for (auto e = sort_keys.begin(); e < sort_keys.end(); e++, q++) {
|
||||
if (q != e->quaternary) {
|
||||
// something rearranged in this subrange, because the quaternary values are out of order.
|
||||
|
|
@ -838,6 +891,7 @@ transforms::load(
|
|||
return nullptr;
|
||||
}
|
||||
}
|
||||
std::sort(newGroup.list.begin(), newGroup.list.end()); // sort list by size, so that longer matches match last
|
||||
transforms->addGroup(newGroup);
|
||||
} else {
|
||||
// internal error - some other type - should have been caught by validation
|
||||
|
|
|
|||
|
|
@ -47,6 +47,11 @@ inline bool uassert_success(const char *file, int line, const char *function, UE
|
|||
|
||||
using km::core::kmx::SimpleUSet;
|
||||
|
||||
/** a reorder weight, such as primary, secondary, etc. */
|
||||
typedef signed char reorder_weight;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Type of a group
|
||||
*/
|
||||
|
|
@ -71,9 +76,9 @@ public:
|
|||
/** @returns true if tertiary base bit set */
|
||||
bool is_tertiary_base() const;
|
||||
/** @returns the primary order */
|
||||
signed char get_order() const;
|
||||
reorder_weight get_order() const;
|
||||
/** @returns the tertiary order */
|
||||
signed char get_tertiary() const;
|
||||
reorder_weight get_tertiary() const;
|
||||
/** @returns raw elem flags */
|
||||
KMX_DWORD get_flags() const;
|
||||
/** @returns true if matches this character*/
|
||||
|
|
@ -156,11 +161,12 @@ public:
|
|||
|
||||
/** a single char, categorized according to reorder rules*/
|
||||
struct reorder_sort_key {
|
||||
km_core_usv ch; // the single char value
|
||||
signed char primary; // primary order value
|
||||
size_t secondary; // index position
|
||||
signed char tertiary; // tertiary value, defaults to 0
|
||||
size_t quaternary; // index again
|
||||
km_core_usv ch; // the single char value
|
||||
reorder_weight primary; // primary order value
|
||||
reorder_weight secondary; // index position
|
||||
reorder_weight tertiary; // tertiary value, defaults to 0
|
||||
reorder_weight quaternary; // index again
|
||||
bool is_tertiary_base; // remember that this key was a tertiary base
|
||||
|
||||
/** @returns -1, 0, 1 depending on ordering */
|
||||
int compare(const reorder_sort_key &other) const;
|
||||
|
|
@ -213,6 +219,11 @@ public:
|
|||
*/
|
||||
size_t match_end(std::u32string &str, size_t offset, size_t len) const;
|
||||
|
||||
/** @returns -1, 0, 1 depending on ordering */
|
||||
int compare(const reorder_entry &other) const;
|
||||
bool operator<(const reorder_entry &other) const;
|
||||
bool operator>(const reorder_entry &other) const;
|
||||
|
||||
public:
|
||||
element_list elements;
|
||||
element_list before;
|
||||
|
|
|
|||
24
core/tests/unit/ldml/keyboards/k_201_reorder_esk-test.xml
Normal file
24
core/tests/unit/ldml/keyboards/k_201_reorder_esk-test.xml
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE keyboardTest3 SYSTEM "../../../../../resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboardTest3.dtd">
|
||||
<keyboardTest3 conformsTo="techpreview">
|
||||
<info keyboard="k_201_reorder_esk.xml" author="Team Keyboard" name="esk-reorder-test" />
|
||||
<tests name="basic">
|
||||
<!-- these are also in C++ form, search for test_reorder_esk -->
|
||||
<test name="1short">
|
||||
<startContext to="" />
|
||||
<keystroke key="a" />
|
||||
<keystroke key="x" />
|
||||
<keystroke key="overbar" />
|
||||
<check result="a\u0305x" />
|
||||
</test>
|
||||
<test name="2longer">
|
||||
<startContext to="" />
|
||||
<keystroke key="a" />
|
||||
<keystroke key="z" />
|
||||
<keystroke key="overbar" />
|
||||
<keystroke key="x" />
|
||||
<keystroke key="underbar" />
|
||||
<check result="a\u0332\u0305xz" />
|
||||
</test>
|
||||
</tests>
|
||||
</keyboardTest3>
|
||||
58
core/tests/unit/ldml/keyboards/k_201_reorder_esk.xml
Normal file
58
core/tests/unit/ldml/keyboards/k_201_reorder_esk.xml
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
||||
<!--
|
||||
Test Keyboard from Spec
|
||||
see https://keyman.com/keyboards/sil_boonkit
|
||||
-->
|
||||
|
||||
<!DOCTYPE keyboard3 SYSTEM "../../../../../resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboard3.dtd">
|
||||
<keyboard3 locale="en-t-k0-esk" conformsTo="techpreview">
|
||||
<info author="srl295" indicator="🙀" layout="qwerty" name="esk reorder test"/>
|
||||
|
||||
<displays>
|
||||
<display keyId="overbar" display="¯" />
|
||||
<display keyId="underbar" display="_" />
|
||||
<display keyId="circumflex" display="^" />
|
||||
</displays>
|
||||
|
||||
<keys>
|
||||
<key id="overbar" output="${overbar}" />
|
||||
<key id="underbar" output="${underbar}" />
|
||||
<key id="circumflex" output="${circumflex}" />
|
||||
</keys>
|
||||
|
||||
<layers formId="us">
|
||||
<layer modifiers="none" id="base">
|
||||
<row keys="gap overbar underbar circumflex" />
|
||||
<row keys="gap gap e gap gap y u i o gap" />
|
||||
<row keys="a" />
|
||||
<row keys="z x" />
|
||||
<row keys="space" />
|
||||
</layer>
|
||||
</layers>
|
||||
|
||||
<variables>
|
||||
<string id="overbar" value="\u{0305}" />
|
||||
<string id="underbar" value="\u{0332}" />
|
||||
<string id="circumflex" value="\u{0302}" />
|
||||
</variables>
|
||||
|
||||
<transforms type="simple">
|
||||
<transformGroup>
|
||||
<!--
|
||||
ordering for this 'language'
|
||||
|
||||
|
||||
[aeiou] [_] [¯^] [x] [y] [z]
|
||||
|
||||
-->
|
||||
|
||||
<reorder from="[aeiou]" order="0" tertiaryBase="true" /> <!-- p=0: implied, but here for pedagogical purposes -->
|
||||
<reorder from="[\u{0332}]" tertiary="1" /> <!-- t=1: lower first -->
|
||||
<reorder from="[\u{0305}\u{0302}]" tertiary="2" /> <!-- t=2: upper next -->
|
||||
<reorder from="x" order="1" /> <!-- p=1: this follows the 'vowel cluster' -->
|
||||
<reorder from="y" order="2" /> <!-- p=2: this follows the 'vowel cluster' -->
|
||||
<reorder from="z" order="3" /> <!-- p=3: this follows the 'vowel cluster' -->
|
||||
</transformGroup>
|
||||
</transforms>
|
||||
</keyboard3>
|
||||
|
|
@ -39,6 +39,7 @@ tests_with_testdata = [
|
|||
'k_008_transform_norm',
|
||||
'k_020_fr', # TODO-LDML: move to cldr above (fix vkey)
|
||||
'k_200_reorder_nod_Lana',
|
||||
'k_201_reorder_esk',
|
||||
'k_210_marker',
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -61,6 +61,28 @@ ldml = executable('ldml',
|
|||
objects: lib.extract_all_objects(recursive: false),
|
||||
)
|
||||
|
||||
|
||||
# Build and run additional test_kmx_plus test
|
||||
|
||||
e = executable('test_kmx_plus', 'test_kmx_plus.cpp',
|
||||
'ldml_test_utils.cpp',
|
||||
cpp_args: defns + warns,
|
||||
include_directories: [inc, libsrc, '../../../../developer/src/ext/json'],
|
||||
link_args: links + tests_flags,
|
||||
dependencies: [icu_uc, icu_i18n],
|
||||
objects: lib.extract_all_objects(recursive: false))
|
||||
test('test_kmx_plus', e, suite: 'ldml')
|
||||
|
||||
# run transforms / ldml utilities unit test
|
||||
|
||||
t = executable('test_transforms', 'test_transforms.cpp',
|
||||
cpp_args: defns + warns,
|
||||
include_directories: [inc, libsrc, '../../../../developer/src/ext/json'],
|
||||
link_args: links + tests_flags,
|
||||
dependencies: [icu_uc, icu_i18n],
|
||||
objects: lib.extract_all_objects(recursive: false))
|
||||
test('test_transforms', t, suite: 'ldml')
|
||||
|
||||
# Run tests on all keyboards (`tests` defined in keyboards/meson.build)
|
||||
|
||||
foreach kbd : tests
|
||||
|
|
@ -77,22 +99,3 @@ foreach kbd : invalid_tests
|
|||
test(kbd, ldml, args: [kbd_src, kbd_obj], suite: 'ldml-invalid-keyboards')
|
||||
# todo: consider if we should use `should_fail: true`?
|
||||
endforeach
|
||||
|
||||
# Build and run additional test_kmx_plus test
|
||||
|
||||
e = executable('test_kmx_plus', 'test_kmx_plus.cpp',
|
||||
'ldml_test_utils.cpp',
|
||||
cpp_args: defns + warns,
|
||||
include_directories: [inc, libsrc, '../../../../developer/src/ext/json'],
|
||||
link_args: links + tests_flags,
|
||||
dependencies: [icu_uc, icu_i18n],
|
||||
objects: lib.extract_all_objects(recursive: false))
|
||||
test('test_kmx_plus', e, suite: 'ldml')
|
||||
|
||||
t = executable('test_transforms', 'test_transforms.cpp',
|
||||
cpp_args: defns + warns,
|
||||
include_directories: [inc, libsrc, '../../../../developer/src/ext/json'],
|
||||
link_args: links + tests_flags,
|
||||
dependencies: [icu_uc, icu_i18n],
|
||||
objects: lib.extract_all_objects(recursive: false))
|
||||
test('test_transforms', t, suite: 'ldml')
|
||||
|
|
|
|||
|
|
@ -15,15 +15,22 @@
|
|||
#define zassert_string_equal(actual, expected) \
|
||||
{ \
|
||||
if (actual != expected) { \
|
||||
std::wcerr << __FILE__ << ":" << __LINE__ << ": " << console_color::fg(console_color::BRIGHT_RED) << "got: " << actual \
|
||||
<< " expected " << expected << console_color::reset() << std::endl; \
|
||||
std::wcerr << __FILE__ << ":" << __LINE__ << ": " << console_color::fg(console_color::BRIGHT_RED) << "got: " << km::core::kmx::Debug_UnicodeString(actual, 0) \
|
||||
<< " expected " << km::core::kmx::Debug_UnicodeString(expected, 1) << console_color::reset() << std::endl; \
|
||||
return EXIT_FAILURE; \
|
||||
} \
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifndef zassert_equal
|
||||
#define zassert_equal(actual, expected) zassert_string_equal(actual, expected)
|
||||
#define zassert_equal(actual, expected) \
|
||||
{ \
|
||||
if (actual != expected) { \
|
||||
std::wcerr << __FILE__ << ":" << __LINE__ << ": " << console_color::fg(console_color::BRIGHT_RED) << "got: " << actual \
|
||||
<< " expected " << expected << console_color::reset() << std::endl; \
|
||||
return EXIT_FAILURE; \
|
||||
} \
|
||||
}
|
||||
#endif
|
||||
|
||||
// needed for streaming operators
|
||||
|
|
@ -151,6 +158,22 @@ int
|
|||
test_reorder_standalone() {
|
||||
std::cout << "== " << __FUNCTION__ << std::endl;
|
||||
|
||||
std::cout << __FILE__ << ":" << __LINE__ << " - element API test " << std::endl;
|
||||
// element API test - not a real element, just here for testing
|
||||
{
|
||||
element es(U'a', 0xF4500000 | LDML_ELEM_FLAGS_PREBASE | LDML_ELEM_FLAGS_TERTIARY_BASE); // tertiary -12, primary 80
|
||||
std::cout << "es flags" << std::hex << es.get_flags() << std::dec << std::endl;
|
||||
// verify element metadata
|
||||
assert_equal(es.is_uset(), false);
|
||||
assert_equal(es.get_order(), 0x50);
|
||||
assert_equal(es.get_tertiary(), -12);
|
||||
assert_equal(es.is_prebase(), true);
|
||||
assert_equal(es.is_tertiary_base(), true);
|
||||
// verify element matching
|
||||
assert_equal(es.matches(U'a'), true);
|
||||
assert_equal(es.matches(U'b'), false);
|
||||
}
|
||||
|
||||
std::cout << __FILE__ << ":" << __LINE__ << " - nod-Lana " << std::endl;
|
||||
{
|
||||
const std::u32string roasts[] = {
|
||||
|
|
@ -172,17 +195,17 @@ test_reorder_standalone() {
|
|||
assert_equal(toneMarks.contains(0x1A76), true);
|
||||
assert_equal(toneMarks.contains(0x1A60), false);
|
||||
|
||||
std::cout << __FILE__ << ":" << __LINE__ << " - element test " << std::endl;
|
||||
std::cout << __FILE__ << ":" << __LINE__ << " - element API test " << std::endl;
|
||||
// element test
|
||||
{
|
||||
element es(U'a', 0xF4500000 | LDML_ELEM_FLAGS_PREBASE | LDML_ELEM_FLAGS_TERTIARY_BASE); // tertiary -12, primary 80
|
||||
element es(U'a', (80 << LDML_ELEM_FLAGS_ORDER_BITSHIFT) | LDML_ELEM_FLAGS_PREBASE); // tertiary -12, primary 80
|
||||
std::cout << "es flags" << std::hex << es.get_flags() << std::dec << std::endl;
|
||||
// verify element metadata
|
||||
assert_equal(es.is_uset(), false);
|
||||
assert_equal(es.get_order(), 0x50);
|
||||
assert_equal(es.get_tertiary(), -12);
|
||||
assert_equal(es.get_tertiary(), 0);
|
||||
assert_equal(es.is_prebase(), true);
|
||||
assert_equal(es.is_tertiary_base(), true);
|
||||
assert_equal(es.is_tertiary_base(), false);
|
||||
// verify element matching
|
||||
assert_equal(es.matches(U'a'), true);
|
||||
assert_equal(es.matches(U'b'), false);
|
||||
|
|
@ -228,7 +251,7 @@ test_reorder_standalone() {
|
|||
l.update_sort_key(0, keylist);
|
||||
std::cout << __FILE__ << ":" << __LINE__ << " updated sortkey" << std::endl;
|
||||
assert_equal(keylist.size(), 2);
|
||||
size_t secondary = 0;
|
||||
reorder_weight secondary = 0;
|
||||
for (auto i = keylist.begin(); i < keylist.end(); i++) {
|
||||
i->dump();
|
||||
assert_equal(i->secondary, secondary);
|
||||
|
|
@ -239,7 +262,7 @@ test_reorder_standalone() {
|
|||
|
||||
// spot check first sortkey
|
||||
assert_equal(keylist.begin()->primary, 80);
|
||||
assert_equal(keylist.begin()->tertiary, -12);
|
||||
assert_equal(keylist.begin()->tertiary, 0);
|
||||
assert_equal(keylist.begin()->ch, 0x61);
|
||||
std::cout << __FILE__ << ":" << __LINE__ << " sorted sortkey" << std::endl;
|
||||
|
||||
|
|
@ -407,6 +430,133 @@ test_reorder_standalone() {
|
|||
return EXIT_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
// this test case is also in XML form under 'k_201_*'
|
||||
int
|
||||
test_reorder_esk() {
|
||||
std::cout << "== " << __FUNCTION__ << std::endl;
|
||||
|
||||
std::cout << __FILE__ << ":" << __LINE__ << " - k_201_reorder_esk (tertiary reordering) " << std::endl;
|
||||
{
|
||||
// now setup the rules
|
||||
// rules are a little bit simplified, having only the vowel 'a'
|
||||
|
||||
std::cout << "now prepare the reorder elements" << std::endl;
|
||||
transforms tr;
|
||||
{
|
||||
reorder_group rg;
|
||||
|
||||
// <reorder from="a" order="0" tertiaryBase="true" />
|
||||
{
|
||||
element_list e;
|
||||
e.emplace_back(U'a', (0 << LDML_ELEM_FLAGS_ORDER_BITSHIFT) | LDML_ELEM_FLAGS_TERTIARY_BASE);
|
||||
rg.list.emplace_back(e);
|
||||
}
|
||||
|
||||
// <reorder from="[\u{0332}]" tertiary="1" />
|
||||
{
|
||||
element_list e;
|
||||
e.emplace_back(0x0332, (1 << LDML_ELEM_FLAGS_TERTIARY_BITSHIFT));
|
||||
rg.list.emplace_back(e);
|
||||
}
|
||||
// <reorder from="[\u{0305}\u{0302}]" tertiary="2" />
|
||||
{
|
||||
element_list e;
|
||||
e.emplace_back(0x305, (2 << LDML_ELEM_FLAGS_TERTIARY_BITSHIFT));
|
||||
// (should be a unicodeset, but for simplicity we're dropping the U+302)
|
||||
// e.emplace_back(0x302, (2 << LDML_ELEM_FLAGS_TERTIARY_BITSHIFT));
|
||||
rg.list.emplace_back(e);
|
||||
}
|
||||
// <reorder from="x" order="1" />
|
||||
{
|
||||
element_list e;
|
||||
e.emplace_back(U'x', (1 << LDML_ELEM_FLAGS_ORDER_BITSHIFT));
|
||||
rg.list.emplace_back(e);
|
||||
}
|
||||
// <reorder from="y" order="2" />
|
||||
{
|
||||
element_list e;
|
||||
e.emplace_back(U'y', (2 << LDML_ELEM_FLAGS_ORDER_BITSHIFT));
|
||||
rg.list.emplace_back(e);
|
||||
}
|
||||
// <reorder from="z" order="3" />
|
||||
{
|
||||
element_list e;
|
||||
e.emplace_back(U'z', (3 << LDML_ELEM_FLAGS_ORDER_BITSHIFT));
|
||||
rg.list.emplace_back(e);
|
||||
}
|
||||
|
||||
tr.addGroup(rg);
|
||||
}
|
||||
|
||||
// now actually test it
|
||||
std::cout << __FILE__ << ":" << __LINE__ << " - cases " << std::endl;
|
||||
const std::u32string orig_expect[] = {
|
||||
// 1short
|
||||
U"ax\u0305", // orig
|
||||
U"a\u0305x", // expect
|
||||
|
||||
// 2longer
|
||||
U"az\u0305x\u0332", // orig
|
||||
U"a\u0332\u0305xz", // expect
|
||||
};
|
||||
// TODO-LDML: move this into test code perhaps
|
||||
for (size_t r = 0; r < sizeof(orig_expect) / sizeof(orig_expect[0]); r+= 2) {
|
||||
const auto &orig = orig_expect[r + 0];
|
||||
const auto &expect = orig_expect[r + 1];
|
||||
std::cout << __FILE__ << ":" << __LINE__ << " - trying str #" << r+1 << "=" << orig << std::endl;
|
||||
// try apply with string
|
||||
{
|
||||
std::cout << "- try apply(text, output)" << std::endl;
|
||||
std::u32string text = orig;
|
||||
std::u32string output;
|
||||
size_t len = tr.apply(text, output);
|
||||
if (len == 0) {
|
||||
std::cout << " (did not apply)" << std::endl;
|
||||
} else {
|
||||
std::cout << " applied, matchLen= " << len << std::endl;
|
||||
text.resize(text.size()-len); // shrink
|
||||
text.append(output);
|
||||
std::cout << " = " << text << std::endl;
|
||||
}
|
||||
zassert_string_equal(text, expect);
|
||||
}
|
||||
// try all-at-once
|
||||
{
|
||||
std::cout << "- try apply(text)" << std::endl;
|
||||
std::u32string text = orig;
|
||||
if (!tr.apply(text)) {
|
||||
std::cout << " (did not apply)" << std::endl;
|
||||
} else if (text == orig) {
|
||||
std::cout << " (suboptimal: apply returned true but made no change)" << std::endl;
|
||||
} else {
|
||||
std::cout << " changed to " << text;
|
||||
}
|
||||
zassert_string_equal(text, expect);
|
||||
std::cout << " matched (converting all at once)!" << std::endl;
|
||||
}
|
||||
// simulate typing this one char at a time;
|
||||
{
|
||||
std::cout << "- try key-at-a-time" << std::endl;
|
||||
std::u32string text;
|
||||
for (auto ch = orig.begin(); ch < orig.end(); ch++) {
|
||||
// append the string
|
||||
text.append(1, *ch);
|
||||
std::cout << "-: " << text << std::endl;
|
||||
if (!tr.apply(text)) {
|
||||
std::cout << " (did not apply)" << std::endl;
|
||||
}
|
||||
}
|
||||
// now the moment of truth
|
||||
zassert_string_equal(text, expect);
|
||||
std::cout << " matched! (converting char at a time)" << std::endl;
|
||||
std::cout << std::endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
|
||||
int test_map() {
|
||||
std::cout << "== " << __FUNCTION__ << std::endl;
|
||||
|
||||
|
|
@ -639,6 +789,10 @@ main(int argc, const char *argv[]) {
|
|||
rc = EXIT_FAILURE;
|
||||
}
|
||||
|
||||
if (test_reorder_esk() != EXIT_SUCCESS) {
|
||||
rc = EXIT_FAILURE;
|
||||
}
|
||||
|
||||
if (test_map() != EXIT_SUCCESS) {
|
||||
rc = EXIT_FAILURE;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -155,8 +155,7 @@
|
|||
<transform from="\m{q}\m{X}" to="\u{09CD}\u{200C}" /> <!-- virama + zwnj-->
|
||||
</transformGroup>
|
||||
<!-- TODO: document these -->
|
||||
<!-- TODO-LDML: not working, probably due to #9707 -->
|
||||
<!-- <transformGroup>
|
||||
<transformGroup>
|
||||
<reorder from="\u{09BC}" tertiary="3"/>
|
||||
<reorder from="\u{09CD}[\u{0980}\u{0985}-\u{098C}\u{098F}\u{0990}\u{0993}-\u{09A8}\u{09AA}-\u{09B0}\u{09B2}\u{09B6}-\u{09B9}\u{09BD}\u{09DC}\u{09DD}\u{09DF}-\u{09E1}\u{09E6}-\u{09F1}\u{09FC}]" order="10" tertiaryBase="true"/>
|
||||
<reorder from="\u{09CD}[\u{200C}\u{200D}][\u{0980}\u{0985}-\u{098C}\u{098F}\u{0990}\u{0993}-\u{09A8}\u{09AA}-\u{09B0}\u{09B2}\u{09B6}-\u{09B9}\u{09BD}\u{09DC}\u{09DD}\u{09DF}-\u{09E1}\u{09E6}-\u{09F1}\u{09FC}]" order="10" tertiaryBase="true"/>
|
||||
|
|
@ -167,6 +166,6 @@
|
|||
<reorder from="\u{0981}" order="85"/>
|
||||
<reorder from="[\u{0982}\u{0983}]" order="95"/>
|
||||
<reorder from="\u{09FE}" order="117"/>
|
||||
</transformGroup> -->
|
||||
</transformGroup>
|
||||
</transforms>
|
||||
</keyboard3>
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue