diff --git a/common/web/types/src/kmx/kmx-plus.ts b/common/web/types/src/kmx/kmx-plus.ts index 2530821b7e..28748c5de7 100644 --- a/common/web/types/src/kmx/kmx-plus.ts +++ b/common/web/types/src/kmx/kmx-plus.ts @@ -2,7 +2,7 @@ import { constants } from '@keymanapp/ldml-keyboard-constants'; import * as r from 'restructure'; import { ElementString } from './element-string.js'; import { ListItem } from './string-list.js'; -import { unescapeString } from '../util/util.js'; +import { isOneChar, toOneChar, unescapeString } from '../util/util.js'; import { KMXFile } from './kmx.js'; import { UnicodeSetParser, UnicodeSet } from '@keymanapp/common-types'; import { VariableParser } from '../ldml-keyboard/pattern-parser.js'; @@ -111,11 +111,6 @@ export class StrsItem { return 0; } - /** True if this string *could* be a UTF-32 single char */ - static isOneChar(value: string) : boolean { - return (value.split('').length) == 1; - } - get isOneChar() { return this.char !== undefined; } @@ -126,10 +121,10 @@ export class StrsItem { */ export class CharStrsItem extends StrsItem { constructor(value: string) { - if (!StrsItem.isOneChar(value)) { + if (!isOneChar(value)) { throw RangeError(`not a 1-char string`); } - super(value, value.charCodeAt(0)); + super(value, toOneChar(value)); } }; @@ -161,7 +156,7 @@ export class Strs extends Section { } // if it's a single char, don't push it into the list - if (singleOk && StrsItem.isOneChar(s)) { + if (singleOk && isOneChar(s)) { return new CharStrsItem(s); } diff --git a/common/web/types/src/util/util.ts b/common/web/types/src/util/util.ts index 7d70d001d4..6145fb4ceb 100644 --- a/common/web/types/src/util/util.ts +++ b/common/web/types/src/util/util.ts @@ -66,3 +66,17 @@ export function unescapeString(s: string): string { return s; } + +/** True if this string *could* be a UTF-32 single char */ +export function +isOneChar(value: string) : boolean { + return [...value].length === 1; +} + +export function +toOneChar(value: string) : number { + if (!isOneChar(value)) { + throw Error(`Not a single char: ${value}`); + } + return value.codePointAt(0); +} diff --git a/common/web/types/test/util/test-unescape.ts b/common/web/types/test/util/test-unescape.ts index 5708046a80..704a3e124a 100644 --- a/common/web/types/test/util/test-unescape.ts +++ b/common/web/types/test/util/test-unescape.ts @@ -1,6 +1,26 @@ import 'mocha'; import {assert} from 'chai'; -import {unescapeString, UnescapeError} from '../../src/util/util.js'; +import {unescapeString, UnescapeError, isOneChar, toOneChar} from '../../src/util/util.js'; + +describe('test UTF32 functions()', function() { + it('should properly categorize strings', () => { + [ + 'x', + '🙀', + ].forEach(s => assert.isTrue(isOneChar(s), `isOneChar(${s})`)); + + [ + 'xx', + '🙀🙀', + ].forEach(s => assert.isFalse(isOneChar(s), `!isOneChar(${s})`)); + }); + + it('should convert to single chars', function() { + assert.equal(toOneChar('ħ'), 295); + assert.equal(toOneChar('🙀'), 0x1F640); + assert.throws(() => toOneChar('ħħħ'), /Not a single char/); + }); +}); describe('test unescapeString()', function() { it("should correctly handle multi strings", function() { diff --git a/core/src/kmx/kmx_plus.cpp b/core/src/kmx/kmx_plus.cpp index d72efb6b28..159fb9b5a8 100644 --- a/core/src/kmx/kmx_plus.cpp +++ b/core/src/kmx/kmx_plus.cpp @@ -270,6 +270,18 @@ COMP_KMXPLUS_STRS::valid(KMX_DWORD _kmn_unused(length)) const { return true; } +/** + * helper for extracting single char values + * @param v field with char type + * @return value a string + */ +std::u16string COMP_KMXPLUS_STRS::str_from_char(KMX_DWORD v) { + char16_single buf; + const int len = Utf32CharToUtf16(v, buf); + return std::u16string(buf.ch, len); +} + + bool COMP_KMXPLUS_SECT::valid(KMX_DWORD length) const { DebugLog("sect: total 0x%X\n", total); @@ -357,12 +369,11 @@ COMP_KMXPLUS_ELEM::getElementList(KMX_DWORD elementNumber, KMX_DWORD &length) co return reinterpret_cast(rawdata + entry.offset); } + std::u16string -COMP_KMXPLUS_ELEM_ELEMENT::get_string() const { +COMP_KMXPLUS_ELEM_ELEMENT::get_element_string() const { assert((flags & LDML_ELEM_FLAGS_TYPE) == LDML_ELEM_FLAGS_TYPE_CHAR); // should only be called on char - char16_single buf; - const int len = Utf32CharToUtf16(element, buf); - return std::u16string(buf.ch, len); + return COMP_KMXPLUS_STRS::str_from_char(element); } // Note: shared with subclass COMP_KMXPLUS_BKSP @@ -802,9 +813,13 @@ COMP_KMXPLUS_KEYS_Helper::findKeyByStringId(KMX_DWORD strId, KMX_DWORD &i) const } const COMP_KMXPLUS_KEYS_KEY* -COMP_KMXPLUS_KEYS_Helper::findKeyByStringTo(KMX_DWORD strId, KMX_DWORD &i) const { +COMP_KMXPLUS_KEYS_Helper::findKeyByStringTo(const std::u16string& str, KMX_DWORD strId, KMX_DWORD &i) const { for (; i < key2->keyCount; i++) { - if (keys[i].to == strId) { + if (keys[i].flags & LDML_KEYS_KEY_FLAGS_EXTEND) { + if (strId != 0 && keys[i].to == strId) { + return &keys[i]; + } + } else if (keys[i].get_to_string() == str) { return &keys[i]; } } @@ -839,11 +854,9 @@ COMP_KMXPLUS_KEYS_Helper::getKmap(KMX_DWORD i) const { } std::u16string -COMP_KMXPLUS_KEYS_KEY::get_string() const { +COMP_KMXPLUS_KEYS_KEY::get_to_string() const { assert(!(flags & LDML_KEYS_KEY_FLAGS_EXTEND)); // should not be called. - char16_single buf; - const int len = Utf32CharToUtf16(to, buf); - return std::u16string(buf.ch, len); + return COMP_KMXPLUS_STRS::str_from_char(to); } // LIST @@ -955,7 +968,7 @@ kmx_plus::kmx_plus(const COMP_KEYBOARD *keyboard, size_t length) const COMP_KEYBOARD_EX* ex = reinterpret_cast(keyboard); DebugLog("kmx_plus(): KMXPlus offset 0x%X, KMXPlus size 0x%X\n", ex->kmxplus.dpKMXPlus, ex->kmxplus.dwKMXPlusSize); - if (ex->kmxplus.dpKMXPlus < sizeof(kmx::COMP_KEYBOARD_EX)) { + if (ex->kmxplus.dpKMXPlus < sizeof(COMP_KEYBOARD_EX)) { DebugLog("dwKMXPlus is not past the end of COMP_KEYBOARD_EX"); valid = false; assert(valid); diff --git a/core/src/kmx/kmx_plus.h b/core/src/kmx/kmx_plus.h index fce128f8ec..0d2f266b5a 100644 --- a/core/src/kmx/kmx_plus.h +++ b/core/src/kmx/kmx_plus.h @@ -93,11 +93,11 @@ struct COMP_KMXPLUS_ELEM_ELEMENT { KMX_DWORD element; // str: output string or UTF-32LE codepoint KMX_DWORD flags; // flag and order values /** - * @brief Get the 'to' as a string, if flags&LDML_ELEM_FLAGS_TYPE = CHAR + * @brief Get the 'element' as a string, if flags&LDML_ELEM_FLAGS_TYPE = CHAR * * @return std::u16string */ - std::u16string get_string() const; + std::u16string get_element_string() const; }; struct COMP_KMXPLUS_ELEM_ENTRY { @@ -234,6 +234,9 @@ struct COMP_KMXPLUS_STRS { * @brief True if section is valid. */ bool valid(KMX_DWORD length) const; + + /** convert a single char to a string*/ + static std::u16string str_from_char(KMX_DWORD v); }; static_assert(sizeof(struct COMP_KMXPLUS_STRS) % 0x4 == 0, "Structs prior to variable part should align to 32-bit boundary"); @@ -506,6 +509,8 @@ struct COMP_KMXPLUS_KEYS_FLICK_ELEMENT { KMXPLUS_LIST directions; KMX_DWORD flags; KMXPLUS_STR to; // string or codepoint + /** get the 'to' string if a char */ + std::u16string get_to_string() const; }; struct COMP_KMXPLUS_KEYS_FLICK_LIST { @@ -525,7 +530,7 @@ struct COMP_KMXPLUS_KEYS_KEY { KMXPLUS_LIST multiTap; KMX_DWORD flicks; // index - std::u16string get_string() const; + std::u16string get_to_string() const; }; struct COMP_KMXPLUS_KEYS_KMAP { @@ -558,11 +563,12 @@ public: const COMP_KMXPLUS_KEYS_KEY *findKeyByStringId(KMX_DWORD strId, KMX_DWORD &index) const; /** * Search for a key by 'to' string id + * @param str string to search for (for single char strings) * @param strID id to search for * @param index on entry, id to start with such as 0. On exit, index of item if found. Undefined otherwise. * @return pointer to key or nullptr */ - const COMP_KMXPLUS_KEYS_KEY *findKeyByStringTo(KMX_DWORD strId, KMX_DWORD &index) const; + const COMP_KMXPLUS_KEYS_KEY *findKeyByStringTo(const std::u16string& str, KMX_DWORD strId, KMX_DWORD &index) const; private: const COMP_KMXPLUS_KEYS *key2; diff --git a/core/src/ldml/ldml_processor.cpp b/core/src/ldml/ldml_processor.cpp index 12ceb1cb1f..442f06ca14 100644 --- a/core/src/ldml/ldml_processor.cpp +++ b/core/src/ldml/ldml_processor.cpp @@ -71,7 +71,8 @@ ldml_processor::ldml_processor(path const & kb_path, const std::vector auto keyEntry = kplus.key2Helper.getKeys(kmapEntry->key); assert(keyEntry != nullptr); - if (keyEntry->flags && LDML_KEYS_KEY_FLAGS_EXTEND) { + // TODO-LDML: LDML_KEYS_KEY_FLAGS_NOTRANSFORM + if (keyEntry->flags & LDML_KEYS_KEY_FLAGS_EXTEND) { if (nullptr == kplus.strs) { DebugLog("for keys: kplus.strs == nullptr"); // need a string table to get strings assert(false); @@ -79,7 +80,7 @@ ldml_processor::ldml_processor(path const & kb_path, const std::vector } str = kplus.strs->get(keyEntry->to); } else { - str = keyEntry->get_string(); + str = keyEntry->get_to_string(); } keys.add((km_kbp_virtual_key)kmapEntry->vkey, (uint16_t)kmapEntry->mod, str); } diff --git a/core/tests/unit/ldml/ldml_test_source.cpp b/core/tests/unit/ldml/ldml_test_source.cpp index 614a460688..da65e62a27 100644 --- a/core/tests/unit/ldml/ldml_test_source.cpp +++ b/core/tests/unit/ldml/ldml_test_source.cpp @@ -565,18 +565,11 @@ LdmlJsonRepertoireTestSource::next_action(ldml_action &fillin) { // First, find the string as an id // TODO-LDML: will not work for multi string cases - KMX_DWORD strId = kmxplus->strs->find(chstr); - if (strId == 0) { // will also get here if id is empty. - fillin.string = u"No string for repertoire test: "; - fillin.string.append(chstr); - fillin.type = LDML_ACTION_FAIL; - return; - } - assert(strId != 0); + KMX_DWORD strId = kmxplus->strs->find(chstr); // not an error if chstr is 0, may be single ch // OK. Now we can search the keybag KMX_DWORD keyIndex = 0; - auto *key2 = kmxplus->key2Helper.findKeyByStringTo(strId, keyIndex); + auto *key2 = kmxplus->key2Helper.findKeyByStringTo(chstr, strId, keyIndex); if (key2 == nullptr) { fillin.string = u"No key for repertoire test: "; fillin.string.append(chstr); diff --git a/core/tests/unit/ldml/test_kmx_plus.cpp b/core/tests/unit/ldml/test_kmx_plus.cpp index 95a40b4b20..29bf9423f1 100644 --- a/core/tests/unit/ldml/test_kmx_plus.cpp +++ b/core/tests/unit/ldml/test_kmx_plus.cpp @@ -32,24 +32,24 @@ int test_COMP_KMXPLUS_KEYS_KEY() { 0x0001F640, // to 0x00000000 // flags: CHAR }}; - std::u16string s0 = e[0].get_string(); + std::u16string s0 = e[0].get_to_string(); assert_equal(s0.length(), 1); assert_equal(s0.at(0), 0x0127); assert(s0 == std::u16string(u"ħ")); - std::u16string s1 = e[1].get_string(); + std::u16string s1 = e[1].get_to_string(); assert_equal(s1.length(), 2); assert_equal(s1.at(0), 0xD83D); assert_equal(s1.at(1), 0xDE40); assert(s1 == std::u16string(u"🙀")); // now, elems. Parallel. - std::u16string es0 = elems[0].get_string(); + std::u16string es0 = elems[0].get_element_string(); assert_equal(es0.length(), 1); assert_equal(es0.at(0), 0x0127); assert(es0 == std::u16string(u"ħ")); - std::u16string es1 = elems[1].get_string(); + std::u16string es1 = elems[1].get_element_string(); assert_equal(es1.length(), 2); assert_equal(es1.at(0), 0xD83D); assert_equal(es1.at(1), 0xDE40); diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts index e243ffd726..7c5cd52e77 100644 --- a/developer/src/kmc-ldml/src/compiler/keys.ts +++ b/developer/src/kmc-ldml/src/compiler/keys.ts @@ -101,10 +101,11 @@ export class KeysCompiler extends SectionCompiler { for (let lkflick of lkflicks.flick) { let flags = 0; - // TODO-LDML: single char const to = sections.strs.allocAndUnescapeString(lkflick.to, true); - flags |= to.isOneChar ? constants.keys_flick_flags_extend : 0; - let directions : ListItem = sections.list.allocListFromSpaces(sections.strs, lkflick.directions); + if (!to.isOneChar) { + flags |= constants.keys_flick_flags_extend; + } + let directions: ListItem = sections.list.allocListFromSpaces(sections.strs, lkflick.directions); flicks.flicks.push({ directions, flags, @@ -138,8 +139,10 @@ export class KeysCompiler extends SectionCompiler { const longPressDefault = sections.strs.allocAndUnescapeString(key.longPressDefault); const multiTap: ListItem = sections.list.allocListFromEscapedSpaces(sections.strs, key.multiTap); const keySwitch = sections.strs.allocString(key.switch); // 'switch' is a reserved word - const to = sections.strs.allocAndUnescapeString(key.to, true); // TODO-LDML: single char - flags |= to.isOneChar ? constants.keys_key_flags_extend : 0; + const to = sections.strs.allocAndUnescapeString(key.to, true); + if (!to.isOneChar) { + flags |= constants.keys_key_flags_extend; + } const width = Math.ceil((key.width || 1) * 10.0); // default, width=1 sect.keys.push({ flags, diff --git a/developer/src/kmc-ldml/test/fixtures/sections/keys/minimal.xml b/developer/src/kmc-ldml/test/fixtures/sections/keys/minimal.xml index dc498b31f6..d8f9a64bff 100644 --- a/developer/src/kmc-ldml/test/fixtures/sections/keys/minimal.xml +++ b/developer/src/kmc-ldml/test/fixtures/sections/keys/minimal.xml @@ -8,11 +8,12 @@ + - + diff --git a/developer/src/kmc-ldml/test/test-keys.ts b/developer/src/kmc-ldml/test/test-keys.ts index 973591c404..e20094d9d4 100644 --- a/developer/src/kmc-ldml/test/test-keys.ts +++ b/developer/src/kmc-ldml/test/test-keys.ts @@ -18,10 +18,16 @@ describe('keys', function () { const keys = sect; assert.ok(keys); assert.equal(compilerTestCallbacks.messages.length, 0); - assert.equal(keys.keys.length, 1); + assert.equal(keys.keys.length, 2); assert.equal(keys.flicks.length, 1); // there's always a 'null' flick - assert.equal(keys.keys[0].to.value, String.fromCodePoint(0x1FAA6)); - assert.equal(keys.keys[0].id.value, 'grave'); + assert.equal(keys.keys[0].to.value, 'oops'); + assert.isFalse(keys.keys[0].to.isOneChar); + assert.equal(keys.keys[0].flags, constants.keys_key_flags_extend); + assert.equal(keys.keys[0].id.value, 'mistake'); + assert.isTrue(keys.keys[1].to.isOneChar); + assert.equal(keys.keys[1].to.value, String.fromCodePoint(0x1FAA6)); + assert.equal(keys.keys[1].flags, 0); + assert.equal(keys.keys[1].id.value, 'grave'); }, }, { @@ -120,7 +126,7 @@ describe('keys.kmap', function () { let keys = await loadSectionFixture(KeysCompiler, 'sections/keys/minimal.xml', compilerTestCallbacks) as Keys; assert.isNotNull(keys); assert.equal(compilerTestCallbacks.messages.length, 0); - assert.equal(keys.kmap.length, 1); + assert.equal(keys.kmap.length, 2); }); testCompilationCases(KeysCompiler, [ diff --git a/developer/src/kmc-ldml/test/test-layr.ts b/developer/src/kmc-ldml/test/test-layr.ts index ec255ec47d..7c99cd5b8a 100644 --- a/developer/src/kmc-ldml/test/test-layr.ts +++ b/developer/src/kmc-ldml/test/test-layr.ts @@ -36,7 +36,7 @@ describe('layr', function () { assert.equal(layer0.rows.length, 1); const row0 = layer0.rows[0]; assert.ok(row0); - assert.equal(row0.keys.length, 1); + assert.equal(row0.keys.length, 2); assert.equal(layer0.id.value, 'base'); assert.equal(layer0.mod, constants.keys_mod_none);