feat(core,common,developer): support UTF-32 single chars 🙀

- C++ side now
- updated tests
- refactored escaping code and squashed bugs
- still todo: basic.xml

For: #9050

common/web/types/src/kmx/kmx-plus.ts

@@ -2,7 +2,7 @@ import { constants } from '@keymanapp/ldml-keyboard-constants';
 import * as r from 'restructure';
 import { ElementString } from './element-string.js';
 import { ListItem } from './string-list.js';
-import { unescapeString } from '../util/util.js';
+import { isOneChar, toOneChar, unescapeString } from '../util/util.js';
 import { KMXFile } from './kmx.js';
 import { UnicodeSetParser, UnicodeSet } from '@keymanapp/common-types';
 import { VariableParser } from '../ldml-keyboard/pattern-parser.js';
@@ -111,11 +111,6 @@ export class StrsItem {
     return 0;
   }
 
-  /** True if this string *could* be a UTF-32 single char */
-  static isOneChar(value: string) : boolean {
-    return (value.split('').length) == 1;
-  }
-
   get isOneChar() {
     return this.char !== undefined;
   }
@@ -126,10 +121,10 @@ export class StrsItem {
  */
 export class CharStrsItem extends StrsItem {
   constructor(value: string) {
-    if (!StrsItem.isOneChar(value)) {
+    if (!isOneChar(value)) {
       throw RangeError(`not a 1-char string`);
     }
-    super(value, value.charCodeAt(0));
+    super(value, toOneChar(value));
   }
 };
 
@@ -161,7 +156,7 @@ export class Strs extends Section {
     }
 
     // if it's a single char, don't push it into the list
-    if (singleOk && StrsItem.isOneChar(s)) {
+    if (singleOk && isOneChar(s)) {
       return new CharStrsItem(s);
     }
 

common/web/types/src/util/util.ts

@@ -66,3 +66,17 @@ export function unescapeString(s: string): string {
 
   return s;
 }
+
+/** True if this string *could* be a UTF-32 single char */
+export function
+isOneChar(value: string) : boolean {
+  return [...value].length === 1;
+}
+
+export function
+toOneChar(value: string) : number {
+  if (!isOneChar(value)) {
+    throw Error(`Not a single char: ${value}`);
+  }
+  return value.codePointAt(0);
+}

common/web/types/test/util/test-unescape.ts

@@ -1,6 +1,26 @@
 import 'mocha';
 import {assert} from 'chai';
-import {unescapeString, UnescapeError} from '../../src/util/util.js';
+import {unescapeString, UnescapeError, isOneChar, toOneChar} from '../../src/util/util.js';
+
+describe('test UTF32 functions()', function() {
+  it('should properly categorize strings', () => {
+    [
+      'x',
+      '🙀',
+    ].forEach(s => assert.isTrue(isOneChar(s), `isOneChar(${s})`));
+
+    [
+      'xx',
+      '🙀🙀',
+    ].forEach(s => assert.isFalse(isOneChar(s), `!isOneChar(${s})`));
+  });
+
+  it('should convert to single chars', function() {
+    assert.equal(toOneChar('ħ'), 295);
+    assert.equal(toOneChar('🙀'), 0x1F640);
+    assert.throws(() => toOneChar('ħħħ'), /Not a single char/);
+  });
+});
 
 describe('test unescapeString()', function() {
   it("should correctly handle multi strings", function() {

core/src/kmx/kmx_plus.cpp

@@ -270,6 +270,18 @@ COMP_KMXPLUS_STRS::valid(KMX_DWORD _kmn_unused(length)) const {
   return true;
 }
 
+/**
+ * helper for extracting single char values
+ * @param v field with char type
+ * @return value a string
+ */
+std::u16string COMP_KMXPLUS_STRS::str_from_char(KMX_DWORD v) {
+  char16_single buf;
+  const int len = Utf32CharToUtf16(v, buf);
+  return std::u16string(buf.ch, len);
+}
+
+
 bool
 COMP_KMXPLUS_SECT::valid(KMX_DWORD length) const {
   DebugLog("sect: total 0x%X\n", total);
@@ -357,12 +369,11 @@ COMP_KMXPLUS_ELEM::getElementList(KMX_DWORD elementNumber, KMX_DWORD &length) co
   return reinterpret_cast<const COMP_KMXPLUS_ELEM_ELEMENT *>(rawdata + entry.offset);
 }
 
+
 std::u16string
-COMP_KMXPLUS_ELEM_ELEMENT::get_string() const {
+COMP_KMXPLUS_ELEM_ELEMENT::get_element_string() const {
   assert((flags & LDML_ELEM_FLAGS_TYPE) == LDML_ELEM_FLAGS_TYPE_CHAR); // should only be called on char
-  char16_single buf;
-  const int len = Utf32CharToUtf16(element, buf);
-  return std::u16string(buf.ch, len);
+  return COMP_KMXPLUS_STRS::str_from_char(element);
 }
 
 // Note: shared with subclass COMP_KMXPLUS_BKSP
@@ -802,9 +813,13 @@ COMP_KMXPLUS_KEYS_Helper::findKeyByStringId(KMX_DWORD strId, KMX_DWORD &i) const
 }
 
 const COMP_KMXPLUS_KEYS_KEY*
-COMP_KMXPLUS_KEYS_Helper::findKeyByStringTo(KMX_DWORD strId, KMX_DWORD &i) const {
+COMP_KMXPLUS_KEYS_Helper::findKeyByStringTo(const std::u16string& str, KMX_DWORD strId, KMX_DWORD &i) const {
   for (; i < key2->keyCount; i++) {
-    if (keys[i].to == strId) {
+    if (keys[i].flags & LDML_KEYS_KEY_FLAGS_EXTEND) {
+      if (strId != 0 && keys[i].to == strId) {
+        return &keys[i];
+      }
+    } else if (keys[i].get_to_string() == str) {
       return &keys[i];
     }
   }
@@ -839,11 +854,9 @@ COMP_KMXPLUS_KEYS_Helper::getKmap(KMX_DWORD i) const {
 }
 
 std::u16string
-COMP_KMXPLUS_KEYS_KEY::get_string() const {
+COMP_KMXPLUS_KEYS_KEY::get_to_string() const {
   assert(!(flags & LDML_KEYS_KEY_FLAGS_EXTEND)); // should not be called.
-  char16_single buf;
-  const int len = Utf32CharToUtf16(to, buf);
-  return std::u16string(buf.ch, len);
+  return COMP_KMXPLUS_STRS::str_from_char(to);
 }
 
 // LIST
@@ -955,7 +968,7 @@ kmx_plus::kmx_plus(const COMP_KEYBOARD *keyboard, size_t length)
   const COMP_KEYBOARD_EX* ex = reinterpret_cast<const COMP_KEYBOARD_EX*>(keyboard);
 
   DebugLog("kmx_plus(): KMXPlus offset 0x%X, KMXPlus size 0x%X\n", ex->kmxplus.dpKMXPlus, ex->kmxplus.dwKMXPlusSize);
-  if (ex->kmxplus.dpKMXPlus < sizeof(kmx::COMP_KEYBOARD_EX)) {
+  if (ex->kmxplus.dpKMXPlus < sizeof(COMP_KEYBOARD_EX)) {
     DebugLog("dwKMXPlus is not past the end of COMP_KEYBOARD_EX");
     valid = false;
     assert(valid);

core/src/kmx/kmx_plus.h

@@ -93,11 +93,11 @@ struct COMP_KMXPLUS_ELEM_ELEMENT {
     KMX_DWORD element;                // str: output string or UTF-32LE codepoint
     KMX_DWORD flags;                  // flag and order values
     /**
-     * @brief Get the 'to' as a string, if flags&LDML_ELEM_FLAGS_TYPE = CHAR
+     * @brief Get the 'element' as a string, if flags&LDML_ELEM_FLAGS_TYPE = CHAR
      *
      * @return std::u16string
      */
-    std::u16string get_string() const;
+    std::u16string get_element_string() const;
 };
 
 struct COMP_KMXPLUS_ELEM_ENTRY {
@@ -234,6 +234,9 @@ struct COMP_KMXPLUS_STRS {
    * @brief True if section is valid.
    */
   bool valid(KMX_DWORD length) const;
+
+  /** convert a single char to a string*/
+  static std::u16string str_from_char(KMX_DWORD v);
 };
 
 static_assert(sizeof(struct COMP_KMXPLUS_STRS) % 0x4 == 0, "Structs prior to variable part should align to 32-bit boundary");
@@ -506,6 +509,8 @@ struct COMP_KMXPLUS_KEYS_FLICK_ELEMENT {
   KMXPLUS_LIST directions;
   KMX_DWORD flags;
   KMXPLUS_STR to; // string or codepoint
+  /** get the 'to' string if a char */
+  std::u16string get_to_string() const;
 };
 
 struct COMP_KMXPLUS_KEYS_FLICK_LIST {
@@ -525,7 +530,7 @@ struct COMP_KMXPLUS_KEYS_KEY {
   KMXPLUS_LIST multiTap;
   KMX_DWORD flicks; // index
 
-  std::u16string get_string() const;
+  std::u16string get_to_string() const;
 };
 
 struct COMP_KMXPLUS_KEYS_KMAP {
@@ -558,11 +563,12 @@ public:
   const COMP_KMXPLUS_KEYS_KEY *findKeyByStringId(KMX_DWORD strId, KMX_DWORD &index) const;
   /**
    * Search for a key by 'to' string id
+   * @param str string to search for (for single char strings)
    * @param strID id to search for
    * @param index on entry, id to start with such as 0. On exit, index of item if found. Undefined otherwise.
    * @return pointer to key or nullptr
    */
-  const COMP_KMXPLUS_KEYS_KEY *findKeyByStringTo(KMX_DWORD strId, KMX_DWORD &index) const;
+  const COMP_KMXPLUS_KEYS_KEY *findKeyByStringTo(const std::u16string& str, KMX_DWORD strId, KMX_DWORD &index) const;
 
 private:
   const COMP_KMXPLUS_KEYS *key2;

core/src/ldml/ldml_processor.cpp

@@ -71,7 +71,8 @@ ldml_processor::ldml_processor(path const & kb_path, const std::vector<uint8_t>
       auto keyEntry = kplus.key2Helper.getKeys(kmapEntry->key);
       assert(keyEntry != nullptr);
 
-      if (keyEntry->flags && LDML_KEYS_KEY_FLAGS_EXTEND) {
+      // TODO-LDML: LDML_KEYS_KEY_FLAGS_NOTRANSFORM
+      if (keyEntry->flags & LDML_KEYS_KEY_FLAGS_EXTEND) {
         if (nullptr == kplus.strs) {
           DebugLog("for keys: kplus.strs == nullptr"); // need a string table to get strings
           assert(false);
@@ -79,7 +80,7 @@ ldml_processor::ldml_processor(path const & kb_path, const std::vector<uint8_t>
         }
         str = kplus.strs->get(keyEntry->to);
       } else {
-        str = keyEntry->get_string();
+        str = keyEntry->get_to_string();
       }
       keys.add((km_kbp_virtual_key)kmapEntry->vkey, (uint16_t)kmapEntry->mod, str);
     }

core/tests/unit/ldml/ldml_test_source.cpp

@@ -565,18 +565,11 @@ LdmlJsonRepertoireTestSource::next_action(ldml_action &fillin) {
 
   // First, find the string as an id
   // TODO-LDML: will not work for multi string cases
-  KMX_DWORD strId = kmxplus->strs->find(chstr);
-  if (strId == 0) { // will also get here if id is empty.
-    fillin.string = u"No string for repertoire test: ";
-    fillin.string.append(chstr);
-    fillin.type = LDML_ACTION_FAIL;
-    return;
-  }
-  assert(strId != 0);
+  KMX_DWORD strId = kmxplus->strs->find(chstr); // not an error if chstr is 0, may be single ch
 
   // OK. Now we can search the keybag
   KMX_DWORD keyIndex = 0;
-  auto *key2 = kmxplus->key2Helper.findKeyByStringTo(strId, keyIndex);
+  auto *key2 = kmxplus->key2Helper.findKeyByStringTo(chstr, strId, keyIndex);
   if (key2 == nullptr) {
     fillin.string = u"No key for repertoire test: ";
     fillin.string.append(chstr);

core/tests/unit/ldml/test_kmx_plus.cpp

@@ -32,24 +32,24 @@ int test_COMP_KMXPLUS_KEYS_KEY() {
           0x0001F640,  // to
           0x00000000   // flags: CHAR
       }};
-  std::u16string s0 = e[0].get_string();
+  std::u16string s0 = e[0].get_to_string();
   assert_equal(s0.length(), 1);
   assert_equal(s0.at(0), 0x0127);
   assert(s0 == std::u16string(u"ħ"));
 
-  std::u16string s1 = e[1].get_string();
+  std::u16string s1 = e[1].get_to_string();
   assert_equal(s1.length(), 2);
   assert_equal(s1.at(0), 0xD83D);
   assert_equal(s1.at(1), 0xDE40);
   assert(s1 == std::u16string(u"🙀"));
 
   // now, elems. Parallel.
-  std::u16string es0 = elems[0].get_string();
+  std::u16string es0 = elems[0].get_element_string();
   assert_equal(es0.length(), 1);
   assert_equal(es0.at(0), 0x0127);
   assert(es0 == std::u16string(u"ħ"));
 
-  std::u16string es1 = elems[1].get_string();
+  std::u16string es1 = elems[1].get_element_string();
   assert_equal(es1.length(), 2);
   assert_equal(es1.at(0), 0xD83D);
   assert_equal(es1.at(1), 0xDE40);

developer/src/kmc-ldml/src/compiler/keys.ts

@@ -101,10 +101,11 @@ export class KeysCompiler extends SectionCompiler {
 
       for (let lkflick of lkflicks.flick) {
         let flags = 0;
-        // TODO-LDML: single char
         const to = sections.strs.allocAndUnescapeString(lkflick.to, true);
-        flags |= to.isOneChar ? constants.keys_flick_flags_extend : 0;
-        let directions : ListItem = sections.list.allocListFromSpaces(sections.strs, lkflick.directions);
+        if (!to.isOneChar) {
+          flags |= constants.keys_flick_flags_extend;
+        }
+        let directions: ListItem = sections.list.allocListFromSpaces(sections.strs, lkflick.directions);
         flicks.flicks.push({
           directions,
           flags,
@@ -138,8 +139,10 @@ export class KeysCompiler extends SectionCompiler {
       const longPressDefault = sections.strs.allocAndUnescapeString(key.longPressDefault);
       const multiTap: ListItem = sections.list.allocListFromEscapedSpaces(sections.strs, key.multiTap);
       const keySwitch = sections.strs.allocString(key.switch); // 'switch' is a reserved word
-      const to = sections.strs.allocAndUnescapeString(key.to, true); // TODO-LDML: single char
-      flags |= to.isOneChar ? constants.keys_key_flags_extend : 0;
+      const to = sections.strs.allocAndUnescapeString(key.to, true);
+      if (!to.isOneChar) {
+        flags |= constants.keys_key_flags_extend;
+      }
       const width = Math.ceil((key.width || 1) * 10.0);  // default, width=1
       sect.keys.push({
         flags,

developer/src/kmc-ldml/test/fixtures/sections/keys/minimal.xml

@@ -8,11 +8,12 @@
 
   <keys>
     <key id="grave" to="🪦" />
+    <key id="mistake" to="oops" />
   </keys>
 
   <layers form="us">
     <layer id="base">
-      <row keys="grave" />
+      <row keys="grave mistake" />
     </layer>
   </layers>
 

developer/src/kmc-ldml/test/test-keys.ts

@@ -18,10 +18,16 @@ describe('keys', function () {
         const keys = <Keys> sect;
         assert.ok(keys);
         assert.equal(compilerTestCallbacks.messages.length, 0);
-        assert.equal(keys.keys.length, 1);
+        assert.equal(keys.keys.length, 2);
         assert.equal(keys.flicks.length, 1); // there's always a 'null' flick
-        assert.equal(keys.keys[0].to.value, String.fromCodePoint(0x1FAA6));
-        assert.equal(keys.keys[0].id.value, 'grave');
+        assert.equal(keys.keys[0].to.value, 'oops');
+        assert.isFalse(keys.keys[0].to.isOneChar);
+        assert.equal(keys.keys[0].flags, constants.keys_key_flags_extend);
+        assert.equal(keys.keys[0].id.value, 'mistake');
+        assert.isTrue(keys.keys[1].to.isOneChar);
+        assert.equal(keys.keys[1].to.value, String.fromCodePoint(0x1FAA6));
+        assert.equal(keys.keys[1].flags, 0);
+        assert.equal(keys.keys[1].id.value, 'grave');
       },
     },
     {
@@ -120,7 +126,7 @@ describe('keys.kmap', function () {
     let keys = await loadSectionFixture(KeysCompiler, 'sections/keys/minimal.xml', compilerTestCallbacks) as Keys;
     assert.isNotNull(keys);
     assert.equal(compilerTestCallbacks.messages.length, 0);
-    assert.equal(keys.kmap.length, 1);
+    assert.equal(keys.kmap.length, 2);
   });
 
   testCompilationCases(KeysCompiler, [

developer/src/kmc-ldml/test/test-layr.ts

@@ -36,7 +36,7 @@ describe('layr', function () {
     assert.equal(layer0.rows.length, 1);
     const row0 = layer0.rows[0];
     assert.ok(row0);
-    assert.equal(row0.keys.length, 1);
+    assert.equal(row0.keys.length, 2);
 
     assert.equal(layer0.id.value, 'base');
     assert.equal(layer0.mod, constants.keys_mod_none);