Merge pull request #7443 from keymanapp/fix/core/7418-utf32-in-epic-ldml

feat(core): ldml: Use new UTF-16 to UTF-32 function 🙀

core/src/ldml/ldml_processor.cpp

@@ -168,8 +168,22 @@ ldml_processor::process_event(
 
     switch (vk) {
     case KM_KBP_VKEY_BKSP:
-      state->context().pop_back();
-      state->actions().push_backspace(KM_KBP_BT_UNKNOWN); // Assuming we don't know the character
+      {
+        KMX_DWORD last_char = 0UL; // Assuming we don't know the character
+        // attempt to get the last char
+        auto end = state->context().rbegin();
+        if(end != state->context().rend()) {
+          if((*end).type == KM_KBP_CT_CHAR) {
+            last_char = (*end).character;
+          }
+        }
+        if (last_char == 0UL) {
+          state->actions().push_backspace(KM_KBP_BT_UNKNOWN);
+        } else {
+          state->actions().push_backspace(KM_KBP_BT_CHAR, last_char);
+        }
+        state->context().pop_back();
+      }
       break;
     default:
       // Look up the key
@@ -179,10 +193,10 @@ ldml_processor::process_event(
         state->actions().commit(); // finish up and
         return KM_KBP_STATUS_OK; // Nothing to do- no key
       }
-      for(size_t i=0; i<str.length(); i++) {
-        // TODO-LDML: needs to be per UTF-32 char? Seems this would push surrogates.
-        state->context().push_character(str[i]);
-        state->actions().push_character(str[i]);
+      const std::u32string str32 = kmx::u16string_to_u32string(str);
+      for(size_t i=0; i<str32.length(); i++) {
+        state->context().push_character(str32[i]);
+        state->actions().push_character(str32[i]);
       }
     }
     state->actions().commit();

core/tests/unit/ldml/keyboards/k_002_tinyu32.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+
+@@keys: [K_BKQUOTE][K_1][K_BKQUOTE][K_2][K_BKSP][K_2][K_BKQUOTE]
+@@expected: \u0127\u1790\u17B6\u0127\uD83D\uDE40\u0127
+
+-->
+<!DOCTYPE keyboard SYSTEM "../../../../../resources/standards-data/ldml-keyboards/techpreview/ldmlKeyboard.dtd">
+<keyboard locale="mt" conformsTo="techpreview">
+  <info author="srl295" indicator="🙀" layout="qwerty" normalization="NFC" />
+
+  <names>
+    <name value="TestKbd-UTF32" />
+  </names>
+
+  <keys>
+    <key id="hmaqtua" to="ħ" />
+    <key id="that" to="ថា" />
+    <key id="screamcat" to="🙀" />
+  </keys>
+
+  <layerMaps form="hardware">
+    <layerMap id="base">
+      <row keys="hmaqtua that screamcat" /> <!-- number row -->
+    </layerMap>
+  </layerMaps>
+</keyboard>

core/tests/unit/ldml/keyboards/meson.build

@@ -7,7 +7,8 @@
 tests = [
 # disabling 000 until we have updates to core or to the keyboard so that it passes
 #  'k_000_null_keyboard',
-  'k_001_tiny'
+  'k_001_tiny',
+  'k_002_tinyu32',
 ]
 
 # Setup kmc