feat(core): implement loading KMX from blob

- split keyboard loading into loading KMX file into blob and then
  loading the keyboard processor from the blob.
- deprecate `km_core_keyboard_load`
- move file access next to deprecated method. This is now the only place
  that loads a file in Core; unit tests have some more places that
  load files.
- introduce GTest and add unit tests for loading from blob

Cherry-picked from `epic/web-core` branch.

Cherry-Pick-Commit: 1deaa323ad
Cherry-Pick-Commit: 59019cc8b7
Cherry-Pick-Commit: bc46458368
Cherry-Pick-Commit: d06aa29956
Cherry-Pick-Commit: 1c88166f6e
Cherry-Pick-Commit: 069cd21ecd
Cherry-Pick-Commit: 052ae2ec35
Cherry-Pick-Commit: 11a2a3ba3a

Part-of: #11293
Part-of: #8093

core/include/keyman/keyman_core_api.h

@@ -1007,7 +1007,11 @@ Provides read-only information about a keyboard.
 typedef struct {
   km_core_cu const * version_string;
   km_core_cu const * id;
+
+  // TODO-web-core: Deprecate this field (#12497)
+  // KMN_DEPRECATED
   km_core_path_name  folder_path;
+
   km_core_option_item const * default_options;
 } km_core_keyboard_attrs;
 
@@ -1022,7 +1026,7 @@ typedef struct {
 : Keyman keyboard ID string.
 
 `folder_path`
-: Path to the unpacked folder containing the keyboard and associated resources.
+: Path to the unpacked folder containing the keyboard and associated resources (deprecated).
 
 `default_options`
 : Set of default values for any options included in the keyboard.
@@ -1096,12 +1100,16 @@ typedef struct {
 
 ## Description
 
+DEPRECATED: use [km_core_keyboard_load_from_blob] instead.
+
 Parse and load keyboard from the supplied path and a pointer to the loaded keyboard
-into the out paramter.
+into the out parameter.
 
 ## Specification
 
 ```c */
+// TODO-web-core: Deprecate this function (#12497)
+// KMN_DEPRECATED_API
 KMN_API
 km_core_status
 km_core_keyboard_load(km_core_path_name kb_path,
@@ -1140,6 +1148,60 @@ km_core_keyboard_load(km_core_path_name kb_path,
 
 -------------------------------------------------------------------------------
 
+# km_core_keyboard_load_from_blob()
+
+## Description
+
+Parse and load keyboard from the supplied blob and a pointer to the loaded keyboard
+into the out paramter.
+
+## Specification
+
+```c */
+KMN_API
+km_core_status km_core_keyboard_load_from_blob(const km_core_path_name kb_name,
+                                               const void* blob,
+                                               const size_t blob_size,
+                                               km_core_keyboard** keyboard);
+
+/*
+```
+
+## Parameters
+
+`kb_name`
+: a string with the name of the keyboard.
+
+`blob`
+: a byte array containing the content of a KMX/KMX+ file.
+
+`blob_size`
+: a size_t variable with the size of the blob in bytes.
+
+`keyboard`
+: A pointer to result variable: A pointer to the opaque keyboard
+  object returned by the Processor. This memory must be freed with a
+  call to [km_core_keyboard_dispose].
+
+## Returns
+
+`KM_CORE_STATUS_OK`
+: On success.
+
+`KM_CORE_STATUS_NO_MEM`
+: In the event an internal memory allocation fails.
+
+`KM_CORE_STATUS_IO_ERROR`
+: In the event the keyboard file is unparseable for any reason
+
+`KM_CORE_STATUS_INVALID_ARGUMENT`
+: In the event `keyboard` is null.
+
+`KM_CORE_STATUS_OS_ERROR`
+: Bit 31 (high bit) set, bits 0-30 are an OS-specific error code.
+
+-------------------------------------------------------------------------------
+
 # km_core_keyboard_dispose()
 
 ## Description

core/include/keyman/keyman_core_api_bits.h

@@ -23,7 +23,6 @@
   #define _kmn_unused(x)        UNUSED_ ## x __attribute__((__unused__))
 #else
   #define _kmn_unused(x)        UNUSED_ ## x
-
 #endif
 
 #if defined _WIN32 || defined __CYGWIN__
@@ -36,7 +35,7 @@
     #undef _kmn_static_flag
   #else  // How MSVC sepcifies function level attributes adn deprecation
     #define _kmn_and
-    #define _kmn_tag_fn(a)       __declspec(a)
+    #define _kmn_tag_fn(a)     __declspec(a)
     #define _kmn_deprecated_flag deprecated
   #endif
   #define _kmn_export_flag     dllexport
@@ -48,6 +47,8 @@
   #define _KM_CORE_EXT_SEPARATOR ('.')
 #endif
 
+#define KMN_DEPRECATED        _kmn_tag_fn(_kmn_deprecated_flag)
+
 #if defined KM_CORE_LIBRARY_STATIC
   #define KMN_API             _kmn_tag_fn(_kmn_static_flag)
   #define KMN_DEPRECATED_API  _kmn_tag_fn(_kmn_deprecated_flag _kmn_and _kmn_static_flag)

core/src/keyboard.cpp

@@ -17,18 +17,16 @@ void keyboard_attributes::render()
   // Make attributes point to the stored values above.
   id              = _keyboard_id.c_str();
   version_string  = _version_string.c_str();
-  folder_path     = _folder_path.c_str();
   default_options = _default_opts.data();
 }
 
 
 keyboard_attributes::keyboard_attributes(std::u16string const & kbid,
     std::u16string const & version,
-    path_type const & path,
     options_store const &opts)
 : _keyboard_id(kbid),
   _version_string(version),
-  _folder_path(path),
+  _folder_path(""),
   _default_opts(opts)
 {
   // Ensure that the default_options array will be properly terminated.
@@ -40,7 +38,7 @@ keyboard_attributes::keyboard_attributes(std::u16string const & kbid,
 keyboard_attributes::keyboard_attributes(keyboard_attributes &&rhs)
 : _keyboard_id(std::move(rhs._keyboard_id)),
   _version_string(std::move(rhs._version_string)),
-  _folder_path(std::move(rhs._folder_path)),
+  _folder_path(""),
   _default_opts(std::move(rhs._default_opts))
 {
     rhs.id = rhs.version_string = nullptr;
@@ -58,7 +56,6 @@ json & km::core::operator << (json & j, km::core::keyboard_attributes const & kb
 {
   j << json::object
       << "id" << kb.id
-      << "folder" << kb._folder_path
       << "version" << kb.version_string
       << "rules" << json::array << json::close;
 

core/src/keyboard.hpp

@@ -26,6 +26,7 @@ namespace core
   {
     std::u16string      _keyboard_id;
     std::u16string      _version_string;
+    // unused and deprecated
     core::path           _folder_path;
     std::vector<option> _default_opts;
 
@@ -33,7 +34,6 @@ namespace core
 
   public:
     using options_store = decltype(_default_opts);
-    using path_type = decltype(_folder_path);
 
     keyboard_attributes()
     : km_core_keyboard_attrs {nullptr, nullptr, nullptr, nullptr} {}
@@ -42,7 +42,6 @@ namespace core
 
     keyboard_attributes(std::u16string const & id,
              std::u16string const & version,
-             path_type const & path,
              options_store const &opts);
 
     keyboard_attributes & operator = (keyboard_attributes const &) = delete;
@@ -52,8 +51,6 @@ namespace core
 
     options_store const &   default_opts_store() const noexcept { return _default_opts; }
     options_store &         default_opts_store() noexcept { return _default_opts; }
-
-    path_type const     &   path() const noexcept { return _folder_path; }
   };
 
   json & operator << (json &, km::core::keyboard_attributes const &);

core/src/km_core_keyboard_api.cpp

@@ -14,47 +14,125 @@
 #include "keyman_core.h"
 
 #include "keyboard.hpp"
-#include "processor.hpp"
 #include "kmx/kmx_processor.hpp"
 #include "ldml/ldml_processor.hpp"
 #include "mock/mock_processor.hpp"
+#include "processor.hpp"
+#include "utfcodec.hpp"
 
 using namespace km::core;
 
 namespace
 {
-  abstract_processor * processor_factory(path const & kb_path) {
-    // Some legacy packages may include upper-case file extensions
-    // TODO-LDML: move file io out of core and into engine
-    if (kb_path.suffix() == ".kmx" || kb_path.suffix() == ".KMX") {
-      std::vector<uint8_t> buf;
-      if(ldml_processor::is_kmxplus_file(kb_path, buf)) {
-        abstract_processor * result = new ldml_processor(kb_path, buf);
-        return result;
-      }
-      return new kmx_processor(kb_path);
+  abstract_processor* processor_factory(path const & kb_name, const std::vector<uint8_t> & buf) {
+    if (ldml_processor::is_handled(buf)) {
+      return new ldml_processor(kb_name, buf);
     }
-    else if (kb_path.suffix() == ".mock") {
-      return new mock_processor(kb_path);
+    if (kmx_processor::is_handled(buf)) {
+      return new kmx_processor(kb_name, buf);
     }
-    else {
-      return new null_processor();
+    if (mock_processor::is_handled(buf)) {
+      return new mock_processor(kb_name);
     }
+    return new null_processor();
   }
 
+}  // namespace
 
-}
 km_core_status
-km_core_keyboard_load(km_core_path_name kb_path, km_core_keyboard **keyboard)
+keyboard_load_from_blob_internal(
+  const km_core_path_name kb_name,
+  const std::vector<uint8_t> & buf,
+  km_core_keyboard** keyboard
+) {
+  assert(keyboard);
+  if (!keyboard) {
+    return KM_CORE_STATUS_INVALID_ARGUMENT;
+  }
+
+  *keyboard = nullptr;
+  try {
+    abstract_processor* kp = processor_factory(kb_name, buf);
+    km_core_status status  = kp->validate();
+    if (status != KM_CORE_STATUS_OK) {
+      delete kp;
+      return status;
+    }
+    *keyboard = static_cast<km_core_keyboard*>(kp);
+  } catch (std::bad_alloc&) {
+    return KM_CORE_STATUS_NO_MEM;
+  }
+  return KM_CORE_STATUS_OK;
+}
+
+km_core_status
+km_core_keyboard_load_from_blob(
+  const km_core_path_name kb_name,
+  const void* blob,
+  const size_t blob_size,
+  km_core_keyboard** keyboard
+) {
+  assert(keyboard);
+  if (!keyboard || !blob) {
+    return KM_CORE_STATUS_INVALID_ARGUMENT;
+  }
+
+  std::vector<uint8_t> buf((uint8_t*)blob, (uint8_t*)blob + blob_size);
+  return keyboard_load_from_blob_internal(kb_name, buf, keyboard);
+}
+
+// TODO-web-core: Remove this code when we remove the deprecated km_core_keyboard_load method
+// BEGIN DEPRECATED
+#include <fstream>
+std::vector<uint8_t> load_kmx_file(path const& kb_path) {
+  std::vector<uint8_t> data;
+  std::ifstream file(static_cast<std::string>(kb_path), std::ios::binary | std::ios::ate);
+  if (!file.good()) {
+    return std::vector<uint8_t>();
+  }
+  const std::streamsize size = file.tellg();
+  if (size >= KMX_MAX_ALLOWED_FILE_SIZE) {
+    return std::vector<uint8_t>();
+  }
+
+  file.seekg(0, std::ios::beg);
+
+  data.resize((size_t)size);
+  if (!file.read((char*)data.data(), size)) {
+    return std::vector<uint8_t>();
+  }
+
+  file.close();
+  return data;
+}
+
+KMN_DEPRECATED_API
+km_core_status
+km_core_keyboard_load(km_core_path_name kb, km_core_keyboard **keyboard)
 {
   assert(keyboard);
-  if (!keyboard)
+  if (!keyboard || !kb) {
     return KM_CORE_STATUS_INVALID_ARGUMENT;
+  }
 
+  path const kb_path(kb);
   try
   {
-    abstract_processor *kp = processor_factory(kb_path);
-    km_core_status status = kp->validate();
+    abstract_processor* kp = nullptr;
+    km_core_status status = KM_CORE_STATUS_OK;
+    // Some legacy packages may include upper-case file extensions
+    if (kb_path.suffix() == ".kmx" || kb_path.suffix() == ".KMX") {
+      std::vector<uint8_t> buf = load_kmx_file(kb_path);
+      status = keyboard_load_from_blob_internal(kb_path.stem().c_str(), buf, (km_core_keyboard**)&kp);
+      if (status != KM_CORE_STATUS_OK) {
+        return status;
+      }
+    } else if (kb_path.suffix() == ".mock") {
+      kp = new mock_processor(kb_path);
+    } else {
+      kp = new null_processor();
+    }
+    status = kp->validate();
     if (status != KM_CORE_STATUS_OK) {
       delete kp;
       return status;
@@ -67,6 +145,7 @@ km_core_keyboard_load(km_core_path_name kb_path, km_core_keyboard **keyboard)
   }
   return KM_CORE_STATUS_OK;
 }
+// END DEPRECATED
 
 void
 km_core_keyboard_dispose(km_core_keyboard *keyboard)

core/src/kmx/kmx_file.cpp

@@ -13,9 +13,9 @@ using namespace kmx;
 #include <share.h>
 #endif
 
-KMX_BOOL KMX_ProcessEvent::Load(km_core_path_name KeyboardName)
-{
-  if(!LoadKeyboard(KeyboardName, &m_keyboard.Keyboard)) return FALSE;   // I5136
+KMX_BOOL KMX_ProcessEvent::Load(PKMX_BYTE buf, size_t sz) {
+  if(!LoadKeyboardFromBlob(buf, sz, &m_keyboard.Keyboard))
+    return FALSE;   // I5136
 
   return TRUE;
 }
@@ -51,50 +51,22 @@ const int km::core::kmx::CODE__SIZE[] = {
 // Ensure that all CODE_### sizes are defined
 static_assert(sizeof(CODE__SIZE) / sizeof(CODE__SIZE[0]) == (CODE_LASTCODE + 1), "Size of array CODE__SIZE not correct");
 
-
-
-KMX_BOOL KMX_ProcessEvent::LoadKeyboard(km_core_path_name fileName, LPKEYBOARD *lpKeyboard)
-{
-  PKMX_BYTE buf;
-  FILE *fp;
+KMX_BOOL KMX_ProcessEvent::LoadKeyboardFromBlob(
+  PKMX_BYTE original_buf,
+  size_t sz,
+  LPKEYBOARD* lpKeyboard
+) {
   LPKEYBOARD kbp;
+  PKMX_BYTE buf;
   PKMX_BYTE filebase;
 
-  DebugLog("Loading file '%s'", fileName);
-  if(!fileName || !lpKeyboard)
-  {
-    DebugLog("Bad Filename");
+
+  if (!lpKeyboard || !original_buf) {
+    DebugLog("Invalid parameter");
     return FALSE;
   }
 
-#if defined(_WIN32) || defined(_WIN64)
-  fp = _wfsopen(fileName, L"rb", _SH_DENYWR);
-#else
-  fp = fopen(fileName, "rb");
-#endif
-  if(fp == NULL)
-  {
-    DebugLog("Could not open file");
-    return FALSE;
-  }
-
-  if (fseek(fp, 0, SEEK_END) != 0) {
-    fclose(fp);
-    DebugLog("Could not fseek file");
-    return FALSE;
-  }
-
-  auto sz = ftell(fp);
-  if (sz < 0) {
-    fclose(fp);
-    return FALSE;
-  }
-
-  if (fseek(fp, 0, SEEK_SET) != 0) {
-    fclose(fp);
-    DebugLog("Could not fseek(set) file");
-    return FALSE;
-  }
+  *lpKeyboard = NULL;
 
 #ifdef KMX_64BIT
   // allocate enough memory for expanded data structure + original data.
@@ -108,57 +80,55 @@ KMX_BOOL KMX_ProcessEvent::LoadKeyboard(km_core_path_name fileName, LPKEYBOARD *
   buf = new KMX_BYTE[sz];
 #endif
 
-  if(!buf)
-  {
-    fclose(fp);
+  if (!buf) {
     DebugLog("Not allocmem");
     return FALSE;
   }
 #ifdef KMX_64BIT
-  filebase = buf + sz*2;
+  filebase = buf + sz * 2;
 #else
   filebase = buf;
 #endif
+  memcpy(filebase, original_buf, sz);
 
-  if (fread(filebase, 1, sz, fp) < (size_t) sz) {
-    fclose(fp);
-    DebugLog("Could not read file");
+  if (*PKMX_DWORD(filebase) != KMX_DWORD(FILEID_COMPILED)) {
+    DebugLog("Invalid keyboard - signature is invalid");
+    delete[] buf;
     return FALSE;
   }
 
-  fclose(fp);
-
-  if(*PKMX_DWORD(filebase) != KMX_DWORD(FILEID_COMPILED))
-  {
-    delete [] buf;
-    DebugLog("Invalid file - signature is invalid");
+  if (!VerifyKeyboard(filebase, sz)) {
+    DebugLog("Verify keyboard failed");
+    delete[] buf;
     return FALSE;
   }
 
-  if(!VerifyKeyboard(filebase, sz)) return FALSE;
-
 #ifdef KMX_64BIT
   kbp = CopyKeyboard(buf, filebase);
 #else
   kbp = FixupKeyboard(buf, filebase);
 #endif
 
-  if(!kbp) return FALSE;
+  if (!kbp) {
+    DebugLog("Can't copy/fixup keyboard");
+    delete[] buf;
+    return FALSE;
+  }
 
-  if(kbp->dwIdentifier != FILEID_COMPILED) {
-    delete [] buf;
+  if (kbp->dwIdentifier != FILEID_COMPILED) {
     DebugLog("errNotFileID");
+    delete[] buf;
     return FALSE;
   }
 
   *lpKeyboard = kbp;
-
   return TRUE;
 }
 
 PKMX_WCHAR KMX_ProcessEvent::StringOffset(PKMX_BYTE base, KMX_DWORD offset)
 {
-  if(offset == 0) return NULL;
+  if(offset == 0)
+    return NULL;
   return (PKMX_WCHAR)(base + offset);
 }
 

core/src/kmx/kmx_processevent.h

@@ -54,7 +54,7 @@ private:
   KMX_DWORD m_modifiers = 0;
 
   /* File loading */
-  KMX_BOOL LoadKeyboard(km_core_path_name fileName, LPKEYBOARD *lpKeyboard);
+  KMX_BOOL LoadKeyboardFromBlob(PKMX_BYTE buf, size_t sz, LPKEYBOARD* lpKeyboard);
   KMX_BOOL VerifyKeyboard(PKMX_BYTE filebase, size_t sz);
   KMX_BOOL VerifyChecksum(PKMX_BYTE buf,  size_t sz);
 #ifdef KMX_64BIT
@@ -96,7 +96,7 @@ public:
   KMX_ProcessEvent();
   ~KMX_ProcessEvent();
 
-  KMX_BOOL Load(km_core_path_name keyboardName);
+  KMX_BOOL Load(PKMX_BYTE buf, size_t sz);
   KMX_BOOL ProcessEvent(km_core_state *state, KMX_UINT vkey, KMX_DWORD modifiers, KMX_BOOL isKeyDown);  // returns FALSE on error or key not matched
 
   KMX_Actions *GetActions();

core/src/kmx/kmx_processor.cpp

@@ -38,9 +38,8 @@ km_core_status kmx_processor::validate() const {
   return _valid ? KM_CORE_STATUS_OK : KM_CORE_STATUS_INVALID_KEYBOARD;
 }
 
-kmx_processor::kmx_processor(core::path p) {
-  p.replace_extension(".kmx");
-  _valid = bool(_kmx.Load(p.c_str()));
+kmx_processor::kmx_processor(std::u16string const& kb_name, const std::vector<uint8_t>& data) {
+  _valid = bool(_kmx.Load((PKMX_BYTE)data.data(), data.size()));
 
   if (!_valid)
     return;
@@ -57,8 +56,7 @@ kmx_processor::kmx_processor(core::path p) {
   auto v = _kmx.GetKeyboard()->Keyboard->version;
   auto vs = std::to_string(v >> 16) + "." + std::to_string(v & 0xffff);
 
-  _attributes = keyboard_attributes(static_cast<std::u16string>(p.stem()),
-                  std::u16string(vs.begin(), vs.end()), p.parent(), defaults);
+  _attributes = keyboard_attributes(kb_name, std::u16string(vs.begin(), vs.end()), defaults);
 }
 
 char16_t const *
@@ -414,3 +412,24 @@ km_core_keyboard_imx * kmx_processor::get_imx_list() const  {
   return imx_list;
 }
 
+/**
+ * Returns true the data is a KMX file, i.e. starts with 'KXTS'.
+ *
+ * @param data  the keyboard blob
+ * @return true if the processor can handle the keyboard, otherwise false.
+ */
+bool kmx_processor::is_handled(const std::vector<uint8_t>& data) {
+  if (data.empty()) {
+    return false;
+  }
+
+  if (data.size() < sizeof(COMP_KEYBOARD)) {  // a KMX file is at least 64 bytes (KMX header)
+    return false;
+  }
+
+  if (data.size() >= KMX_MAX_ALLOWED_FILE_SIZE) {
+    return false;
+  }
+
+  return ((kmx::PCOMP_KEYBOARD)data.data())->dwIdentifier == KMX_DWORD(FILEID_COMPILED);  // 'KXTS'
+}

core/src/kmx/kmx_processor.hpp

@@ -29,7 +29,7 @@ namespace core
       );
 
   public:
-    kmx_processor(path);
+    kmx_processor(std::u16string const& kb_name, const std::vector<uint8_t>& data);
 
     km_core_status
     process_event(
@@ -84,6 +84,8 @@ namespace core
     supports_normalization() const override {
       return false;
     }
+
+    static bool is_handled(const std::vector<uint8_t>& buf);
   };
 
 } // namespace core

core/src/ldml/ldml_processor.cpp

@@ -5,7 +5,6 @@
   Authors:      Marc Durdin (MD)
 */
 
-#include <fstream>
 #include <algorithm>
 #include "ldml/ldml_processor.hpp"
 #include "ldml/ldml_transforms.hpp"
@@ -15,6 +14,7 @@
 #include "kmx/kmx_plus.h"
 #include "kmx/kmx_xstring.h"
 #include "kmx/kmx_processevent.h"
+#include "kmx/kmx_processor.hpp"
 #include "ldml/keyman_core_ldml.h"
 #include "kmx/kmx_file_validator.hpp"
 #include "debuglog.h"
@@ -36,19 +36,15 @@ namespace {
 namespace km {
 namespace core {
 
-
-ldml_processor::ldml_processor(path const & kb_path, const std::vector<uint8_t> &data)
-: abstract_processor(
-    keyboard_attributes(kb_path.stem(), KM_CORE_LMDL_PROCESSOR_VERSION, kb_path.parent(), {})
-  ), _valid(false), transforms(), bksp_transforms(), keys(), normalization_disabled(false)
-{
-
+ldml_processor::ldml_processor(std::u16string const& kb_name, const std::vector<uint8_t>& data)
+    : abstract_processor(keyboard_attributes(kb_name, KM_CORE_LMDL_PROCESSOR_VERSION, {})),
+      _valid(false), transforms(), bksp_transforms(), keys(), normalization_disabled(false) {
   if(data.size() <= sizeof(kmx::COMP_KEYBOARD_EX)) {
     DebugLog("data.size %zu too small", data.size());
     return;
   }
 
-//   // Locate the structs here, but still retain ptrs to the raw structs.
+  // Locate the structs here, but still retain ptrs to the raw structs.
   kmx::KMX_FileValidator* comp_keyboard = (kmx::KMX_FileValidator*)data.data();
 
   // Perform the standard validation
@@ -116,40 +112,24 @@ ldml_processor::ldml_processor(path const & kb_path, const std::vector<uint8_t>
   _valid = true;
 }
 
-bool ldml_processor::is_kmxplus_file(path const & kb_path, std::vector<uint8_t>& data) {
-// TODO-LDML: we should refactor all the core components to delegate file loading
-//            to the Engine, which requires an API change, but this makes delivery
-//            of keyboard files more flexible under more WASM.
-
-  std::ifstream file(static_cast<std::string>(kb_path), std::ios::binary | std::ios::ate);
-  if(!file.good()) {
+/**
+ * Returns true if the data is a KMX+ file.
+ *
+ * @param data  the keyboard blob
+ * @return true if the processor can handle the keyboard, otherwise false.
+ */
+bool ldml_processor::is_handled(const std::vector<uint8_t>& data) {
+  // Check if it's a blob from a KMX file
+  if (!kmx_processor::is_handled(data)) {
     return false;
   }
-  const std::streamsize size = file.tellg();
-  if(size >= KMX_MAX_ALLOWED_FILE_SIZE) {
-    return false;
-  }
-
-  file.seekg(0, std::ios::beg);
-
-  data.resize((size_t)size);
-  if(!file.read((char *) data.data(), size)) {
-    return false;
-  }
-
-  file.close();
 
   const kmx::PCOMP_KEYBOARD comp_keyboard = (kmx::PCOMP_KEYBOARD)data.data();
-
-  if(comp_keyboard->dwIdentifier != KMX_DWORD(FILEID_COMPILED)) {
+  if (comp_keyboard->dwFileVersion < VERSION_160 || (comp_keyboard->dwFlags & KF_KMXPLUS) == 0) {
     return false;
   }
 
-  if(comp_keyboard->dwFileVersion < VERSION_160 || (comp_keyboard->dwFlags & KF_KMXPLUS) == 0) {
-    return false;
-  }
-
-  // A KMXPlus file is in the buffer (although more validation is required and will
+  // The buffer contains KMXPlus data (although more validation is required and will
   // be done in the constructor)
   return true;
 }

core/src/ldml/ldml_processor.hpp

@@ -30,13 +30,12 @@ class ldml_event_state;
 class ldml_processor : public abstract_processor {
   public:
     ldml_processor(
-      path const & kb_path,
+      std::u16string const& kb_name,
       const std::vector<uint8_t> & data
     );
 
     static bool is_kmxplus_file(
-      path const & kb_path,
-      std::vector<uint8_t>& data
+      const std::vector<uint8_t> & data
     );
 
     km_core_status
@@ -86,9 +85,11 @@ class ldml_processor : public abstract_processor {
       return !normalization_disabled;
     }
 
+    static bool is_handled(const std::vector<uint8_t> & buf);
+
   private:
     /** process a key-up */
-    void process_key_up(ldml_event_state &ldml_state) const;
+    void process_key_up(ldml_event_state& ldml_state) const;
 
     /** process a key-down (if it wasn't handled exceptionally) */
     void process_key_down(ldml_event_state &ldml_state) const;

core/src/meson.build

@@ -48,23 +48,27 @@ endif
 generated_headers = []
 
 if cpp_compiler.get_id() == 'emscripten'
+  links += [
+    '-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\',\'stringToNewUTF8\',\'wasmExports\']',
+    # Forcing inclusion of debug symbols
+    '-g', '-Wlimited-postlink-optimizations',
+    '-lembind'
+  ]
 
-util_normalize_table_generator = executable('util_normalize_table_generator',
-    ['util_normalize_table_generator.cpp'],
-    cpp_args: defns + warns,
-    include_directories: [inc],
-    link_args: links,
-    dependencies: [icu_uc, icu_i18n],
-    )
-
-util_normalize_table_h = custom_target('util_normalize_table.h',
-                    output: 'util_normalize_table.h',
-                    command: [util_normalize_table_generator],
-                    capture:true)
-
-generated_headers += util_normalize_table_h
+  util_normalize_table_generator = executable('util_normalize_table_generator',
+      ['util_normalize_table_generator.cpp'],
+      cpp_args: defns + warns,
+      include_directories: [inc],
+      link_args: links,
+      dependencies: [icu_uc, icu_i18n],
+      )
 
+  util_normalize_table_h = custom_target('util_normalize_table.h',
+                      output: 'util_normalize_table.h',
+                      command: [util_normalize_table_generator],
+                      capture:true)
 
+  generated_headers += util_normalize_table_h
 endif
 
 

core/src/mock/mock_processor.cpp

@@ -3,7 +3,7 @@
   Description:  This is a test implementation of the keyboard processor API to
                 enable testing API clients against a basic keyboard and give
                 them something to link against and load.
-                TODO: Add a mecahnism to trigger output of PERSIST_OPT &
+                TODO: Add a mechanism to trigger output of PERSIST_OPT &
                 RESET_OPT actions items, options support and context matching.
   Create Date:  17 Oct 2018
   Authors:      Tim Eves (TSE)
@@ -74,7 +74,7 @@ namespace km {
   {
     mock_processor::mock_processor(core::path const & path)
     : abstract_processor(
-        keyboard_attributes(path.stem(), u"3.145", path.parent(), {
+        keyboard_attributes(path.stem(), u"3.145", {
           option{KM_CORE_OPT_KEYBOARD, u"__test_point", u"not tiggered"},
         })),
       _options({
@@ -228,5 +228,23 @@ namespace km {
     km_core_status mock_processor::validate() const { return KM_CORE_STATUS_OK; }
 
     km_core_status null_processor::validate() const { return KM_CORE_STATUS_INVALID_ARGUMENT; }
-  } // namespace core
+
+    /**
+     * Returns true if the data starts with 'MOCK'.
+     *
+     * @param data  the keyboard blob
+     * @return true if the processor can handle the keyboard, otherwise false.
+     */
+    bool mock_processor::is_handled(const std::vector<uint8_t>& data) {
+      if (data.empty()) {
+        return false;
+      }
+
+      if (data.size() < 4) {  // a MOCK file is at least 4 bytes (MOCK)
+        return false;
+      }
+
+      return ((char*)data.data())[0] == 'M' && ((char*)data.data())[1] == 'O' && ((char*)data.data())[2] == 'C' && ((char*)data.data())[3] == 'K';
+    }
+  }  // namespace core
 } // namespace km

core/src/mock/mock_processor.hpp

@@ -69,13 +69,21 @@ namespace core
     supports_normalization() const override {
       return true;
     }
+
+    /**
+     * Returns true if the data starts with 'MOCK'
+     *
+     * @param buf   the keyboard blob
+     * @return true if the processor can handle the keyboard, otherwise false.
+     */
+    static bool is_handled(const std::vector<uint8_t>& data);
   };
 
   class null_processor : public mock_processor {
   public:
     null_processor(): mock_processor(path())
     {
-      _attributes = keyboard_attributes(u"null", u"0.0", path(), {});
+      _attributes = keyboard_attributes(u"null", u"0.0", {});
     }
 
     km_core_status validate() const override;

core/src/util_normalize_table_generator.cpp

@@ -17,7 +17,6 @@
 
 #include "core_icu.h"
 
-#include <fstream>
 #include <iostream>
 #include <string>
 #include <vector>

core/subprojects/.gitignore

@@ -2,3 +2,4 @@
 /*.zip
 /*.tgz
 /packagecache
+/googletest*/

core/subprojects/gtest.wrap

@@ -0,0 +1,16 @@
+[wrap-file]
+directory = googletest-1.14.0
+source_url = https://github.com/google/googletest/archive/refs/tags/v1.14.0.tar.gz
+source_filename = gtest-1.14.0.tar.gz
+source_hash = 8ad598c73ad796e0d8280b082cebd82a630d73e73cd3c70057938a6501bba5d7
+patch_filename = gtest_1.14.0-2_patch.zip
+patch_url = https://github.com/mesonbuild/wrapdb/releases/download/gtest_1.14.0-2/gtest_1.14.0-2_patch.zip
+patch_hash = 4ec7f767364386a99f7b2d61678287a73ad6ba0f9998be43b51794c464a63732
+source_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/gtest_1.14.0-2/gtest-1.14.0.tar.gz
+wrapdb_version = 1.14.0-2
+
+[provide]
+gtest = gtest_dep
+gtest_main = gtest_main_dep
+gmock = gmock_dep
+gmock_main = gmock_main_dep

core/tests/unit/km_core_keyboard_api.tests.cpp

@@ -0,0 +1,128 @@
+// Copyright (c) 2024 SIL International
+// This software is licensed under the MIT license (http://opensource.org/licenses/MIT)
+
+#include <fstream>
+#include <gtest/gtest.h>
+
+#include <keyman/keyman_core_api.h>
+#include "emscripten_filesystem.h"
+#include "load_kmx_file.hpp"
+
+km::core::path test_dir;
+
+// TODO-web-core: Remove this code when we remove the deprecated km_core_keyboard_load method
+// BEGIN DEPRECATED
+#if defined(__GNUC__) || defined(__clang__)
+#define PRAGMA(X) _Pragma(#X)
+#define DISABLE_WARNING_PUSH PRAGMA(GCC diagnostic push)
+#define DISABLE_WARNING_POP PRAGMA(GCC diagnostic pop)
+#define DISABLE_WARNING(W) PRAGMA(GCC diagnostic ignored #W)
+#define DISABLE_WARNING_DEPRECATED_DECLARATIONS DISABLE_WARNING(-Wdeprecated-declarations)
+#else
+#define DISABLE_WARNING_PUSH
+#define DISABLE_WARNING_POP
+#define DISABLE_WARNING_DEPRECATED_DECLARATIONS
+#endif
+
+class KmCoreKeyboardApiTests : public testing::Test {
+protected:
+  km_core_keyboard* keyboard = nullptr;
+  void TearDown() override {
+    if (this->keyboard) {
+      km_core_keyboard_dispose(this->keyboard);
+      this->keyboard = nullptr;
+    }
+    }
+};
+
+TEST_F(KmCoreKeyboardApiTests, LoadFromFile) {
+  // Setup
+  km::core::path kmxfile = km::core::path(test_dir / "kmx/k_020___deadkeys_and_backspace.kmx");
+
+  // Execute
+  DISABLE_WARNING_PUSH
+  DISABLE_WARNING_DEPRECATED_DECLARATIONS
+  auto status = km_core_keyboard_load(kmxfile.c_str(), &this->keyboard);
+  DISABLE_WARNING_POP
+
+  // Verify
+  EXPECT_EQ(status, KM_CORE_STATUS_OK);
+  EXPECT_TRUE(this->keyboard != nullptr);
+}
+// END DEPRECATED
+
+TEST_F(KmCoreKeyboardApiTests, LoadFromBlob) {
+  // Setup
+  km::core::path kmxfile = km::core::path(test_dir / "kmx/k_020___deadkeys_and_backspace.kmx");
+
+  std::vector<uint8_t> data = km::tests::load_kmx_file(kmxfile.native());
+  ASSERT_GT(data.size(), (size_t)0);
+
+  // Execute
+  auto status = km_core_keyboard_load_from_blob(kmxfile.stem().c_str(), data.data(), data.size(), &this->keyboard);
+
+  // Verify
+  EXPECT_EQ(status, KM_CORE_STATUS_OK);
+  EXPECT_TRUE(this->keyboard != nullptr);
+}
+
+TEST_F(KmCoreKeyboardApiTests, LoadFromBlobMock) {
+  // Setup
+  km::core::path kmxfile = "mock_keyboard.mock";
+  std::string blob_string = "MOCK";
+
+  std::vector<uint8_t> data = std::vector<uint8_t>(blob_string.begin(), blob_string.end());
+  ASSERT_GT(data.size(), (size_t)0);
+
+  // Execute
+  auto status = km_core_keyboard_load_from_blob(kmxfile.stem().c_str(), data.data(), data.size(), &this->keyboard);
+
+  // Verify
+  EXPECT_EQ(status, KM_CORE_STATUS_OK);
+  EXPECT_TRUE(this->keyboard != nullptr);
+}
+
+TEST_F(KmCoreKeyboardApiTests, LoadFromBlobNull) {
+  // Setup
+  km::core::path kmxfile  = "";
+
+  std::unique_ptr<uint8_t> data(new uint8_t[0]);
+
+  // Execute
+  auto status = km_core_keyboard_load_from_blob(kmxfile.stem().c_str(), data.get(), 0, &this->keyboard);
+
+  // Verify
+  EXPECT_EQ(status, KM_CORE_STATUS_INVALID_ARGUMENT);
+  EXPECT_TRUE(this->keyboard == nullptr);
+}
+
+TEST_F(KmCoreKeyboardApiTests, LoadFromBlobInvalidKeyboard) {
+  // Setup
+  km::core::path kmxfile  = "invalid_keyboard.kmx";
+  std::string blob_string = "KXTS";
+
+  std::vector<uint8_t> data = std::vector<uint8_t>(blob_string.begin(), blob_string.end());
+  for (auto i = data.size(); i < 64; i++) {
+    data.push_back(0);
+  }
+  ASSERT_GT(data.size(), (size_t)0);
+
+  // Execute
+  auto status = km_core_keyboard_load_from_blob(kmxfile.stem().c_str(), data.data(), data.size(), &this->keyboard);
+
+  // Verify
+  EXPECT_EQ(status, KM_CORE_STATUS_INVALID_KEYBOARD);
+  EXPECT_TRUE(this->keyboard == nullptr);
+}
+
+// provide our own `main` so that we can get the path of the exe so that
+// we have a well-defined location to find our test keyboards
+int main(int argc, char **argv) {
+#ifdef __EMSCRIPTEN__
+  test_dir = get_wasm_file_path(km::core::path(argv[0]).parent());
+#else
+  test_dir = km::core::path(argv[0]).parent();
+#endif
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

core/tests/unit/kmnkbd/action_set_api.tests.cpp

@@ -14,6 +14,7 @@
 
 #include <test_assert.h>
 #include "../emscripten_filesystem.h"
+#include "../load_kmx_file.hpp"
 
 const km_core_action_item alert_action_item();
 const km_core_action_item bksp_action_item(uint8_t type, uintptr_t value);
@@ -57,7 +58,9 @@ void setup(const char *keyboard, const km_core_cu* context) {
   teardown();
 
   km::core::path path = km::core::path::join(arg_path, keyboard);
-  try_status(km_core_keyboard_load(path.native().c_str(), &test_kb));
+
+  auto blob = km::tests::load_kmx_file(path.native().c_str());
+  try_status(km_core_keyboard_load_from_blob(path.stem().c_str(), blob.data(), blob.size(), &test_kb));
   try_status(km_core_state_create(test_kb, test_env_opts, &test_state));
   try_status(context_items_from_utf16(context, &citems));
   try_status(km_core_context_set(km_core_state_context(test_state), citems));

core/tests/unit/kmnkbd/actions_get_api.tests.cpp

@@ -15,6 +15,7 @@
 
 #include <test_assert.h>
 #include "../emscripten_filesystem.h"
+#include "../load_kmx_file.hpp"
 
 km_core_option_item test_env_opts[] =
 {
@@ -48,7 +49,8 @@ void setup(const km_core_cu *app_context, const km_core_cu *cached_context, int
   teardown();
 
   km::core::path path = km::core::path::join(arg_path, "..", "ldml", "keyboards", "k_001_tiny.kmx");
-  try_status(km_core_keyboard_load(path.native().c_str(), &test_kb));
+  auto blob = km::tests::load_kmx_file(path.native().c_str());
+  try_status(km_core_keyboard_load_from_blob(path.stem().c_str(), blob.data(), blob.size(), &test_kb));
   try_status(km_core_state_create(test_kb, test_env_opts, &test_state));
 
   try_status(set_context_from_string(km_core_state_context(test_state), cached_context));

core/tests/unit/kmnkbd/actions_normalize.tests.cpp

@@ -15,6 +15,7 @@
 
 #include <test_assert.h>
 #include "../emscripten_filesystem.h"
+#include "../load_kmx_file.hpp"
 
 void compare_context(km_core_context *app_context, const km_core_cu* expected_final_app_context);
 
@@ -46,7 +47,8 @@ void setup(const km_core_cu *app_context, const km_core_cu *cached_context_strin
   teardown();
 
   km::core::path path = km::core::path::join(arg_path, "..", "ldml", "keyboards", "k_001_tiny.kmx");
-  try_status(km_core_keyboard_load(path.native().c_str(), &test_kb));
+  auto blob = km::tests::load_kmx_file(path.native().c_str());
+  try_status(km_core_keyboard_load_from_blob(path.stem().c_str(), blob.data(), blob.size(), &test_kb));
   try_status(km_core_state_create(test_kb, test_env_opts, &test_state));
 
   if(cached_context_string) {

core/tests/unit/kmnkbd/debug_api.tests.cpp

@@ -22,6 +22,7 @@
 
 #include <test_assert.h>
 #include "../emscripten_filesystem.h"
+#include "../load_kmx_file.hpp"
 
 using namespace km::core::kmx;
 
@@ -54,8 +55,8 @@ void setup(const char *keyboard) {
   teardown();
 
   km::core::path path = km::core::path::join(arg_path, keyboard);
-
-  try_status(km_core_keyboard_load(path.native().c_str(), &test_kb));
+  auto blob = km::tests::load_kmx_file(path.native().c_str());
+  try_status(km_core_keyboard_load_from_blob(path.stem().c_str(), blob.data(), blob.size(), &test_kb));
   try_status(km_core_state_create(test_kb, test_env_opts, &test_state));
   try_status(context_items_from_utf16(u"Hello 😁", &citems));
 

core/tests/unit/kmnkbd/keyboard_api.tests.cpp

@@ -8,8 +8,7 @@
 
 #include "keyman_core.h"
 #include "path.hpp"
-
-//#include "keyboard.hpp"
+#include "mock/mock_processor.hpp"
 
 namespace
 {
@@ -26,14 +25,11 @@ int main(int, char *[])
   km_core_keyboard_key * kb_key_list = nullptr;
   km_core_keyboard_imx * kb_imx_list = nullptr;
 
-  try_status(km_core_keyboard_load(test_kb_path.c_str(), &test_kb));
+  test_kb = (km_core_keyboard *)new km::core::mock_processor(test_kb_path);
   try_status(km_core_keyboard_get_attrs(test_kb, &kb_attrs));
   try_status(km_core_keyboard_get_key_list(test_kb,&kb_key_list));
   try_status(km_core_keyboard_get_imx_list(test_kb,&kb_imx_list));
 
-  if (kb_attrs->folder_path != test_kb_path.parent())
-    return __LINE__;
-
   km_core_keyboard_dispose(test_kb);
   km_core_keyboard_key_list_dispose(kb_key_list);
   km_core_keyboard_imx_list_dispose(kb_imx_list);

core/tests/unit/kmnkbd/state_api.tests.cpp

@@ -14,6 +14,7 @@
 #include "path.hpp"
 #include "state.hpp"
 #include "action_items.hpp"
+#include "mock/mock_processor.hpp"
 
 #include <test_assert.h>
 
@@ -52,7 +53,6 @@ constexpr char const *doc1_expected = u8"\
     \"$schema\" : \"keyman/core/docs/introspection.schema\",\n\
     \"keyboard\" : {\n\
         \"id\" : \"dummy\",\n\
-        \"folder\" : \"\",\n\
         \"version\" : \"3.145\",\n\
         \"rules\" : []\n\
     },\n\
@@ -78,7 +78,6 @@ constexpr char const *doc2_expected = u8"\
     \"$schema\" : \"keyman/core/docs/introspection.schema\",\n\
     \"keyboard\" : {\n\
         \"id\" : \"dummy\",\n\
-        \"folder\" : \"\",\n\
         \"version\" : \"3.145\",\n\
         \"rules\" : []\n\
     },\n\
@@ -112,7 +111,7 @@ int main(int argc, char * argv[])
   km_core_keyboard * test_kb = nullptr;
   km_core_state * test_state = nullptr,
                * test_clone = nullptr;
-  try_status(km_core_keyboard_load(km::core::path("dummy.mock").c_str(), &test_kb));
+  test_kb = (km_core_keyboard *)new km::core::mock_processor(km::core::path("dummy.mock"));
 
   // Simple sanity tests.
   try_status(km_core_state_create(test_kb, test_env_opts, &test_state));

core/tests/unit/kmnkbd/state_context_api.tests.cpp

@@ -5,8 +5,10 @@
 
 #include "context.hpp"
 #include "path.hpp"
+#include "mock/mock_processor.hpp"
 
 #include "../emscripten_filesystem.h"
+#include "../load_kmx_file.hpp"
 #include <test_assert.h>
 
 //-------------------------------------------------------------------------------------
@@ -41,7 +43,14 @@ setup(const char *keyboard, const km_core_cu *context, bool setup_app_context =
   teardown();
 
   km::core::path path = km::core::path::join(arg_path, keyboard);
-  try_status(km_core_keyboard_load(path.native().c_str(), &test_kb));
+  auto blob = km::tests::load_kmx_file(path.native().c_str());
+  const auto mock_extension = ".mock";
+  if (strlen(keyboard) > strlen(mock_extension) && strcmp(keyboard + strlen(keyboard) - strlen(mock_extension), mock_extension) == 0) {
+    km::core::abstract_processor* kp = new km::core::mock_processor(keyboard);
+    test_kb = static_cast<km_core_keyboard*>(kp);
+  } else {
+    try_status(km_core_keyboard_load_from_blob(path.stem().c_str(), blob.data(), blob.size(), &test_kb));
+  }
   try_status(km_core_state_create(test_kb, test_env_opts, &test_state));
   try_status(context_items_from_utf16(context, &citems));
   try_status(km_core_context_set(km_core_state_context(test_state), citems));

core/tests/unit/kmx/kmx.cpp

@@ -29,6 +29,7 @@
 #include <test_assert.h>
 #include <test_color.h>
 #include "../emscripten_filesystem.h"
+#include "../load_kmx_file.hpp"
 
 #include "kmx_test_source.hpp"
 
@@ -191,7 +192,8 @@ run_test(const km::core::path &source, const km::core::path &compiled) {
   km_core_keyboard * test_kb = nullptr;
   km_core_state * test_state = nullptr;
 
-  try_status(km_core_keyboard_load(compiled.c_str(), &test_kb));
+  auto blob = km::tests::load_kmx_file(compiled.native().c_str());
+  try_status(km_core_keyboard_load_from_blob(compiled.stem().c_str(), blob.data(), blob.size(), &test_kb));
 
   // Setup state, environment
   try_status(km_core_state_create(test_kb, test_env_opts, &test_state));

core/tests/unit/kmx/kmx_external_event.tests.cpp

@@ -14,6 +14,7 @@
 #include <test_assert.h>
 #include <test_color.h>
 #include "../emscripten_filesystem.h"
+#include "../load_kmx_file.hpp"
 
 #include <map>
 #include <iostream>
@@ -43,7 +44,8 @@ void test_external_event(const km::core::path &source_file){
 
   km::core::path full_path = source_file;
 
-  try_status(km_core_keyboard_load(full_path.native().c_str(), &test_kb));
+  auto blob = km::tests::load_kmx_file(full_path.native().c_str());
+  try_status(km_core_keyboard_load_from_blob(full_path.stem().c_str(), blob.data(), blob.size(), &test_kb));
 
   // Setup state, environment
   try_status(km_core_state_create(test_kb, test_env_opts, &test_state));

core/tests/unit/kmx/kmx_imx.tests.cpp

@@ -14,6 +14,7 @@
 #include <test_assert.h>
 #include <test_color.h>
 #include "../emscripten_filesystem.h"
+#include "../load_kmx_file.hpp"
 #include "utfcodec.hpp"
 
 #include <map>
@@ -164,7 +165,8 @@ void test_imx_list(const km::core::path &source_file){
 
   km::core::path full_path = source_file;
 
-  try_status(km_core_keyboard_load(full_path.native().c_str(), &test_kb));
+  auto blob = km::tests::load_kmx_file(full_path.native().c_str());
+  try_status(km_core_keyboard_load_from_blob(full_path.stem().c_str(), blob.data(), blob.size(), &test_kb));
 
   // Setup state, environment
   try_status(km_core_state_create(test_kb, test_env_opts, &test_state));
@@ -211,7 +213,8 @@ void test_queue_actions (const km::core::path &source_keyboard) {
 
   km::core::path full_path = source_keyboard;
 
-  try_status(km_core_keyboard_load(full_path.native().c_str(), &test_kb));
+  auto blob = km::tests::load_kmx_file(full_path.native().c_str());
+  try_status(km_core_keyboard_load_from_blob(full_path.stem().c_str(), blob.data(), blob.size(), &test_kb));
 
   // Setup state, environment
   try_status(km_core_state_create(test_kb, test_env_opts, &test_state));

core/tests/unit/kmx/kmx_key_list.tests.cpp

@@ -15,6 +15,7 @@
 #include <test_assert.h>
 #include <test_color.h>
 #include "../emscripten_filesystem.h"
+#include "../load_kmx_file.hpp"
 
 using namespace km::core::kmx;
 
@@ -56,7 +57,8 @@ void test_key_list(const km::core::path &source_file){
 
   km::core::path full_path = source_file;
 
-  try_status(km_core_keyboard_load(full_path.native().c_str(), &test_kb));
+  auto blob = km::tests::load_kmx_file(full_path.native().c_str());
+  try_status(km_core_keyboard_load_from_blob(full_path.stem().c_str(), blob.data(), blob.size(), &test_kb));
 
   // Setup state, environment
   try_status(km_core_state_create(test_kb, test_env_opts, &test_state));

core/tests/unit/ldml/context_normalization.tests.cpp

@@ -13,6 +13,7 @@
 
 #include <test_assert.h>
 #include "../emscripten_filesystem.h"
+#include "../load_kmx_file.hpp"
 
 //-------------------------------------------------------------------------------------
 // Context normalization tests
@@ -42,7 +43,8 @@ void setup(const char *keyboard) {
   teardown();
 
   km::core::path path = km::core::path::join(arg_path, "keyboards", keyboard);
-  try_status(km_core_keyboard_load(path.native().c_str(), &test_kb));
+  auto blob = km::tests::load_kmx_file(path.native().c_str());
+  try_status(km_core_keyboard_load_from_blob(path.stem().c_str(), blob.data(), blob.size(), &test_kb));
   try_status(km_core_state_create(test_kb, test_env_opts, &test_state));
 }
 

core/tests/unit/ldml/core_ldml_min.tests.cpp

@@ -14,6 +14,7 @@
 #include <assert.h>
 
 #include "keyman_core.h"
+#include "../load_kmx_file.hpp"
 
 int main(int argc, const char *argv[]) {
 
@@ -22,7 +23,8 @@ int main(int argc, const char *argv[]) {
 
     km_core_status status;
     km_core_path_name nowhere = {0}; // this is a narrow or wide char string
-    status = km_core_keyboard_load(nowhere, &test_kb);
+    auto blob = km::tests::load_kmx_file(nowhere);
+    status = km_core_keyboard_load_from_blob(nowhere, blob.data(), blob.size(), &test_kb);
 
     std::cerr << "null km_core_keyboard_load = " << status << std::endl;
     assert(status == KM_CORE_STATUS_INVALID_ARGUMENT);

core/tests/unit/ldml/ldml.cpp

@@ -35,6 +35,7 @@
 #include "ldml/ldml_markers.hpp"
 
 #include "processor.hpp"
+#include "../load_kmx_file.hpp"
 
 namespace {
 
@@ -273,7 +274,8 @@ run_test(const km::core::path &source, const km::core::path &compiled, km::tests
   km_core_state * test_state = nullptr;
 
   const km_core_status expect_load_status = test_source.get_expected_load_status();
-  assert_equal(km_core_keyboard_load(compiled.c_str(), &test_kb), expect_load_status);
+  auto blob = km::tests::load_kmx_file(compiled.native().c_str());
+  assert_equal(km_core_keyboard_load_from_blob(compiled.stem().c_str(), blob.data(), blob.size(), &test_kb), expect_load_status);
 
   if (expect_load_status != KM_CORE_STATUS_OK) {
     std::cout << "Keyboard was expected to be invalid, so exiting " << std::endl;

core/tests/unit/ldml/ldml_test_source.cpp

@@ -40,6 +40,7 @@
 #include "unicode/uniset.h"
 #include "unicode/usetiter.h"
 
+#include "../load_kmx_file.hpp"
 #include <test_color.h>
 
 #define assert_or_return(expr) if(!(expr)) { \
@@ -56,8 +57,6 @@ namespace km {
 namespace tests {
 
 
-
-
 /** string munging */
 static void append_to_str(std::u16string &str, const char *buf) {
   const PKMX_WCHAR p = km::core::kmx::strtowstr((char *)buf); /** cast away const, unused*/
@@ -757,7 +756,8 @@ int LdmlJsonTestSourceFactory::load(const km::core::path &compiled, const km::co
   }
 
   // check and load the KMX (yes, once again)
-  if(!km::core::ldml_processor::is_kmxplus_file(compiled, rawdata)) {
+  rawdata = km::tests::load_kmx_file(compiled);
+  if (!km::core::ldml_processor::is_handled(rawdata)) {
     std::cerr << "Reading KMX for test purposes failed: " << compiled << std::endl;
     return __LINE__;
   }

core/tests/unit/ldml/meson.build

@@ -83,7 +83,10 @@ ldml = executable('ldml',
     )
 
 core_ldml_min = executable('core_ldml_min_tests',
-    ['core_ldml_min.tests.cpp'],
+    [
+      'core_ldml_min.tests.cpp',
+      meson.current_source_dir() / '../load_kmx_file.cpp',
+    ],
     cpp_args: defns + warns,
     include_directories: [inc, libsrc],
     link_args: links,

core/tests/unit/load_kmx_file.cpp

@@ -0,0 +1,34 @@
+#include <fstream>
+#include <iostream>
+#include <list>
+#include <string>
+
+#include "kmx_file.h"
+#include "path.hpp"
+#include "utfcodec.hpp"
+
+namespace km::tests {
+
+std::vector<uint8_t> load_kmx_file(km::core::path const& kb_path) {
+  std::vector<uint8_t> data;
+  std::ifstream file(static_cast<std::string>(kb_path), std::ios::binary | std::ios::ate);
+  if (file.fail()) {
+    return std::vector<uint8_t>();
+  }
+  const std::streamsize size = file.tellg();
+  if (size >= KMX_MAX_ALLOWED_FILE_SIZE) {
+    return std::vector<uint8_t>();
+  }
+
+  file.seekg(0, std::ios::beg);
+
+  data.resize((size_t)size);
+  if (!file.read((char*)data.data(), size)) {
+    return std::vector<uint8_t>();
+  }
+
+  file.close();
+  return data;
+}
+
+}

core/tests/unit/load_kmx_file.hpp

@@ -0,0 +1,14 @@
+#ifndef __LOAD_KMX_FILE_HPP__
+#define __LOAD_KMX_FILE_HPP__
+
+#include "path.hpp"
+
+namespace km {
+namespace tests {
+
+std::vector<uint8_t> load_kmx_file(km::core::path const& kb_path);
+
+}
+}
+
+#endif  // __LOAD_KMX_FILE_HPP__

core/tests/unit/meson.build

@@ -1,13 +1,41 @@
 node = find_program('node', required: true)
 
-common_test_files = [
+gtest = subproject('gtest')
+gtest_dep = gtest.get_variable('gtest_dep')
+gmock_dep = gtest.get_variable('gmock_dep')
+
+test_util_files = [
   meson.current_source_dir() / 'emscripten_filesystem.cpp',
+  meson.current_source_dir() / 'load_kmx_file.cpp',
+]
+
+common_test_files = [
+  test_util_files,
   meson.global_source_root() / '../common/include/test_color.cpp'
 ]
 
 hextobin_root = meson.global_source_root() / '../common/tools/hextobin/build/hextobin.js'
 hextobin_cmd = [node, hextobin_root]
 
+if cpp_compiler.get_id() == 'emscripten'
+  extra_link_args = [ '-lnodefs.js' ]
+else
+  extra_link_args = []
+endif
+
+kmcorekeyboardapitests = executable('km_core_keyboard_api.tests',
+  [
+    'km_core_keyboard_api.tests.cpp',
+    test_util_files,
+  ],
+  include_directories: [inc, libsrc],
+  link_args: [ links, extra_link_args ],
+  dependencies: [icu_uc, icu_i18n, gtest_dep, gmock_dep],
+  objects: lib.extract_all_objects(recursive: false),
+)
+
+test('km-core-keyboard-api-tests', kmcorekeyboardapitests)
+
 subdir('json')
 subdir('utftest')
 subdir('kmnkbd')

linux/debian/libkeymancore2.symbols

@@ -2,7 +2,7 @@ libkeymancore.so.2 libkeymancore2 #MINVER#
 * Build-Depends-Package: libkeymancore-dev
 
  (c++|optional)"typeinfo name for std::codecvt_utf8_utf16<char16_t, 1114111ul, (std::codecvt_mode)0>@Base" 17.0.244
- (c++|optional)std::piecewise_construct@Base 18.0.118
+ (c++|optional)std::piecewise_construct@Base 18.0.145
  km_core_context_clear@Base 17.0.195
  km_core_context_get@Base 17.0.195
  km_core_context_item_list_size@Base 17.0.195
@@ -19,6 +19,7 @@ libkeymancore.so.2 libkeymancore2 #MINVER#
  km_core_keyboard_imx_list_dispose@Base 17.0.195
  km_core_keyboard_key_list_dispose@Base 17.0.195
  km_core_keyboard_load@Base 17.0.195
+ km_core_keyboard_load_from_blob@Base 18.0.101
  km_core_options_list_size@Base 17.0.195
  km_core_process_event@Base 17.0.195
  km_core_process_queued_actions@Base 17.0.195