From bfb26eaf64c564f676ee8b1501f0423e1d6da60b Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 15 Jan 2024 07:19:31 +0700 Subject: [PATCH 01/45] feat(core): adds `km_core_state_context_debug` Fixes #10365. Adds a `km_core_state_context_debug` function that returns a km_core_cp string containing the current cached or intermediate context. The function must be used only for debug logging purposes -- the string is not intended to parsed for other purposes and the format may change in the future. This commit adds the function and corresponding unit tests. Subsequent commits will update existing context debug functions in various engines to use this API endpoint instead. --- core/include/keyman/keyman_core_api.h | 52 ++++++++++++++++++ core/src/km_core_state_api.cpp | 79 +++++++++++++++++++++++++++ core/tests/unit/kmnkbd/action_api.cpp | 40 ++++++++++++++ 3 files changed, 171 insertions(+) diff --git a/core/include/keyman/keyman_core_api.h b/core/include/keyman/keyman_core_api.h index dc15b8b8ac..cab2f768db 100644 --- a/core/include/keyman/keyman_core_api.h +++ b/core/include/keyman/keyman_core_api.h @@ -1140,6 +1140,58 @@ KMN_API km_core_status km_core_state_get_intermediate_context(km_core_state *state, km_core_context_item ** context_items); +/* +``` +### Context Debug Reporting +As of version 17, the cached context is an internal property of the +state, not exposed to the consumer of the API -- apart from the +Keyman Developer Keyboard Debugger. However, for other debug +purposes, it is helpful to be able to examine the cached context, so +a debug-formatted version of the context is made available with +`km_core_state_context_debug`. This is not intended to be parsed for +reading the context for other purposes, and the format may change. + +```c +*/ +enum km_core_debug_context_type { + KM_CORE_DEBUG_CONTEXT_CACHED = 0, + KM_CORE_DEBUG_CONTEXT_INTERMEDIATE = 1 +}; + +/* +``` +### `km_core_state_context_debug` +##### Description: +Returns a debug formatted string of the context from the state. +##### Return: +A pointer to a km_core_cp UTF-16 string. Must be disposed of by a call +to `km_core_cp_dispose`. +##### Parameters: +- __state__: A pointer to the opaque state object to be queried. +- __context_type__: the type of context to retrieve from the state + +```c +*/ +KMN_API +km_core_cp * +km_core_state_context_debug(km_core_state *state, km_core_debug_context_type context_type); + +/* +``` +### `km_core_cp_dispose` +##### Description: +Free the allocated memory belonging to a `km_core_cp` array previously +returned by `km_core_state_context_debug`. May be `nullptr`. +##### Parameters: +- __cp__: A pointer to the start of the `km_core_cp` array + to be disposed of. + +```c +*/ +KMN_API +void +km_core_cp_dispose(km_core_cp *cp); + /* ``` ### `km_core_state_action_items` diff --git a/core/src/km_core_state_api.cpp b/core/src/km_core_state_api.cpp index 2a25d6fcc7..de77d97acf 100644 --- a/core/src/km_core_state_api.cpp +++ b/core/src/km_core_state_api.cpp @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include #include "jsonpp.hpp" @@ -360,3 +362,80 @@ km_core_status km_core_state_context_clear( km_core_context_clear(km_core_state_context(state)); return KM_CORE_STATUS_OK; } + +void km_core_cp_dispose( + km_core_cp *cp +) { + if(cp != nullptr) { + delete [] cp; + } +} + +km_core_cp * _new_error_string(std::u16string const str) { + km_core_cp* result = new km_core_cp[str.size()+1]; + str.copy(result, str.size()); + result[str.size()] = 0; + return result; +} + +km_core_cp * km_core_state_context_debug( + km_core_state *state, + km_core_debug_context_type context_type +) { + km_core_context_item * context_items = nullptr; + + if(context_type == KM_CORE_DEBUG_CONTEXT_INTERMEDIATE) { + if(km_core_state_get_intermediate_context(state, &context_items) != KM_CORE_STATUS_OK) { + return _new_error_string(u""); + } + } else if(context_type == KM_CORE_DEBUG_CONTEXT_CACHED) { + if(km_core_context_get(km_core_state_context(state), &context_items) != KM_CORE_STATUS_OK) { + return _new_error_string(u""); + } + } else { + return _new_error_string(u""); + } + + size_t buf_size; + if(km_core_context_items_to_utf8(context_items, nullptr, &buf_size) != KM_CORE_STATUS_OK) { + km_core_context_items_dispose(context_items); + return _new_error_string(u""); + } + + std::vector context_buffer(buf_size); + if(km_core_context_items_to_utf8(context_items, &context_buffer[0], &buf_size) != KM_CORE_STATUS_OK) { + km_core_context_items_dispose(context_items); + return _new_error_string(u""); + } + + // construct the error message + + std::stringstream buffer; + + int context_item_length = 0; + for(auto cp = context_items; cp->type != KM_CORE_CT_END; cp++, context_item_length++); + + buffer << "|" << std::string(&context_buffer[0]) << "| (len: " << context_item_length << ") ["; + for(auto cp = context_items; cp->type != KM_CORE_CT_END; cp++) { + auto flags = buffer.flags(); + if(cp->type == KM_CORE_CT_CHAR) { + // A single Unicode codepoint + buffer << " U+" << std::hex << std::setfill('0') << std::setw(4) << std::hex << cp->character; + } else { + // A marker + buffer << " M(" << cp->marker << ")"; + } + buffer.flags(flags); + } + buffer << " ]"; + + km_core_context_items_dispose(context_items); + + std::u16string s = std::wstring_convert, char16_t>{}.from_bytes(buffer.str()); + + km_core_cp* result = new km_core_cp[s.size() + 1]; + s.copy(result, s.size()); + result[s.size()] = 0; + + return result; +} \ No newline at end of file diff --git a/core/tests/unit/kmnkbd/action_api.cpp b/core/tests/unit/kmnkbd/action_api.cpp index dd0e46f005..4d933c86ff 100644 --- a/core/tests/unit/kmnkbd/action_api.cpp +++ b/core/tests/unit/kmnkbd/action_api.cpp @@ -356,6 +356,45 @@ void test_context_clear() { teardown(); } +void test_context_debug_empty() { + km_core_cp const *cached_context = u""; + setup("k_000___null_keyboard.kmx", cached_context); + auto str = km_core_state_context_debug(test_state, KM_CORE_DEBUG_CONTEXT_CACHED); + // std::cout << str << std::endl; + assert(std::u16string(str) == u"|| (len: 0) [ ]"); + km_core_cp_dispose(str); +} + +void test_context_debug_various() { + km_core_cp const *cached_context = u""; + setup("k_000___null_keyboard.kmx", cached_context); + + km_core_context_item const citems[] = { + { KM_CORE_CT_MARKER, {0}, { 5 } }, + { KM_CORE_CT_CHAR, {0}, { '1' } }, + { KM_CORE_CT_MARKER, {0}, { 1 } }, + { KM_CORE_CT_CHAR, {0}, { '2' } }, + { KM_CORE_CT_MARKER, {0}, { 2 } }, + { KM_CORE_CT_CHAR, {0}, { '3' } }, + { KM_CORE_CT_MARKER, {0}, { 3 } }, + { KM_CORE_CT_MARKER, {0}, { 4 } }, + { KM_CORE_CT_CHAR, {0}, { 0x1F923 /* 🤣 */ } }, + KM_CORE_CONTEXT_ITEM_END + }; + + try_status(km_core_context_set(km_core_state_context(test_state), citems)); + + auto str = km_core_state_context_debug(test_state, KM_CORE_DEBUG_CONTEXT_CACHED); + // std::cout << str << std::endl; + assert(std::u16string(str) == u"|123🤣| (len: 9) [ M(5) U+0031 M(1) U+0032 M(2) U+0033 M(3) M(4) U+1f923 ]"); + km_core_cp_dispose(str); +} + +void test_context_debug() { + test_context_debug_empty(); + test_context_debug_various(); +} + //------------------------------------------------------------------------------------- // Launcher //------------------------------------------------------------------------------------- @@ -402,6 +441,7 @@ int main(int argc, char *argv []) { // context -- todo move to another file test_context_set_if_needed(); test_context_clear(); + test_context_debug(); } //------------------------------------------------------------------------------------- From 32ee4bc3865cfab2be2a7f33cbba446f9c376269 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 15 Jan 2024 08:00:32 +0700 Subject: [PATCH 02/45] fix(core): typedef enum --- core/include/keyman/keyman_core_api.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/include/keyman/keyman_core_api.h b/core/include/keyman/keyman_core_api.h index cab2f768db..857d29c36d 100644 --- a/core/include/keyman/keyman_core_api.h +++ b/core/include/keyman/keyman_core_api.h @@ -1153,10 +1153,10 @@ reading the context for other purposes, and the format may change. ```c */ -enum km_core_debug_context_type { +typedef enum { KM_CORE_DEBUG_CONTEXT_CACHED = 0, KM_CORE_DEBUG_CONTEXT_INTERMEDIATE = 1 -}; +} km_core_debug_context_type; /* ``` From bafe43315c35de4df20e8c7ed0d572e38bf5ba3b Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 15 Jan 2024 12:05:42 +1100 Subject: [PATCH 03/45] chore: Update core/src/km_core_state_api.cpp --- core/src/km_core_state_api.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/km_core_state_api.cpp b/core/src/km_core_state_api.cpp index de77d97acf..b7d65a2628 100644 --- a/core/src/km_core_state_api.cpp +++ b/core/src/km_core_state_api.cpp @@ -408,7 +408,7 @@ km_core_cp * km_core_state_context_debug( return _new_error_string(u""); } - // construct the error message + // construct the log message std::stringstream buffer; From 1b29ad32f92b4cbec87cf81e6b443b49839fd306 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 15 Jan 2024 08:26:43 +0700 Subject: [PATCH 04/45] chore(linux): update .symbols --- linux/debian/libkeymancore.symbols | 2 ++ 1 file changed, 2 insertions(+) diff --git a/linux/debian/libkeymancore.symbols b/linux/debian/libkeymancore.symbols index 9482ded8ae..8843b1418d 100644 --- a/linux/debian/libkeymancore.symbols +++ b/linux/debian/libkeymancore.symbols @@ -15,6 +15,7 @@ libkeymancore.so.1 libkeymancore #MINVER# km_core_context_length@Base 17.0.195 km_core_context_set@Base 17.0.195 km_core_context_shrink@Base 17.0.195 + km_core_cp_dispose@Base 17.0.241 km_core_event@Base 17.0.195 km_core_get_engine_attrs@Base 17.0.195 km_core_keyboard_dispose@Base 17.0.195 @@ -31,6 +32,7 @@ libkeymancore.so.1 libkeymancore #MINVER# km_core_state_clone@Base 17.0.195 km_core_state_context@Base 17.0.195 km_core_state_context_clear@Base 17.0.197 + km_core_state_context_debug@Base 17.0.241 km_core_state_context_set_if_needed@Base 17.0.197 km_core_state_create@Base 17.0.195 km_core_state_debug_get@Base 17.0.195 From 307f7da84651e957b6b7705f7b9daf84f4f74845 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 15 Jan 2024 07:49:54 +0700 Subject: [PATCH 05/45] refactor(linux): use `km_core_state_context_debug` Relates to #10365. Fixes #10378. --- linux/ibus-keyman/src/engine.c | 36 ++++++++++++---------------------- 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/linux/ibus-keyman/src/engine.c b/linux/ibus-keyman/src/engine.c index 3666e9c119..e12ed49813 100644 --- a/linux/ibus-keyman/src/engine.c +++ b/linux/ibus-keyman/src/engine.c @@ -205,27 +205,17 @@ ibus_keyman_engine_class_init (IBusKeymanEngineClass *klass) } static gchar * -get_context_debug(IBusEngine *engine, km_core_context *context) { +get_context_debug(IBusEngine *engine) { IBusKeymanEngine *keyman = (IBusKeymanEngine *)engine; - km_core_context *state_context = context ? context : km_core_state_context(keyman->state); - size_t buf_size8 = 512, buf_size16 = 512; - km_core_context_item *context_items; - gchar *current_context_utf8 = g_new0(gchar, buf_size8); - km_core_cp *current_context_utf16 = g_new0(km_core_cp, buf_size16); - if (km_core_context_get(state_context, &context_items) == KM_CORE_STATUS_OK) { - km_core_context_items_to_utf8(context_items, current_context_utf8, &buf_size8); - km_core_context_items_to_utf16(context_items, current_context_utf16, &buf_size16); + km_core_cp *buf = km_core_state_context_debug(keyman->state, KM_CORE_DEBUG_CONTEXT_CACHED); + gchar *result = g_utf16_to_utf8((gunichar2 *)buf, -1, NULL, NULL, NULL); + km_core_cp_dispose(buf); + if(result) { + return result; } - GString *output = g_string_new(""); - g_string_append_printf(output, "|%s| (len:%zu) [", current_context_utf8, km_core_context_length(state_context)); - for (int i = 0; i < buf_size16 - 1; i++) { - g_string_append_printf(output, "U+%04x ", current_context_utf16[i]); - } - g_string_append(output, "]"); - km_core_context_items_dispose(context_items); - g_free(current_context_utf16); - g_free(current_context_utf8); + + GString *output = g_string_new("Error converting debug context"); #if GLIB_CHECK_VERSION(2, 76, 0) return g_string_free_and_steal(output); #else @@ -264,7 +254,7 @@ set_context_if_needed(IBusEngine *engine) { guint cursor_pos, anchor_pos, context_start, context_end; g_autofree gchar *debug_context = NULL; - g_message("%s: current core context : %s", __FUNCTION__, debug_context = get_context_debug(engine, NULL)); + g_message("%s: current core context : %s", __FUNCTION__, debug_context = get_context_debug(engine)); ibus_engine_get_surrounding_text(engine, &text, &cursor_pos, &anchor_pos); @@ -934,10 +924,10 @@ ibus_keyman_engine_process_key_event( km_mod_state |= KM_CORE_MODIFIER_CAPS; } g_message("DAR: %s - km_mod_state=0x%x", __FUNCTION__, km_mod_state); - g_autofree gchar *debug_context = NULL; - g_message("%s: before process key event: %s", __FUNCTION__, debug_context = get_context_debug(engine, NULL)); + g_autofree gchar *debug_context0 = NULL, *debug_context1 = NULL, *debug_context2 = NULL; + g_message("%s: before process key event: %s", __FUNCTION__, debug_context0 = get_context_debug(engine)); km_core_process_event(keyman->state, keycode_to_vk[keycode], km_mod_state, isKeyDown, KM_CORE_EVENT_FLAG_DEFAULT); - g_message("%s: after process key event : %s", __FUNCTION__, debug_context = get_context_debug(engine, NULL)); + g_message("%s: after process key event : %s", __FUNCTION__, debug_context1 = get_context_debug(engine)); // km_core_state_action_items to get action items size_t num_action_items; @@ -953,7 +943,7 @@ ibus_keyman_engine_process_key_event( return FALSE; } - g_message("%s: after processing all actions: %s", __FUNCTION__, debug_context = get_context_debug(engine, NULL)); + g_message("%s: after processing all actions: %s", __FUNCTION__, debug_context2 = get_context_debug(engine)); return TRUE; } From 7cd7ab3346060a3bc614e5dadcfdb6099a37cd77 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 15 Jan 2024 08:20:51 +0700 Subject: [PATCH 06/45] refactor(windows): use `km_core_state_context_debug` Relates to #10365. --- windows/src/engine/keyman32/calldll.cpp | 41 ++++++----------------- windows/src/engine/keyman32/kmprocess.cpp | 39 +++++++-------------- 2 files changed, 22 insertions(+), 58 deletions(-) diff --git a/windows/src/engine/keyman32/calldll.cpp b/windows/src/engine/keyman32/calldll.cpp index d71b73145d..8bb3e494df 100644 --- a/windows/src/engine/keyman32/calldll.cpp +++ b/windows/src/engine/keyman32/calldll.cpp @@ -154,38 +154,17 @@ LogContext(km_core_state *lpCoreKeyboardState, uint8_t context_type) { if (!lpCoreKeyboardState) { return FALSE; } - km_core_context_item *citems = nullptr; - km_core_status error_status; - char *log_str_title = nullptr; - char* const int_context = "Intermediate Context"; - char* const core_context = "Core Context"; - switch(context_type){ - case CONTEXT_CORE: - error_status = (km_core_status_codes)km_core_context_get(km_core_state_context(lpCoreKeyboardState), &citems); - log_str_title = int_context; - break; - case CONTEXT_INT: - error_status = (km_core_status_codes)km_core_state_get_intermediate_context(lpCoreKeyboardState, &citems); - log_str_title = core_context; - break; - default: - error_status = KM_CORE_STATUS_INVALID_ARGUMENT; - } - if (error_status != KM_CORE_STATUS_OK){ - km_core_context_items_dispose(citems); - return FALSE; - } - DWORD context_length = (DWORD)km_core_context_item_list_size(citems); - WCHAR *buf = new WCHAR[(context_length * 3) +1 ]; // *3 if every context item was a deadkey - if (!ContextItemToAppContext(citems, buf, context_length)) { - km_core_context_items_dispose(citems); - delete[] buf; - return FALSE; - } - km_core_context_items_dispose(citems); - SendDebugMessageFormat(0, sdmKeyboard, 0, "%s: [%s]", log_str_title, Debug_UnicodeString(buf)); - delete[] buf; + km_core_cp* buffer = km_core_state_context_debug( + lpCoreKeyboardState, + context_type == CONTEXT_CORE ? KM_CORE_DEBUG_CONTEXT_CACHED : KM_CORE_DEBUG_CONTEXT_INTERMEDIATE + ); + char* const log_str_title = context_type == CONTEXT_CORE ? "Core Context" : "Intermediate Context"; + + SendDebugMessageFormat(0, sdmKeyboard, 0, "%s: %ls", log_str_title, buffer); + + km_core_cp_dispose(buffer); + return TRUE; } diff --git a/windows/src/engine/keyman32/kmprocess.cpp b/windows/src/engine/keyman32/kmprocess.cpp index b83079b03d..c95c7ce5fd 100644 --- a/windows/src/engine/keyman32/kmprocess.cpp +++ b/windows/src/engine/keyman32/kmprocess.cpp @@ -70,31 +70,6 @@ BOOL fOutputKeystroke; -char *getcontext_debug() { - - PKEYMAN64THREADDATA _td = ThreadGlobals(); - if (!_td || !_td->lpActiveKeyboard || !_td->lpActiveKeyboard->lpCoreKeyboardState){ - return ""; - } - - WCHAR buf[(MAXCONTEXT * 3) + 1]; // *3 if every context item was a deadkey - km_core_context_item *citems = nullptr; - - if (KM_CORE_STATUS_OK != km_core_context_get( - km_core_state_context(_td->lpActiveKeyboard->lpCoreKeyboardState), &citems)) { - return ""; - } - - DWORD context_length = (DWORD)km_core_context_item_list_size(citems); - if (!ContextItemToAppContext(citems, buf, context_length)) { - km_core_context_items_dispose(citems); - return ""; - } - km_core_context_items_dispose(citems); - return Debug_UnicodeString(buf); - -} - /** * Process the key stroke using the core processor * @@ -148,8 +123,18 @@ BOOL ProcessHook() if(_td->state.msg.message == wm_keymankeydown) { // I4827 if (ShouldDebug(sdmKeyboard)) { - SendDebugMessageFormat(_td->state.msg.hwnd, sdmKeyboard, 0, "Key pressed: %s Context '%s'", - Debug_VirtualKey(_td->state.vkey), getcontext_debug()); + if(!_td->lpActiveKeyboard || !_td->lpActiveKeyboard->lpCoreKeyboardState) { + SendDebugMessageFormat(_td->state.msg.hwnd, sdmKeyboard, 0, "Key pressed: %s Context ", + Debug_VirtualKey(_td->state.vkey)); + } else { + km_core_cp* debug_context = km_core_state_context_debug( + _td->lpActiveKeyboard->lpCoreKeyboardState, + KM_CORE_DEBUG_CONTEXT_CACHED + ); + SendDebugMessageFormat(_td->state.msg.hwnd, sdmKeyboard, 0, "Key pressed: %s Context '%ls'", + Debug_VirtualKey(_td->state.vkey), debug_context); + km_core_cp_dispose(debug_context); + } } } From a1fbbd1ab509d7dab9c5b65d2f86bbb1fa01d85a Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 15 Jan 2024 08:39:30 +0700 Subject: [PATCH 07/45] refactor(mac): remove unused context calls --- .../KeymanEngine4Mac/CoreWrapper/CoreWrapper.m | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m index 59c791aabf..33c6443395 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m @@ -310,7 +310,7 @@ const int CORE_ENVIRONMENT_ARRAY_LENGTH = 6; return action; } --(NSString *)getContextAsStringUsingCore { +/*-(NSString *)getContextAsStringUsingCore { km_core_context * context = km_core_state_context(self.coreState); km_core_context_item * contextItemsArray = nil; @@ -341,7 +341,7 @@ const int CORE_ENVIRONMENT_ARRAY_LENGTH = 6; [self.coreHelper logDebugMessage:@"CoreWrapper getContextAsStringUsingCore = %@", immutableString]; return immutableString; -} +}*/ -(void)clearContextUsingCore { km_core_state_context_clear(self.coreState); @@ -374,9 +374,9 @@ const int CORE_ENVIRONMENT_ARRAY_LENGTH = 6; } --(NSString*)context { +/*-(NSString*)context { return [self getContextAsStringUsingCore]; -} +}*/ //TODO: create and save as static +(BOOL)setupCoreEnvironment:(km_core_option_item *) coreOptionArray { From cd6940596075bdcffd67d644b6097e8d61bab98c Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 15 Jan 2024 09:35:35 +0700 Subject: [PATCH 08/45] feat(core): add app_context and failing tests Adds the application context member of state and a corresponding, failing unit test, which starts the process of verifying the handling of the application context. --- core/include/keyman/keyman_core_api.h | 21 +++- core/src/km_core_state_api.cpp | 12 ++ core/src/state.hpp | 4 + core/tests/unit/ldml/meson.build | 10 ++ .../unit/ldml/test_context_normalization.cpp | 116 ++++++++++++++++++ 5 files changed, 162 insertions(+), 1 deletion(-) create mode 100644 core/tests/unit/ldml/test_context_normalization.cpp diff --git a/core/include/keyman/keyman_core_api.h b/core/include/keyman/keyman_core_api.h index 857d29c36d..3e4451f612 100644 --- a/core/include/keyman/keyman_core_api.h +++ b/core/include/keyman/keyman_core_api.h @@ -1122,6 +1122,10 @@ km_core_context * km_core_state_context(km_core_state *state); +KMN_API +km_core_context * +km_core_state_app_context(km_core_state *state); + /* ``` ### `km_core_state_get_intermediate_context` @@ -1151,11 +1155,26 @@ a debug-formatted version of the context is made available with `km_core_state_context_debug`. This is not intended to be parsed for reading the context for other purposes, and the format may change. +The three context types are: +* cached: the internal context used by Core, which may be normalized + and may contain markers. This is set via + km_core_state_context_set_if_needed, and will be modified + during keystroke event processing. +* intermediate: internal context used by IMX, only valid during + keystroke event processing. +* app: an exact copy of the current context passed in to + km_core_state_context_set_if_needed, which is used to verify + the precise text manipulations required when emitted changes. + This input context is in "NFU" -- normalization form unknown, + and may be mixed normalization so may require fixups when + it is manipulated by keyboard processors that support + normalization, such as the LDML keyboard processor. ```c */ typedef enum { KM_CORE_DEBUG_CONTEXT_CACHED = 0, - KM_CORE_DEBUG_CONTEXT_INTERMEDIATE = 1 + KM_CORE_DEBUG_CONTEXT_INTERMEDIATE = 1, + KM_CORE_DEBUG_CONTEXT_APP = 2 } km_core_debug_context_type; /* diff --git a/core/src/km_core_state_api.cpp b/core/src/km_core_state_api.cpp index b7d65a2628..bf8b41ae74 100644 --- a/core/src/km_core_state_api.cpp +++ b/core/src/km_core_state_api.cpp @@ -72,6 +72,14 @@ km_core_context *km_core_state_context(km_core_state *state) return static_cast(&state->context()); } +km_core_context *km_core_state_app_context(km_core_state *state) +{ + assert(state); + if (!state) return nullptr; + + return static_cast(&state->app_context()); +} + km_core_status km_core_state_get_intermediate_context( km_core_state *state, km_core_context_item ** context_items @@ -392,6 +400,10 @@ km_core_cp * km_core_state_context_debug( if(km_core_context_get(km_core_state_context(state), &context_items) != KM_CORE_STATUS_OK) { return _new_error_string(u""); } + } else if(context_type == KM_CORE_DEBUG_CONTEXT_APP) { + if(km_core_context_get(km_core_state_app_context(state), &context_items) != KM_CORE_STATUS_OK) { + return _new_error_string(u""); + } } else { return _new_error_string(u""); } diff --git a/core/src/state.hpp b/core/src/state.hpp index b7f8baa486..0b98cc9e24 100644 --- a/core/src/state.hpp +++ b/core/src/state.hpp @@ -123,6 +123,7 @@ class state { protected: core::context _ctxt; + core::context _app_ctxt; core::abstract_processor & _processor; core::actions _actions; core::debug_items _debug_items; @@ -138,6 +139,9 @@ public: core::context & context() noexcept { return _ctxt; } core::context const & context() const noexcept { return _ctxt; } + core::context & app_context() noexcept { return _app_ctxt; } + core::context const & app_context() const noexcept { return _app_ctxt; } + core::abstract_processor const & processor() const noexcept { return _processor; } core::abstract_processor & processor() noexcept { return _processor; } diff --git a/core/tests/unit/ldml/meson.build b/core/tests/unit/ldml/meson.build index f85014bebe..d0216acef2 100644 --- a/core/tests/unit/ldml/meson.build +++ b/core/tests/unit/ldml/meson.build @@ -83,6 +83,16 @@ t = executable('test_transforms', 'test_transforms.cpp', objects: lib.extract_all_objects(recursive: false)) test('test_transforms', t, suite: 'ldml') +# run test_context_normalization ldml unit test + +t = executable('test_context_normalization', 'test_context_normalization.cpp', + cpp_args: defns + warns, + include_directories: [inc, libsrc, '../../../../developer/src/ext/json'], + link_args: links + tests_flags, + dependencies: [icu_uc, icu_i18n], + objects: lib.extract_all_objects(recursive: false)) +test('test_context_normalization', t, suite: 'ldml') + # Run tests on all keyboards (`tests` defined in keyboards/meson.build) foreach kbd : tests diff --git a/core/tests/unit/ldml/test_context_normalization.cpp b/core/tests/unit/ldml/test_context_normalization.cpp new file mode 100644 index 0000000000..061c09335d --- /dev/null +++ b/core/tests/unit/ldml/test_context_normalization.cpp @@ -0,0 +1,116 @@ +/* + Copyright: © 2018 SIL International. + Description: Tests for normalization in the context API. + Create Date: 15 Jan 2024 + Authors: Marc Durdin + History: 15 Jan 2024 - MCD - Initial implementation. +*/ +#include +#include + +#include "path.hpp" +#include "action.hpp" + +#include +#include "../emscripten_filesystem.h" + +//------------------------------------------------------------------------------------- +// Context normalization tests +//------------------------------------------------------------------------------------- + +km_core_option_item test_env_opts[] = +{ + KM_CORE_OPTIONS_END +}; + +km_core_keyboard * test_kb = nullptr; +km_core_state * test_state = nullptr; +std::string arg_path; + +void teardown() { + if(test_state) { + km_core_state_dispose(test_state); + test_state = nullptr; + } + if(test_kb) { + km_core_keyboard_dispose(test_kb); + test_kb = nullptr; + } +} + +void setup(const char *keyboard) { + teardown(); + + km::core::path path = km::core::path::join(arg_path, "keyboards", keyboard); + try_status(km_core_keyboard_load(path.native().c_str(), &test_kb)); + try_status(km_core_state_create(test_kb, test_env_opts, &test_state)); +} + +bool is_identical_context(km_core_cp const *cached_context, km_core_debug_context_type context_type) { + size_t buf_size; + km_core_context_item * citems = nullptr; + + if(context_type == KM_CORE_DEBUG_CONTEXT_APP) { + try_status(km_core_context_get(km_core_state_app_context(test_state), &citems)); + } else { + try_status(km_core_context_get(km_core_state_context(test_state), &citems)); + } + try_status(km_core_context_items_to_utf16(citems, nullptr, &buf_size)); + km_core_cp* new_cached_context = new km_core_cp[buf_size]; + try_status(km_core_context_items_to_utf16(citems, new_cached_context, &buf_size)); + + km_core_context_items_dispose(citems); + + bool result = std::u16string(cached_context) == new_cached_context; + delete[] new_cached_context; + return result; +} + +void test_context_set_if_needed_for_ldml_normalization() { + km_core_cp const *application_context = u"This is a test À"; + km_core_cp const *cached_context = u"This is a test A\u0300"; + setup("k_001_tiny.kmx"); + assert(km_core_state_context_set_if_needed(test_state, application_context) == KM_CORE_CONTEXT_STATUS_UPDATED); + assert(is_identical_context(cached_context, KM_CORE_DEBUG_CONTEXT_CACHED)); + assert(is_identical_context(application_context, KM_CORE_DEBUG_CONTEXT_APP)); + teardown(); +} + +void test_context_set_if_needed() { + test_context_set_if_needed_for_ldml_normalization(); +} + +//------------------------------------------------------------------------------------- +// Launcher +//------------------------------------------------------------------------------------- + +constexpr const auto help_str = "\ +context_normalization [--color]\n\ +\n\ + --color Force color output\n"; + +int error_args() { + std::cerr << "context_normalization: Invalid arguments." << std::endl; + std::cout << help_str; + return 1; +} + +int main(int argc, char *argv []) { + auto arg_color = argc > 1 && std::string(argv[1]) == "--color"; + console_color::enabled = console_color::isaterminal() || arg_color; + + // Get the path of the current executable + arg_path = argv[0]; + auto last = arg_path.find_last_of("/\\"); + if(last == std::string::npos) { + std::cerr << "could not parse argv[0]: " << argv[0] << std::endl; + return 1; + } + arg_path.resize(last+1); + +#ifdef __EMSCRIPTEN__ + arg_path = get_wasm_file_path(arg_path); +#endif + + test_context_set_if_needed(); +} From 036c32af1f046970be50a0f09d490777773d3b6c Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 15 Jan 2024 11:47:22 +0700 Subject: [PATCH 09/45] feat(core): normalize input context First half of #9999. Adds support for normalization (to NFD) of input app context into the cached context. The keyboard processor will work with the NFD cached context. Adds unit tests for the normalization as part of the LDML keyboard processor test suite. TODO: * Comparing modified cached context to app context to determine the transform required to send to the app * Handling illegal unicode and unpaired surrogates on input context --- core/src/km_core_state_api.cpp | 90 +------ .../km_core_state_context_set_if_needed.cpp | 222 ++++++++++++++++++ core/src/kmx/kmx_processor.hpp | 4 + core/src/ldml/ldml_processor.hpp | 5 + core/src/meson.build | 1 + core/src/mock/mock_processor.hpp | 4 + core/src/processor.hpp | 3 + .../unit/ldml/test_context_normalization.cpp | 56 ++++- 8 files changed, 291 insertions(+), 94 deletions(-) create mode 100644 core/src/km_core_state_context_set_if_needed.cpp diff --git a/core/src/km_core_state_api.cpp b/core/src/km_core_state_api.cpp index bf8b41ae74..d0e0cb44ec 100644 --- a/core/src/km_core_state_api.cpp +++ b/core/src/km_core_state_api.cpp @@ -21,6 +21,7 @@ #include "processor.hpp" #include "state.hpp" + using namespace km::core; // Forward declarations @@ -271,95 +272,6 @@ void km_core_state_imx_deregister_callback(km_core_state *state) state->imx_deregister_callback(); } -bool is_context_valid(km_core_cp const * context, km_core_cp const * cached_context) { - if (context == nullptr || cached_context == nullptr || *cached_context == '\0') { - // If the cached_context is "empty" then it needs updating - return false; - } - km_core_cp const* context_p = context; - while(*context_p) { - context_p++; - } - - km_core_cp const* cached_context_p = cached_context; - while(*cached_context_p) { - cached_context_p++; - } - - // we need to compare from the end of the cached context - for(; context_p >= context && cached_context_p >= cached_context; context_p--, cached_context_p--) { - if(*context_p != *cached_context_p) { - // The cached context doesn't match the application context, so it is - // invalid - return false; - } - } - - if(cached_context_p > cached_context) { - // if the cached context is longer than the application context, then we also - // assume that it is invalid - return false; - } - - // It's acceptable for the application context to be longer than the cached - // context, so if we match the whole cached context, we can safely return true - return true; -} - -km_core_context_status km_core_state_context_set_if_needed( - km_core_state *state, - km_core_cp const *application_context -) { - assert(state != nullptr); - assert(application_context != nullptr); - if(state == nullptr || application_context == nullptr) { - return KM_CORE_CONTEXT_STATUS_INVALID_ARGUMENT; - } - - size_t buf_size; - km_core_context_item* context_items = nullptr; - - auto context = km_core_state_context(state); - if(km_core_context_get(context, &context_items) != KM_CORE_STATUS_OK) { - return KM_CORE_CONTEXT_STATUS_ERROR; - } - - if(km_core_context_items_to_utf16(context_items, nullptr, &buf_size) != KM_CORE_STATUS_OK) { - km_core_context_items_dispose(context_items); - return KM_CORE_CONTEXT_STATUS_ERROR; - } - - std::unique_ptr cached_context(new km_core_cp[buf_size]); - - km_core_status status = km_core_context_items_to_utf16(context_items, cached_context.get(), &buf_size); - km_core_context_items_dispose(context_items); - - if(status != KM_CORE_STATUS_OK) { - return KM_CORE_CONTEXT_STATUS_ERROR; - } - - bool is_valid = is_context_valid(application_context, cached_context.get()); - - if(is_valid) { - // We keep the context as is - return KM_CORE_CONTEXT_STATUS_UNCHANGED; - } - - km_core_context_item* new_context_items = nullptr; - - // We replace the cached context with the current application context - status = km_core_context_items_from_utf16(application_context, &new_context_items); - if (status != KM_CORE_STATUS_OK) { - km_core_context_clear(context); - return KM_CORE_CONTEXT_STATUS_CLEARED; - } - - km_core_context_set(context, new_context_items); - km_core_context_items_dispose(new_context_items); - return KM_CORE_CONTEXT_STATUS_UPDATED; -} - - km_core_status km_core_state_context_clear( km_core_state *state ) { diff --git a/core/src/km_core_state_context_set_if_needed.cpp b/core/src/km_core_state_context_set_if_needed.cpp new file mode 100644 index 0000000000..84775772d3 --- /dev/null +++ b/core/src/km_core_state_context_set_if_needed.cpp @@ -0,0 +1,222 @@ +/* + Copyright: © 2018-2024 SIL International. + Description: Implementation of the state API functions using internal + data structures and functions. + Create Date: 15 Jan 2024 + Authors: Marc Durdin + History: 15 Jan 2024 - MCD - Refactor our km_core_state_context_set_if_needed + and implement normalization +*/ +#include + +#include + +#include "processor.hpp" +#include "state.hpp" +#include "debuglog.h" + +#if !defined(HAVE_ICU4C) +#error icu4c is required for this code +#endif + +#define U_FALLTHROUGH +#include "unicode/utypes.h" +#include "unicode/unistr.h" +#include "unicode/normalizer2.h" + +using namespace km::core; + +// Forward declarations + +bool should_normalize(km_core_state *state); +bool is_context_valid(km_core_cp const * context, km_core_cp const * cached_context); +km_core_cp* get_context_as_string(km_core_context *context); +bool set_context_from_string(km_core_context *context, km_core_cp const *new_context); +bool do_normalize_nfd(km_core_cp const * src, std::u16string &dst); +km_core_context_status do_fail(km_core_context *app_context, km_core_context *cached_context, const char* error); + +// --------------------------------------------------------------------------- + +km_core_context_status km_core_state_context_set_if_needed( + km_core_state *state, + km_core_cp const *new_app_context +) { + assert(state != nullptr); + assert(new_app_context != nullptr); + if(state == nullptr || new_app_context == nullptr) { + return KM_CORE_CONTEXT_STATUS_INVALID_ARGUMENT; + } + + auto app_context = km_core_state_app_context(state); + auto cached_context = km_core_state_context(state); + + // Retrieve the existing internally cached app context for comparison + + std::unique_ptr app_context_string(get_context_as_string(app_context)); + + // Compare the internal app context with the passed-in application context + + bool is_valid = is_context_valid(new_app_context, app_context_string.get()); + + if(is_valid) { + // We keep the context as is + return KM_CORE_CONTEXT_STATUS_UNCHANGED; + } + + // We replace the internal app context with the passed-in application context + + if(!set_context_from_string(app_context, new_app_context)) { + return do_fail(app_context, cached_context, "could not set new app context"); + } + + // Finally, we normalize and replace the cached context + + std::u16string normalized_buffer; + km_core_cp const *new_cached_context = nullptr; + + if(should_normalize(state)) { + if(!do_normalize_nfd(new_app_context, normalized_buffer)) { + return do_fail(app_context, cached_context, "could not normalize string"); + } + new_cached_context = normalized_buffer.c_str(); + } else { + new_cached_context = new_app_context; + } + + // TODO: #10100 will alter how we replace the cached context here -- maintaining + // markers as far as possible + + if(!set_context_from_string(cached_context, new_cached_context)) { + return do_fail(app_context, cached_context, "could not set new cached context"); + } + + return KM_CORE_CONTEXT_STATUS_UPDATED; +} + +/** + * Returns true if the current keyboard processor wants a normalized cached context + */ +bool should_normalize(km_core_state *state) { + return state->processor().supports_normalization(); +} + +/** + * Returns true if the internal app context does not need to be updated to the new + * app context + * + * TODO: #10100 will alter some of the assumptions here + */ +bool is_context_valid(km_core_cp const * new_app_context, km_core_cp const * app_context) { + if (new_app_context == nullptr || app_context == nullptr || *app_context == '\0') { + // If the app_context is "empty" then it needs updating + return false; + } + km_core_cp const* new_app_context_p = new_app_context; + while(*new_app_context_p) { + new_app_context_p++; + } + + km_core_cp const* app_context_p = app_context; + while(*app_context_p) { + app_context_p++; + } + + // we need to compare from the end of the cached context + for(; new_app_context_p >= new_app_context && app_context_p >= app_context; new_app_context_p--, app_context_p--) { + if(*new_app_context_p != *app_context_p) { + // The cached context doesn't match the application context, so it is + // invalid + return false; + } + } + + if(app_context_p > app_context) { + // if the cached context is longer than the application context, then we also + // assume that it is invalid + return false; + } + + // It's acceptable for the application context to be longer than the cached + // context, so if we match the whole cached context, we can safely return true + return true; +} + +/** + * Retrieves the context as a km_core_cp string, dropping markers + */ +km_core_cp* get_context_as_string(km_core_context *context) { + size_t buf_size = 0; + km_core_context_item* context_items = nullptr; + + if(km_core_context_get(context, &context_items) != KM_CORE_STATUS_OK) { + return nullptr; + } + + if(km_core_context_items_to_utf16(context_items, nullptr, &buf_size) != KM_CORE_STATUS_OK) { + km_core_context_items_dispose(context_items); + return nullptr; + } + + km_core_cp *app_context_string = new km_core_cp[buf_size]; + + km_core_status status = km_core_context_items_to_utf16(context_items, app_context_string, &buf_size); + km_core_context_items_dispose(context_items); + + if(status != KM_CORE_STATUS_OK) { + return nullptr; + } + + return app_context_string; +} + +/** + * Updates the context from the new_context km_core_cp string + */ +bool set_context_from_string(km_core_context *context, km_core_cp const *new_context) { + km_core_context_item* new_context_items = nullptr; + + km_core_status status = km_core_context_items_from_utf16(new_context, &new_context_items); + if (status != KM_CORE_STATUS_OK) { + return false; + } + + km_core_context_set(context, new_context_items); + km_core_context_items_dispose(new_context_items); + + return true; +} + +/** + * Normalize the input string using ICU + */ +bool do_normalize_nfd(km_core_cp const * src, std::u16string &dst) { + UErrorCode icu_status = U_ZERO_ERROR; + const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(icu_status); + assert(U_SUCCESS(icu_status)); + if(!U_SUCCESS(icu_status)) { + // TODO: log the failure code + return false; + } + icu::UnicodeString udst; + icu::UnicodeString usrc = icu::UnicodeString(src); + nfd->normalize(usrc, udst, icu_status); + assert(U_SUCCESS(icu_status)); + if(!U_SUCCESS(icu_status)) { + // TODO: log the failure code + return false; + } + + dst.assign(udst.getBuffer(), udst.length()); + return true; +} + +/** + * Clear the context when we have a failure so we don't end up with inconsistent + * context buffers, and log the error to our diagnostic log. + */ +km_core_context_status do_fail(km_core_context *app_context, km_core_context *cached_context, const char* error) { + DebugLog("%s", error); + km_core_context_clear(app_context); + km_core_context_clear(cached_context); + return KM_CORE_CONTEXT_STATUS_CLEARED; +} \ No newline at end of file diff --git a/core/src/kmx/kmx_processor.hpp b/core/src/kmx/kmx_processor.hpp index b83e8bba9e..490bda53c6 100644 --- a/core/src/kmx/kmx_processor.hpp +++ b/core/src/kmx/kmx_processor.hpp @@ -80,6 +80,10 @@ namespace core km_core_keyboard_imx * get_imx_list() const override; + bool + supports_normalization() const override { + return false; + } }; } // namespace core diff --git a/core/src/ldml/ldml_processor.hpp b/core/src/ldml/ldml_processor.hpp index 7ad997a944..a66d151fde 100644 --- a/core/src/ldml/ldml_processor.hpp +++ b/core/src/ldml/ldml_processor.hpp @@ -84,6 +84,11 @@ namespace core { km_core_keyboard_imx * get_imx_list() const override; + bool + supports_normalization() const override { + return true; + } + private: /** emit text to context and actions */ static void emit_text(km_core_state *state, const std::u16string &str); diff --git a/core/src/meson.build b/core/src/meson.build index e5314294ad..a80eaaa94a 100644 --- a/core/src/meson.build +++ b/core/src/meson.build @@ -52,6 +52,7 @@ kmx_files = files( 'km_core_keyboard_api.cpp', 'km_core_options_api.cpp', 'km_core_state_api.cpp', + 'km_core_state_context_set_if_needed.cpp', 'km_core_debug_api.cpp', 'km_core_processevent_api.cpp', 'jsonpp.cpp', diff --git a/core/src/mock/mock_processor.hpp b/core/src/mock/mock_processor.hpp index b10f4ad8d2..a3ec3c1caa 100644 --- a/core/src/mock/mock_processor.hpp +++ b/core/src/mock/mock_processor.hpp @@ -65,6 +65,10 @@ namespace core km_core_keyboard_imx * get_imx_list() const override; + bool + supports_normalization() const override { + return false; + } }; class null_processor : public mock_processor { diff --git a/core/src/processor.hpp b/core/src/processor.hpp index 2dd52e82d4..7a6ed5aa33 100644 --- a/core/src/processor.hpp +++ b/core/src/processor.hpp @@ -122,6 +122,9 @@ namespace core virtual km_core_keyboard_imx * get_imx_list() const = 0; + virtual bool + supports_normalization() const = 0; + friend json & operator << (json &j, abstract_processor const &opts); }; diff --git a/core/tests/unit/ldml/test_context_normalization.cpp b/core/tests/unit/ldml/test_context_normalization.cpp index 061c09335d..c3215ce4a3 100644 --- a/core/tests/unit/ldml/test_context_normalization.cpp +++ b/core/tests/unit/ldml/test_context_normalization.cpp @@ -46,10 +46,22 @@ void setup(const char *keyboard) { try_status(km_core_state_create(test_kb, test_env_opts, &test_state)); } +void debug_context(km_core_debug_context_type context_type) { + auto context = km_core_state_context_debug(test_state, context_type); + if(context_type == KM_CORE_DEBUG_CONTEXT_APP) { + std::cout << "app context: " << context << std::endl; + } else { + std::cout << "cached context: " << context << std::endl; + } + km_core_cp_dispose(context); +} + bool is_identical_context(km_core_cp const *cached_context, km_core_debug_context_type context_type) { size_t buf_size; km_core_context_item * citems = nullptr; + debug_context(context_type); + if(context_type == KM_CORE_DEBUG_CONTEXT_APP) { try_status(km_core_context_get(km_core_state_app_context(test_state), &citems)); } else { @@ -66,18 +78,52 @@ bool is_identical_context(km_core_cp const *cached_context, km_core_debug_contex return result; } -void test_context_set_if_needed_for_ldml_normalization() { +void test_context_normalization_already_nfd() { + km_core_cp const *app_context_nfd = u"A\u0300"; + setup("k_001_tiny.kmx"); + assert(km_core_state_context_set_if_needed(test_state, app_context_nfd) == KM_CORE_CONTEXT_STATUS_UPDATED); + assert(is_identical_context(app_context_nfd, KM_CORE_DEBUG_CONTEXT_APP)); + assert(is_identical_context(app_context_nfd, KM_CORE_DEBUG_CONTEXT_CACHED)); + teardown(); +} + +void test_context_normalization_basic() { km_core_cp const *application_context = u"This is a test À"; km_core_cp const *cached_context = u"This is a test A\u0300"; setup("k_001_tiny.kmx"); assert(km_core_state_context_set_if_needed(test_state, application_context) == KM_CORE_CONTEXT_STATUS_UPDATED); - assert(is_identical_context(cached_context, KM_CORE_DEBUG_CONTEXT_CACHED)); assert(is_identical_context(application_context, KM_CORE_DEBUG_CONTEXT_APP)); + assert(is_identical_context(cached_context, KM_CORE_DEBUG_CONTEXT_CACHED)); teardown(); } -void test_context_set_if_needed() { - test_context_set_if_needed_for_ldml_normalization(); +void test_context_normalization_hefty() { + // Latin Latin "ṩ" "Å" Tirhuta U+114bc -> U+114B9 U+114B0 + km_core_cp const *application_context = u"À" u"é̖" u"\u1e69" u"\u212b" u"\U000114BC"; + km_core_cp const *cached_context = u"A\u0300" u"e\u0316\u0301" u"\u0073\u0323\u0307" u"\u0041\u030a" u"\U000114B9\U000114B0"; + setup("k_001_tiny.kmx"); + assert(km_core_state_context_set_if_needed(test_state, application_context) == KM_CORE_CONTEXT_STATUS_UPDATED); + assert(is_identical_context(application_context, KM_CORE_DEBUG_CONTEXT_APP)); + assert(is_identical_context(cached_context, KM_CORE_DEBUG_CONTEXT_CACHED)); + teardown(); +} + +void test_context_normalization_invalid_unicode() { + // unpaired surrogate illegal + km_core_cp const application_context[] = { 0xDC01, 0x0020, 0x0020, 0xFFFF, 0x0000 }; + km_core_cp const cached_context[] = { 0xDC01, 0x0020, 0x0020, 0xFFFF, 0x0000 }; + setup("k_001_tiny.kmx"); + assert(km_core_state_context_set_if_needed(test_state, application_context) == KM_CORE_CONTEXT_STATUS_UPDATED); + assert(is_identical_context(application_context, KM_CORE_DEBUG_CONTEXT_APP)); + assert(is_identical_context(cached_context, KM_CORE_DEBUG_CONTEXT_CACHED)); + teardown(); +} + +void test_context_normalization() { + test_context_normalization_already_nfd(); + test_context_normalization_basic(); + test_context_normalization_hefty(); + // TODO: we need to strip illegal chars: test_context_normalization_invalid_unicode(); // -- unpaired surrogate, illegals } //------------------------------------------------------------------------------------- @@ -112,5 +158,5 @@ int main(int argc, char *argv []) { arg_path = get_wasm_file_path(arg_path); #endif - test_context_set_if_needed(); + test_context_normalization(); } From bd46fb271b1d790cc2c90d2bc698f8b31184882d Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 15 Jan 2024 12:34:00 +0700 Subject: [PATCH 10/45] feat(core): unit tests need to set both app context and cached context --- core/src/km_core_state_api.cpp | 1 + core/src/km_core_state_context_set_if_needed.cpp | 4 +--- core/tests/unit/kmnkbd/action_api.cpp | 3 +++ core/tests/unit/kmnkbd/action_set_api.cpp | 1 + core/tests/unit/kmnkbd/debug_api.cpp | 2 ++ 5 files changed, 8 insertions(+), 3 deletions(-) diff --git a/core/src/km_core_state_api.cpp b/core/src/km_core_state_api.cpp index d0e0cb44ec..4374572883 100644 --- a/core/src/km_core_state_api.cpp +++ b/core/src/km_core_state_api.cpp @@ -280,6 +280,7 @@ km_core_status km_core_state_context_clear( return KM_CORE_STATUS_INVALID_ARGUMENT; } km_core_context_clear(km_core_state_context(state)); + km_core_context_clear(km_core_state_app_context(state)); return KM_CORE_STATUS_OK; } diff --git a/core/src/km_core_state_context_set_if_needed.cpp b/core/src/km_core_state_context_set_if_needed.cpp index 84775772d3..b79044455e 100644 --- a/core/src/km_core_state_context_set_if_needed.cpp +++ b/core/src/km_core_state_context_set_if_needed.cpp @@ -56,9 +56,7 @@ km_core_context_status km_core_state_context_set_if_needed( // Compare the internal app context with the passed-in application context - bool is_valid = is_context_valid(new_app_context, app_context_string.get()); - - if(is_valid) { + if(is_context_valid(new_app_context, app_context_string.get())) { // We keep the context as is return KM_CORE_CONTEXT_STATUS_UNCHANGED; } diff --git a/core/tests/unit/kmnkbd/action_api.cpp b/core/tests/unit/kmnkbd/action_api.cpp index 4d933c86ff..9b75256548 100644 --- a/core/tests/unit/kmnkbd/action_api.cpp +++ b/core/tests/unit/kmnkbd/action_api.cpp @@ -226,6 +226,7 @@ void setup(const char *keyboard, const km_core_cp* context) { try_status(km_core_state_create(test_kb, test_env_opts, &test_state)); try_status(km_core_context_items_from_utf16(context, &citems)); try_status(km_core_context_set(km_core_state_context(test_state), citems)); + try_status(km_core_context_set(km_core_state_app_context(test_state), citems)); } bool is_identical_context(km_core_cp const *cached_context) { @@ -319,6 +320,7 @@ void test_context_set_if_needed_cached_context_has_markers() { }; try_status(km_core_context_set(km_core_state_context(test_state), citems)); + try_status(km_core_context_set(km_core_state_app_context(test_state), citems)); assert(km_core_state_context_set_if_needed(test_state, application_context) == KM_CORE_CONTEXT_STATUS_UNCHANGED); km_core_context_item* citems_new; @@ -383,6 +385,7 @@ void test_context_debug_various() { }; try_status(km_core_context_set(km_core_state_context(test_state), citems)); + try_status(km_core_context_set(km_core_state_app_context(test_state), citems)); auto str = km_core_state_context_debug(test_state, KM_CORE_DEBUG_CONTEXT_CACHED); // std::cout << str << std::endl; diff --git a/core/tests/unit/kmnkbd/action_set_api.cpp b/core/tests/unit/kmnkbd/action_set_api.cpp index 378f3bcab1..d3c06cc9c7 100644 --- a/core/tests/unit/kmnkbd/action_set_api.cpp +++ b/core/tests/unit/kmnkbd/action_set_api.cpp @@ -60,6 +60,7 @@ void setup(const char *keyboard, const km_core_cp* context) { try_status(km_core_state_create(test_kb, test_env_opts, &test_state)); try_status(km_core_context_items_from_utf16(context, &citems)); try_status(km_core_context_set(km_core_state_context(test_state), citems)); + try_status(km_core_context_set(km_core_state_app_context(test_state), citems)); } void run_test(km_core_action_item const * action_items, const km_core_actions &actions) { diff --git a/core/tests/unit/kmnkbd/debug_api.cpp b/core/tests/unit/kmnkbd/debug_api.cpp index de0cfce8e8..502f95cc47 100644 --- a/core/tests/unit/kmnkbd/debug_api.cpp +++ b/core/tests/unit/kmnkbd/debug_api.cpp @@ -66,6 +66,7 @@ void setup(const char *keyboard) { })); try_status(km_core_context_set(km_core_state_context(test_state), citems)); + try_status(km_core_context_set(km_core_state_app_context(test_state), citems)); } /** @@ -423,6 +424,7 @@ void test_backspace_markers() { {KM_CORE_CT_END} }; try_status(km_core_context_set(km_core_state_context(test_state), marker_context)); + try_status(km_core_context_set(km_core_state_app_context(test_state), marker_context)); // TODO: this needs to remove markers DEBUG_GROUP gp = {u"Main"}; From 932ad9d1df175b9c7c5c5c11dd578515afc15490 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 15 Jan 2024 12:50:41 +0700 Subject: [PATCH 11/45] chore(core): add unit test deps for wasm --- core/tests/unit/ldml/meson.build | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/tests/unit/ldml/meson.build b/core/tests/unit/ldml/meson.build index d0216acef2..6f922f1fe9 100644 --- a/core/tests/unit/ldml/meson.build +++ b/core/tests/unit/ldml/meson.build @@ -85,7 +85,8 @@ test('test_transforms', t, suite: 'ldml') # run test_context_normalization ldml unit test -t = executable('test_context_normalization', 'test_context_normalization.cpp', +t = executable('test_context_normalization', + ['test_context_normalization.cpp', '../emscripten_filesystem.cpp'], cpp_args: defns + warns, include_directories: [inc, libsrc, '../../../../developer/src/ext/json'], link_args: links + tests_flags, From aea95c4f9d891fcbe6775efdcffb5c64240044f4 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 15 Jan 2024 13:12:15 +0700 Subject: [PATCH 12/45] chore(mac): use only debug context functions --- .../CoreWrapper/CoreWrapper.h | 6 +-- .../CoreWrapper/CoreWrapper.m | 43 +++---------------- .../KeymanEngine4Mac/KME/KMEngine.h | 2 +- .../KeymanEngine4Mac/KME/KMEngine.m | 8 ++-- .../KeymanEngine4MacTests/KMEngineTests.m | 40 ++++++++++------- 5 files changed, 39 insertions(+), 60 deletions(-) diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.h b/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.h index a3111d425e..8cc1c68c9d 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.h +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.h @@ -1,9 +1,9 @@ /** * Keyman is copyright (C) SIL International. MIT License. - * + * * CoreWrapper.h * Keyman - * + * * Created by Shawn Schantz on 2022-12-12. */ @@ -29,7 +29,7 @@ NS_ASSUME_NONNULL_BEGIN withKeyDown:(BOOL)isKeyDown; -(void)setContextIfNeeded:(NSString*)context; -(void)setContext:(NSString*)context; --(NSString*)context; +-(NSString*)contextDebug; -(void)clearCoreContext; -(void)dealloc; diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m index 33c6443395..d736d80279 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m @@ -310,39 +310,6 @@ const int CORE_ENVIRONMENT_ARRAY_LENGTH = 6; return action; } -/*-(NSString *)getContextAsStringUsingCore { - km_core_context * context = km_core_state_context(self.coreState); - - km_core_context_item * contextItemsArray = nil; - size_t contextLength = km_core_context_length(context); - - NSMutableString *contextString = [[NSMutableString alloc]init]; - - if (contextLength==0) { - [self.coreHelper logDebugMessage:@"CoreWrapper getContextAsStringUsingCore, context is empty."]; - } else { - km_core_status result = km_core_context_get(context, &contextItemsArray); - if (result==KM_CORE_STATUS_OK) { - for (int i = 0; i < contextLength; i++) { - km_core_context_item contextItem = contextItemsArray[i]; - if (contextItem.type == KM_CORE_CT_CHAR) { - NSString *unicodeString = [self.coreHelper utf32ValueToString:contextItem.character]; - [contextString appendString:unicodeString]; - } - } - } - } - NSString *immutableString = [NSString stringWithString:contextString]; - - // dispose of context items array - if (contextItemsArray) { - km_core_context_items_dispose(contextItemsArray); - } - - [self.coreHelper logDebugMessage:@"CoreWrapper getContextAsStringUsingCore = %@", immutableString]; - return immutableString; -}*/ - -(void)clearContextUsingCore { km_core_state_context_clear(self.coreState); [self.coreHelper logDebugMessage:@"km_core_state_context_clear called"]; @@ -373,10 +340,14 @@ const int CORE_ENVIRONMENT_ARRAY_LENGTH = 6; } } +-(NSString*)contextDebug { + km_core_cp * context = km_core_state_context_debug(self.coreState, KM_CORE_DEBUG_CONTEXT_CACHED); + NSString *debugString = [self.coreHelper createNSStringFromUnicharString:context]; + km_core_cp_dispose(context); -/*-(NSString*)context { - return [self getContextAsStringUsingCore]; -}*/ + [self.coreHelper logDebugMessage:@"CoreWrapper contextDebug = %@", debugString]; + return debugString; +} //TODO: create and save as static +(BOOL)setupCoreEnvironment:(km_core_option_item *) coreOptionArray { diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.h b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.h index 4ec0c595cb..711c34b9ba 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.h +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.h @@ -21,7 +21,7 @@ @property (assign, nonatomic) BOOL debugMode; - (id)initWithKMX:(KMXFile *)kmx context:(NSString *)ctxBuf verboseLogging:(BOOL)enableDebugLogging; -- (NSString *)getCoreContext; +- (NSString *)getCoreContextDebug; - (void)clearCoreContext; - (void)setCoreContextIfNeeded:(NSString *)context; - (void)setCoreContext:(NSString *)context; diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m index 16c1b0c704..7afd3d40b5 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m @@ -36,7 +36,7 @@ const NSString* kEasterEggKmxName = @"EnglishSpanish.kmx"; self.debugMode = enableDebugLogging; _kmx = kmx; _coreHelper = [[CoreHelper alloc] initWithDebugMode:enableDebugLogging]; - + if (kmx) { [self loadCoreWrapperFromKmxFile:self.kmx.filePath]; [self.coreWrapper setContext:contextString]; @@ -71,7 +71,7 @@ const NSString* kEasterEggKmxName = @"EnglishSpanish.kmx"; if (self.coreHelper) { self.coreHelper.debugMode = useVerboseLogging; } - + if (useVerboseLogging) { NSLog(@"KMEngine - Turning verbose logging on"); // In Keyman Engine if "debugMode" is turned on (explicitly) with "English plus Spanish" as the current keyboard and you type "Sentrycrash#KME", @@ -89,8 +89,8 @@ const NSString* kEasterEggKmxName = @"EnglishSpanish.kmx"; NSLog(@"KMEngine - Turning verbose logging off"); } -- (NSString *)getCoreContext { - return self.coreWrapper.context; +- (NSString *)getCoreContextDebug { + return self.coreWrapper.contextDebug; } - (void)clearCoreContext { diff --git a/mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMEngineTests.m b/mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMEngineTests.m index 9ea38e0a10..dbce782bef 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMEngineTests.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMEngineTests.m @@ -41,21 +41,27 @@ NSString * names[nCombinations]; KMXFile *kmxFile = [KeymanEngineTestsStaticHelperMethods getKmxFileTestMacEngine]; KMEngine *engine = [[KMEngine alloc] initWithKMX:kmxFile context:@"" verboseLogging:YES]; XCTAssert(engine != nil, @"Expected non-nil engine"); - XCTAssert(engine.getCoreContext.length == 0, @"Expected empty context buffer"); + // Note: relying on km_core_state_context_debug output format is just barely + // acceptable for a unit test + XCTAssert(engine.getCoreContextDebug isEqualToString:@"|| (len: 0) [ ]", @"Expected empty context buffer"); } - (void)testinitWithKMX_ValidKmxNonEmptyContext_InitializedWithContext { KMXFile *kmxFile = [KeymanEngineTestsStaticHelperMethods getKmxFileTestMacEngine]; KMEngine *engine = [[KMEngine alloc] initWithKMX:kmxFile context:@"abc" verboseLogging:YES]; XCTAssert(engine != nil, @"Expected non-nil engine"); - XCTAssert([engine.getCoreContext isEqualToString:@"abc"], @"Expected 'abc' in context buffer"); + // Note: relying on km_core_state_context_debug output format is just barely + // acceptable for a unit test + XCTAssert([engine.getCoreContextDebug isEqualToString:@"|abc| (len: 3) [ U+0061 U+0062 U+0063 ]"], @"Expected 'abc' in context buffer"); } - (void)testsetCoreContextIfNeeded_NonEmptyContext_InitialContextUpdated { KMXFile *kmxFile = [KeymanEngineTestsStaticHelperMethods getKmxFileTestMacEngine]; KMEngine *engine = [[KMEngine alloc] initWithKMX:kmxFile context:@"a" verboseLogging:YES]; [engine setCoreContextIfNeeded:@"xyz"]; - XCTAssert([engine.getCoreContext isEqualToString:@"xyz"], @"Expected 'xyz' in context buffer"); + // Note: relying on km_core_state_context_debug output format is just barely + // acceptable for a unit test + XCTAssert([engine.getCoreContextDebug isEqualToString:@"|xyz| (len: 3) [ U+0078 U+0079 U+007a ]"], @"Expected 'xyz' in context buffer"); } // TODO: re-enable this one after the core API km_core_state_context_set_if_needed is fixed @@ -64,7 +70,9 @@ NSString * names[nCombinations]; KMXFile *kmxFile = [KeymanEngineTestsStaticHelperMethods getKmxFileTestMacEngine]; KMEngine *engine = [[KMEngine alloc] initWithKMX:kmxFile context:@"" verboseLogging:YES]; [engine setCoreContextIfNeeded:@"xyz"]; - XCTAssert([engine.getCoreContext isEqualToString:@"xyz"], @"Expected 'xyz' in context buffer"); + // Note: relying on km_core_state_context_debug output format is just barely + // acceptable for a unit test + XCTAssert([engine.getCoreContextDebug isEqualToString:@"|xyz| (len: 3) [ U+0078 U+0079 U+007a ]"], @"Expected 'xyz' in context buffer"); } */ @@ -278,7 +286,7 @@ NSString * names[nCombinations]; NSString * characters = charactersIgnoringModifiers; if (modifiers[i] & (LEFT_ALT_FLAG | RIGHT_ALT_FLAG)) characters = [characters stringByAppendingString:@"\u030A"]; - + NSLog(@"Test case: %lu", (NSUInteger)modifiers[i]); // NOTE: 'a' happens to be keyCode 0 (see initVirtualKeyMapping in CoreHelper) NSEvent *event = [NSEvent keyEventWithType:NSEventTypeKeyDown location:NSMakePoint(0, 0) modifierFlags:modifiers[i] timestamp:0 windowNumber:0 context:nil characters:characters charactersIgnoringModifiers:charactersIgnoringModifiers isARepeat:NO keyCode:0]; @@ -335,10 +343,10 @@ NSString * names[nCombinations]; - (void)testprocessEvent_eventForSWithModifiers_ReturnsCharacterActionWithExpectedCharacterBasedOnKmx { int i = 0; [KMEngineTests fillInNamesAndModifiersForAllChiralCombinations]; - + KMXFile *kmxFile = [KeymanEngineTestsStaticHelperMethods getKmxFileTestMacEngine]; KMEngine *engine = [[KMEngine alloc] initWithKMX:kmxFile context:@"" verboseLogging:YES]; - + for (i = 0; i < nCombinations; i++) { [engine clearCoreContext]; NSString *charactersIgnoringModifiers = (modifiers[i] & (LEFT_SHIFT_FLAG | RIGHT_SHIFT_FLAG)) ? @"S" : @"s"; @@ -349,7 +357,7 @@ NSString * names[nCombinations]; else characters = @"ß"; } - + NSLog(@"Test case: %lu", (NSUInteger)modifiers[i]); // NOTE: 's' happens to be keyCode 1 (see initVirtualKeyMapping in CoreHelper) NSEvent *event = [NSEvent keyEventWithType:NSEventTypeKeyDown location:NSMakePoint(0, 0) modifierFlags:modifiers[i] timestamp:0 windowNumber:0 context:nil characters:characters charactersIgnoringModifiers:charactersIgnoringModifiers isARepeat:NO keyCode:1]; @@ -554,8 +562,8 @@ NSString * names[nCombinations]; NSLog(@"output: %@", output); XCTAssert(output.codePointsToDeleteBeforeInsert == 1, @"Expected output to delete one code point"); XCTAssert(!output.hasTextToInsert, @"expected to insert nothing"); - NSString *context = engine.getCoreContext; - XCTAssert([context isEqualToString:@""], @"Context should be empty."); + NSString *context = engine.getCoreContextDebug; + XCTAssert([context isEqualToString:@"|| (len: 0) [ ]"], @"Context should be empty."); } - (void)testCoreProcessEvent_eventReturnWithElNuerKmx_EmitWithContextEmpty { @@ -564,8 +572,8 @@ NSString * names[nCombinations]; NSEvent *event = [NSEvent keyEventWithType:NSEventTypeKeyDown location:NSMakePoint(0, 0) modifierFlags:0 timestamp:0 windowNumber:0 context:nil characters:@"\n" charactersIgnoringModifiers:@"\n" isARepeat:NO keyCode:kVK_Return]; CoreKeyOutput *output = [engine processEvent:event]; XCTAssert(output.emitKeystroke, @"Expected emitKeystroke==YES"); - NSString *context = engine.getCoreContext; - XCTAssert([context isEqualToString:@""], @"Context should be cleared."); + NSString *context = engine.getCoreContextDebug; + XCTAssert([context isEqualToString:@"|| (len: 0) [ ]"], @"Context should be cleared."); } - (void)testCoreProcessEvent_eventTabWithElNuerKmx_EmitWithContextEmpty { @@ -574,8 +582,8 @@ NSString * names[nCombinations]; NSEvent *event = [NSEvent keyEventWithType:NSEventTypeKeyDown location:NSMakePoint(0, 0) modifierFlags:0 timestamp:0 windowNumber:0 context:nil characters:@"\t" charactersIgnoringModifiers:@"\t" isARepeat:NO keyCode:kVK_Tab]; CoreKeyOutput *output = [engine processEvent:event]; XCTAssert(output.emitKeystroke, @"Expected emitKeystroke==YES"); - NSString *context = engine.getCoreContext; - XCTAssert([context isEqualToString:@""], @"Context should be cleared."); + NSString *context = engine.getCoreContextDebug; + XCTAssert([context isEqualToString:@"|| (len: 0) [ ]"], @"Context should be cleared."); } - (void)testCoreProcessEvent_eventSingleQuoteWithElNuerKmx_ReturnsDiacritic { @@ -585,8 +593,8 @@ NSString * names[nCombinations]; NSEvent *event = [NSEvent keyEventWithType:NSEventTypeKeyDown location:NSMakePoint(0, 0) modifierFlags:0 timestamp:0 windowNumber:0 context:nil characters:@"'" charactersIgnoringModifiers:@"'" isARepeat:NO keyCode:kVK_ANSI_Quote]; CoreKeyOutput *output = [engine processEvent:event]; XCTAssert(output.hasTextToInsert, @"returns text to insert"); - context = engine.getCoreContext; - XCTAssert([context isEqualToString:@"\u025B\u0308"], @"Context updated with diacritic."); + context = engine.getCoreContextDebug; + XCTAssert([context isEqualToString:@"|\u025B\u0308| (len: 2) [ U+025b U+0308 ]"], @"Context updated with diacritic."); } @end From 84785fdd9ab852dec4b9f172431bde718b27ae02 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 15 Jan 2024 13:23:29 +0700 Subject: [PATCH 13/45] chore(core): wasm linker flags --- core/tests/unit/ldml/meson.build | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/core/tests/unit/ldml/meson.build b/core/tests/unit/ldml/meson.build index 6f922f1fe9..9e65ff343d 100644 --- a/core/tests/unit/ldml/meson.build +++ b/core/tests/unit/ldml/meson.build @@ -85,11 +85,17 @@ test('test_transforms', t, suite: 'ldml') # run test_context_normalization ldml unit test +normalization_tests_flags = tests_flags + +if cpp_compiler.get_id() == 'emscripten' + normalization_tests_flags += ['-lnodefs.js', '-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\']'] +endif + t = executable('test_context_normalization', ['test_context_normalization.cpp', '../emscripten_filesystem.cpp'], cpp_args: defns + warns, include_directories: [inc, libsrc, '../../../../developer/src/ext/json'], - link_args: links + tests_flags, + link_args: links + normalization_tests_flags, dependencies: [icu_uc, icu_i18n], objects: lib.extract_all_objects(recursive: false)) test('test_context_normalization', t, suite: 'ldml') From 6999b248bd1e5939f64793c7c9e49ea1a91b126d Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 15 Jan 2024 13:26:43 +0700 Subject: [PATCH 14/45] chore(mac): update one more test --- .../KeymanEngine4MacTests/CoreWrapperTests.m | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mac/KeymanEngine4Mac/KeymanEngine4MacTests/CoreWrapperTests.m b/mac/KeymanEngine4Mac/KeymanEngine4MacTests/CoreWrapperTests.m index d3e35bf464..9d0e1e2b94 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4MacTests/CoreWrapperTests.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4MacTests/CoreWrapperTests.m @@ -1,11 +1,11 @@ /** * Keyman is copyright (C) SIL International. MIT License. - * + * * CoreWrapperTest.m * CoreWrapperTests - * + * * Created by Shawn Schantz on 2023-02-17. - * + * * Description... */ @@ -29,7 +29,7 @@ CoreWrapper *mockWrapper; NSString *khmerKeyboardPath = [[[NSBundle mainBundle] resourcePath] stringByAppendingPathComponent:@"khmer_angkor.kmx"]; NSLog(@"mockKmxFilePath = %@\n", mockKmxFilePath); - + mockWrapper = [[CoreWrapper alloc] initWithHelper: [CoreTestStaticHelperMethods helper] kmxFilePath:mockKmxFilePath]; } @@ -57,7 +57,7 @@ CoreWrapper *mockWrapper; - (void)testprocessEvent_lowercaseA_returnsExpectedCharacterForKmx { NSString *kmxPath = [CoreTestStaticHelperMethods getKmxFilePathTestMacEngine]; CoreWrapper *core = [[CoreWrapper alloc] initWithHelper: [CoreTestStaticHelperMethods helper] kmxFilePath:kmxPath]; - + // expecting character with 'Ç' CoreKeyOutput *coreOutput = [core processMacVirtualKey:MVK_A withModifiers:0 withKeyDown:YES]; XCTAssert([coreOutput.textToInsert isEqualToString:@"\u00C7"], @"Expected capital C cedille (U+00C7)"); @@ -67,8 +67,8 @@ CoreWrapper *mockWrapper; NSString *kmxPath = [CoreTestStaticHelperMethods getKmxFilePathTestMacEngine]; CoreWrapper *core = [[CoreWrapper alloc] initWithHelper: [CoreTestStaticHelperMethods helper] kmxFilePath:kmxPath]; [core setContext:@"🤔?👍🏻✅"]; - NSString *finalContext = core.context; - XCTAssert([finalContext isEqualToString:@"🤔?👍🏻✅"], @"Expected '🤔?👍🏻✅' in context buffer"); + NSString *finalContext = core.contextDebug; + XCTAssert([finalContext isEqualToString:@"|🤔?👍🏻✅| (len:5) [ U+1f914 U+003f U+1f44d U+1f3fb U+2705 ]"], @"Expected '🤔?👍🏻✅' in context buffer"); } @end From 6aef3090e03da075a992690bcf6dad0bf67c624d Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 15 Jan 2024 13:39:41 +0700 Subject: [PATCH 15/45] chore(mac): missing square brackets --- mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMEngineTests.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMEngineTests.m b/mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMEngineTests.m index dbce782bef..9618cdb734 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMEngineTests.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMEngineTests.m @@ -43,7 +43,7 @@ NSString * names[nCombinations]; XCTAssert(engine != nil, @"Expected non-nil engine"); // Note: relying on km_core_state_context_debug output format is just barely // acceptable for a unit test - XCTAssert(engine.getCoreContextDebug isEqualToString:@"|| (len: 0) [ ]", @"Expected empty context buffer"); + XCTAssert([engine.getCoreContextDebug isEqualToString:@"|| (len: 0) [ ]", @"Expected empty context buffer"]); } - (void)testinitWithKMX_ValidKmxNonEmptyContext_InitializedWithContext { From 8334325316f5039875a62e39b1b45f5825df7696 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 15 Jan 2024 15:49:41 +0700 Subject: [PATCH 16/45] chore(mac): typo in test --- mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMEngineTests.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMEngineTests.m b/mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMEngineTests.m index 9618cdb734..1f989f2bdf 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMEngineTests.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMEngineTests.m @@ -43,7 +43,7 @@ NSString * names[nCombinations]; XCTAssert(engine != nil, @"Expected non-nil engine"); // Note: relying on km_core_state_context_debug output format is just barely // acceptable for a unit test - XCTAssert([engine.getCoreContextDebug isEqualToString:@"|| (len: 0) [ ]", @"Expected empty context buffer"]); + XCTAssert([engine.getCoreContextDebug isEqualToString:@"|| (len: 0) [ ]"], @"Expected empty context buffer"); } - (void)testinitWithKMX_ValidKmxNonEmptyContext_InitializedWithContext { From e853a6859abe984d02672225f4456514a153b945 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 15 Jan 2024 19:51:56 +0700 Subject: [PATCH 17/45] chore(mac): another test typo --- mac/KeymanEngine4Mac/KeymanEngine4MacTests/CoreWrapperTests.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mac/KeymanEngine4Mac/KeymanEngine4MacTests/CoreWrapperTests.m b/mac/KeymanEngine4Mac/KeymanEngine4MacTests/CoreWrapperTests.m index 9d0e1e2b94..dab7850cb8 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4MacTests/CoreWrapperTests.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4MacTests/CoreWrapperTests.m @@ -68,7 +68,7 @@ CoreWrapper *mockWrapper; CoreWrapper *core = [[CoreWrapper alloc] initWithHelper: [CoreTestStaticHelperMethods helper] kmxFilePath:kmxPath]; [core setContext:@"🤔?👍🏻✅"]; NSString *finalContext = core.contextDebug; - XCTAssert([finalContext isEqualToString:@"|🤔?👍🏻✅| (len:5) [ U+1f914 U+003f U+1f44d U+1f3fb U+2705 ]"], @"Expected '🤔?👍🏻✅' in context buffer"); + XCTAssert([finalContext isEqualToString:@"|🤔?👍🏻✅| (len: 5) [ U+1f914 U+003f U+1f44d U+1f3fb U+2705 ]"], @"Expected '🤔?👍🏻✅' in context buffer"); } @end From df177569a55602d7a1b653e5c003633f8d16e388 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Tue, 16 Jan 2024 05:55:02 +0700 Subject: [PATCH 18/45] chore(linux): libkeymancore.symbols to .244 --- linux/debian/libkeymancore.symbols | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/linux/debian/libkeymancore.symbols b/linux/debian/libkeymancore.symbols index ce57823b03..5b46e97776 100644 --- a/linux/debian/libkeymancore.symbols +++ b/linux/debian/libkeymancore.symbols @@ -16,7 +16,7 @@ libkeymancore.so.1 libkeymancore #MINVER# km_core_context_length@Base 17.0.195 km_core_context_set@Base 17.0.195 km_core_context_shrink@Base 17.0.195 - km_core_cp_dispose@Base 17.0.241 + km_core_cp_dispose@Base 17.0.244 km_core_event@Base 17.0.195 km_core_get_engine_attrs@Base 17.0.195 km_core_keyboard_dispose@Base 17.0.195 @@ -33,7 +33,7 @@ libkeymancore.so.1 libkeymancore #MINVER# km_core_state_clone@Base 17.0.195 km_core_state_context@Base 17.0.195 km_core_state_context_clear@Base 17.0.197 - km_core_state_context_debug@Base 17.0.241 + km_core_state_context_debug@Base 17.0.244 km_core_state_context_set_if_needed@Base 17.0.197 km_core_state_create@Base 17.0.195 km_core_state_debug_get@Base 17.0.195 From bd870de672ea0493643f4d433a25c27eecf21372 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Tue, 16 Jan 2024 10:01:35 +1100 Subject: [PATCH 19/45] chore: Apply suggestions from code review Co-authored-by: Steven R. Loomis --- core/src/km_core_state_api.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/km_core_state_api.cpp b/core/src/km_core_state_api.cpp index b7d65a2628..ec85911c9f 100644 --- a/core/src/km_core_state_api.cpp +++ b/core/src/km_core_state_api.cpp @@ -420,7 +420,7 @@ km_core_cp * km_core_state_context_debug( auto flags = buffer.flags(); if(cp->type == KM_CORE_CT_CHAR) { // A single Unicode codepoint - buffer << " U+" << std::hex << std::setfill('0') << std::setw(4) << std::hex << cp->character; + buffer << " U+" << std::setfill('0') << std::setw(4) << std::hex << cp->character; } else { // A marker buffer << " M(" << cp->marker << ")"; From 0878ee3b56bbc2d9ca73b011d2a3d5f3f076fc84 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Tue, 16 Jan 2024 06:21:38 +0700 Subject: [PATCH 20/45] feat(core): seed Keyman Core normalization Relates to #9999 --- core/README.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 core/README.md diff --git a/core/README.md b/core/README.md new file mode 100644 index 0000000000..b1ddaa2f41 --- /dev/null +++ b/core/README.md @@ -0,0 +1,3 @@ +# Keyman Core + +Version 1.0.0 of libkeymancore released with Keyman 17.0 \ No newline at end of file From c8ee73321b8183dda7781a67dd9265a5b6a250fe Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Tue, 16 Jan 2024 13:33:00 +1100 Subject: [PATCH 21/45] chore: Apply suggestions from code review --- .../KeymanEngine4MacTests/CoreWrapperTests.m | 2 ++ .../KeymanEngine4MacTests/KMEngineTests.m | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/mac/KeymanEngine4Mac/KeymanEngine4MacTests/CoreWrapperTests.m b/mac/KeymanEngine4Mac/KeymanEngine4MacTests/CoreWrapperTests.m index dab7850cb8..826d803328 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4MacTests/CoreWrapperTests.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4MacTests/CoreWrapperTests.m @@ -68,6 +68,8 @@ CoreWrapper *mockWrapper; CoreWrapper *core = [[CoreWrapper alloc] initWithHelper: [CoreTestStaticHelperMethods helper] kmxFilePath:kmxPath]; [core setContext:@"🤔?👍🏻✅"]; NSString *finalContext = core.contextDebug; + // Note: relying on km_core_state_context_debug output format is just barely + // acceptable for a unit test XCTAssert([finalContext isEqualToString:@"|🤔?👍🏻✅| (len: 5) [ U+1f914 U+003f U+1f44d U+1f3fb U+2705 ]"], @"Expected '🤔?👍🏻✅' in context buffer"); } diff --git a/mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMEngineTests.m b/mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMEngineTests.m index 1f989f2bdf..7bb1ed97fa 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMEngineTests.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMEngineTests.m @@ -563,6 +563,8 @@ NSString * names[nCombinations]; XCTAssert(output.codePointsToDeleteBeforeInsert == 1, @"Expected output to delete one code point"); XCTAssert(!output.hasTextToInsert, @"expected to insert nothing"); NSString *context = engine.getCoreContextDebug; + // Note: relying on km_core_state_context_debug output format is just barely + // acceptable for a unit test XCTAssert([context isEqualToString:@"|| (len: 0) [ ]"], @"Context should be empty."); } @@ -573,6 +575,8 @@ NSString * names[nCombinations]; CoreKeyOutput *output = [engine processEvent:event]; XCTAssert(output.emitKeystroke, @"Expected emitKeystroke==YES"); NSString *context = engine.getCoreContextDebug; + // Note: relying on km_core_state_context_debug output format is just barely + // acceptable for a unit test XCTAssert([context isEqualToString:@"|| (len: 0) [ ]"], @"Context should be cleared."); } @@ -583,6 +587,8 @@ NSString * names[nCombinations]; CoreKeyOutput *output = [engine processEvent:event]; XCTAssert(output.emitKeystroke, @"Expected emitKeystroke==YES"); NSString *context = engine.getCoreContextDebug; + // Note: relying on km_core_state_context_debug output format is just barely + // acceptable for a unit test XCTAssert([context isEqualToString:@"|| (len: 0) [ ]"], @"Context should be cleared."); } @@ -594,6 +600,8 @@ NSString * names[nCombinations]; CoreKeyOutput *output = [engine processEvent:event]; XCTAssert(output.hasTextToInsert, @"returns text to insert"); context = engine.getCoreContextDebug; + // Note: relying on km_core_state_context_debug output format is just barely + // acceptable for a unit test XCTAssert([context isEqualToString:@"|\u025B\u0308| (len: 2) [ U+025b U+0308 ]"], @"Context updated with diacritic."); } From 7b4417599a052eefdeb336e3edac1273700724b3 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Tue, 16 Jan 2024 11:12:01 +0700 Subject: [PATCH 22/45] refactor(mac): remove legacy Core context API calls Relates to #9999. --- .../KeymanTests/TestAppDelegate.m | 6 +++--- .../CoreWrapper/CoreWrapper.h | 1 - .../CoreWrapper/CoreWrapper.m | 19 ------------------- .../KeymanEngine4Mac/KME/KMEngine.h | 1 - .../KeymanEngine4Mac/KME/KMEngine.m | 6 +----- .../KeymanEngine4MacTests/CoreWrapperTests.m | 2 +- 6 files changed, 5 insertions(+), 30 deletions(-) diff --git a/mac/Keyman4MacIM/KeymanTests/TestAppDelegate.m b/mac/Keyman4MacIM/KeymanTests/TestAppDelegate.m index a17e1f78da..e646e33905 100644 --- a/mac/Keyman4MacIM/KeymanTests/TestAppDelegate.m +++ b/mac/Keyman4MacIM/KeymanTests/TestAppDelegate.m @@ -18,7 +18,7 @@ if (_kme == nil) { _kme = [[KMEngine alloc] initWithKMX:nil context:self.contextBuffer verboseLogging:self.debugMode]; } - + return _kme; } @@ -31,7 +31,7 @@ if (_contextBuffer == nil) { _contextBuffer = [[NSMutableString alloc] initWithString:@""]; } - + return _contextBuffer; } @@ -39,7 +39,7 @@ _contextBuffer = [contextBuffer mutableCopy]; if (_contextBuffer.length) [_contextBuffer replaceOccurrencesOfString:@"\0" withString:[NSString nullChar] options:0 range:NSMakeRange(0, 1)]; - [self.kme setCoreContext:self.contextBuffer]; + [self.kme setCoreContextIfNeeded:self.contextBuffer]; } - (BOOL)debugMode { diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.h b/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.h index 8cc1c68c9d..1109b1130f 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.h +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.h @@ -28,7 +28,6 @@ NS_ASSUME_NONNULL_BEGIN withModifiers:(NSEventModifierFlags)modifierState withKeyDown:(BOOL)isKeyDown; -(void)setContextIfNeeded:(NSString*)context; --(void)setContext:(NSString*)context; -(NSString*)contextDebug; -(void)clearCoreContext; -(void)dealloc; diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m index d736d80279..8472d70d39 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m @@ -321,25 +321,6 @@ const int CORE_ENVIRONMENT_ARRAY_LENGTH = 6; [self.coreHelper logDebugMessage:@"CoreWrapper setContextIfNeeded, context=%@, km_core_state_context_set_if_needed result=%i", context, result]; } --(void)setContext:(NSString*)context { - if (context.length == 0) { - [self clearContextUsingCore]; - } else { - char const *coreString = [context cStringUsingEncoding:NSUTF8StringEncoding]; - km_core_context_item *contextItemArray; - - // create array of context items - km_core_status result = km_core_context_items_from_utf8(coreString, &contextItemArray); - [self.coreHelper logDebugMessage:@"km_core_context_items_from_utf8, result=%i", result]; - - // set the context in core using the array - km_core_context * coreContext = km_core_state_context(self.coreState); - km_core_context_set(coreContext, contextItemArray); - // dispose - km_core_context_items_dispose(contextItemArray); - } -} - -(NSString*)contextDebug { km_core_cp * context = km_core_state_context_debug(self.coreState, KM_CORE_DEBUG_CONTEXT_CACHED); NSString *debugString = [self.coreHelper createNSStringFromUnicharString:context]; diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.h b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.h index 711c34b9ba..c79b02ba32 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.h +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.h @@ -24,7 +24,6 @@ - (NSString *)getCoreContextDebug; - (void)clearCoreContext; - (void)setCoreContextIfNeeded:(NSString *)context; -- (void)setCoreContext:(NSString *)context; - (void)setCoreOptions:(NSString *)key withValue:(NSString *)value; - (CoreKeyOutput *)processEvent:(NSEvent *)event; diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m index 7afd3d40b5..ea59fb950d 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m @@ -39,7 +39,7 @@ const NSString* kEasterEggKmxName = @"EnglishSpanish.kmx"; if (kmx) { [self loadCoreWrapperFromKmxFile:self.kmx.filePath]; - [self.coreWrapper setContext:contextString]; + [self.coreWrapper setContextIfNeeded:contextString]; } } @@ -101,10 +101,6 @@ const NSString* kEasterEggKmxName = @"EnglishSpanish.kmx"; [self.coreWrapper setContextIfNeeded:context]; } -- (void)setCoreContext:(NSString *)context { - [self.coreWrapper setContext:context]; -} - - (void)setCoreOptions:(NSString *)key withValue:(NSString *)value { BOOL success = [self.coreWrapper setOptionsForCore:key value:value]; [self.coreHelper logDebugMessage:@"setCoreOptions for key: %@, value: %@ succeeded = %@", key, value, success ? @"YES" : @"NO"]; diff --git a/mac/KeymanEngine4Mac/KeymanEngine4MacTests/CoreWrapperTests.m b/mac/KeymanEngine4Mac/KeymanEngine4MacTests/CoreWrapperTests.m index 826d803328..2748695a81 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4MacTests/CoreWrapperTests.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4MacTests/CoreWrapperTests.m @@ -66,7 +66,7 @@ CoreWrapper *mockWrapper; - (void)testgetContextAsString_ContextContainsEmojis_ReturnsSameContext { NSString *kmxPath = [CoreTestStaticHelperMethods getKmxFilePathTestMacEngine]; CoreWrapper *core = [[CoreWrapper alloc] initWithHelper: [CoreTestStaticHelperMethods helper] kmxFilePath:kmxPath]; - [core setContext:@"🤔?👍🏻✅"]; + [core setContextIfNeeded:@"🤔?👍🏻✅"]; NSString *finalContext = core.contextDebug; // Note: relying on km_core_state_context_debug output format is just barely // acceptable for a unit test From e5c25252410fa60823f8fb29fe2bd101f08d7eb6 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Tue, 16 Jan 2024 12:04:41 +0700 Subject: [PATCH 23/45] refactor(core): split context API from Core primary API Relates to #9999. Fixes #10384. The context API endpoints should no longer be considered as part of the standard Core API. The only consumers that have a need to access these APIs are the IMX integration in Engine for Windows, and the Keyman Developer Debugger. These symbols are currently used by Developer: * `km_core_context` struct * `km_core_context_type` enum * `km_core_context_item` struct * `KM_CORE_CONTEXT_ITEM_END` macro * `km_core_state_context()` * `km_core_context_set()` * `km_core_context_clear()` These symbols are currently used by Windows IMX: * `km_core_context` struct * `km_core_context_type` enum * `km_core_context_item` struct * `KM_CORE_CONTEXT_ITEM_END` macro * `km_core_context_items_dispose()` * `km_core_context_item_list_size()` * `km_core_state_get_intermediate_context()` The following functions and symbols are moving to keyman_core_api_context.h: * `km_core_context` struct * `km_core_context_type` enum * `km_core_context_item` struct * `KM_CORE_CONTEXT_ITEM_END` macro * `km_core_state_context()` function * `km_core_state_get_intermediate_context()` function * `km_core_context_set()` function * `km_core_context_clear()` function * `km_core_context_get()` function * `km_core_context_items_from_utf16()` function * `km_core_context_items_from_utf8()` function * `km_core_context_items_to_utf8()` function * `km_core_context_items_to_utf16()` function * `km_core_context_items_to_utf32()` function * `km_core_context_items_dispose()` function * `km_core_context_length()` function * `km_core_context_append()` function * `km_core_context_shrink()` function * `km_core_context_item_list_size()` function --- core/include/keyman/keyman_core_api.h | 391 -------------- core/include/keyman/keyman_core_api_context.h | 478 ++++++++++++++++++ core/meson.build | 3 + core/src/action.cpp | 1 - core/src/action.hpp | 2 +- core/src/context.hpp | 2 +- core/src/debug.hpp | 3 +- core/src/keyboard.hpp | 2 +- core/src/keyman_core.h | 5 + core/src/km_core_action_api.cpp | 3 +- core/src/km_core_context_api.cpp | 2 +- core/src/km_core_debug_api.cpp | 2 +- core/src/km_core_keyboard_api.cpp | 3 +- core/src/km_core_options_api.cpp | 3 +- core/src/km_core_processevent_api.cpp | 3 +- core/src/km_core_state_api.cpp | 2 +- .../km_core_state_context_set_if_needed.cpp | 2 +- core/src/kmx/kmx_base.h | 3 +- core/src/kmx/kmx_options.h | 2 +- core/src/kmx/kmx_processor.cpp | 2 +- core/src/kmx/kmx_processor.hpp | 2 +- core/src/ldml/ldml_processor.hpp | 2 +- core/src/ldml/ldml_vkeys.hpp | 2 +- core/src/mock/mock_processor.hpp | 2 +- core/src/option.hpp | 3 +- core/src/path.hpp | 3 +- core/src/processor.hpp | 2 +- core/src/state.hpp | 2 +- core/tests/unit/kmnkbd/action_api.cpp | 2 +- core/tests/unit/kmnkbd/action_items.hpp | 2 +- core/tests/unit/kmnkbd/action_set_api.cpp | 3 +- core/tests/unit/kmnkbd/context_api.cpp | 3 +- core/tests/unit/kmnkbd/debug_api.cpp | 4 +- core/tests/unit/kmnkbd/debug_items.hpp | 3 +- core/tests/unit/kmnkbd/keyboard_api.cpp | 2 +- core/tests/unit/kmnkbd/options_api.cpp | 2 +- core/tests/unit/kmnkbd/state_api.cpp | 2 +- core/tests/unit/kmx/kmx.cpp | 2 + core/tests/unit/ldml/ldml.cpp | 2 + core/tests/unit/ldml/ldml_test_utils.hpp | 3 +- .../unit/ldml/test_context_normalization.cpp | 2 +- windows/src/engine/keyman32/keymanengine.h | 1 + 42 files changed, 536 insertions(+), 429 deletions(-) create mode 100644 core/include/keyman/keyman_core_api_context.h create mode 100644 core/src/keyman_core.h diff --git a/core/include/keyman/keyman_core_api.h b/core/include/keyman/keyman_core_api.h index 3e4451f612..e744e6bf60 100644 --- a/core/include/keyman/keyman_core_api.h +++ b/core/include/keyman/keyman_core_api.h @@ -115,7 +115,6 @@ typedef uint32_t km_core_status; // Status return code. // Opaque object types. // -typedef struct km_core_context km_core_context; typedef struct km_core_keyboard km_core_keyboard; typedef struct km_core_state km_core_state; typedef struct km_core_options km_core_options; @@ -155,357 +154,7 @@ The final status code KM_CORE_STATUS_OS_ERROR is intended to allow encapsulating a platform error code; the remaining 31 low bits are the error code returned by the OS for cases where the failure mode is platform specific. For HRESULT codes this only permits failure codes to be passed. - - -### Context -The context is the text prior to the insertion point (caret, cursor). -The context is constructed by the Platform layer, typically by interrogating the -Client Application. The context will be updated by the engine for keystroke -events. If the Platform layer code caches the context, the context should be -reset when a context state change is detected. Context state changes can occur -when the user uses the mouse to move the insertion point, uses cursor keys, -switches applications or input fields, or presses hotkeys such as Ctrl+N to -start a new document. The full set of context state change triggers is up to the -Platform layer. - -Context can also contain positional Markers (also known as 'deadkeys' in kmn -keyboards), which are transitory state flags that are erased whenever a context -state change is detected. Markers are always controlled by the Engine. - -Contexts are always owned by their state. They may be set to a list of -context_items or interrogated for their current list of context items. -```c */ -enum km_core_context_type { - KM_CORE_CT_END, - KM_CORE_CT_CHAR, - KM_CORE_CT_MARKER -}; - -typedef struct { - uint8_t type; - uint8_t _reserved[3]; - union { - km_core_usv character; - uint32_t marker; - }; -} km_core_context_item; - -#define KM_CORE_CONTEXT_ITEM_END {KM_CORE_CT_END, {0,}, {0,}} -/* -``` -### `km_core_context_items_from_utf16` -##### Description: -Convert a UTF16 encoded Unicode string into an array of `km_core_context_item` -structures. Allocates memory as needed. -##### Return status: -- `KM_CORE_STATUS_OK`: On success. -- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. -- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated for the - output buffer. -- `KM_CORE_STATUS_INVALID_UTF`: In the event the UTF16 string cannot be decoded - because it contains unpaired surrogate codeunits. -##### Parameters: -- __text__: a pointer to a null terminated array of utf16 encoded data. -- __out_ptr__: a pointer to the result variable: - A pointer to the start of the `km_core_context_item` array containing the - representation of the input string. - Terminated with a type of `KM_CORE_CT_END`. Must be disposed of with - `km_core_context_items_dispose`. - -```c -*/ -KMN_API -km_core_status -km_core_context_items_from_utf16(km_core_cp const *text, - km_core_context_item **out_ptr); - -/* -``` -### `km_core_context_items_from_utf8` -##### Description: -Convert an UTF8 encoded Unicode string into an array of `km_core_context_item` -structures. Allocates memory as needed. -##### Status: -- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. -- `KM_CORE_STATUS_NO_MEM`: In the event it cannot allocate enough memory for the - output buffer. -- `KM_CORE_STATUS_INVALID_UTF`: In the event the UTF8 string cannot be -decoded. -##### Parameters: -- __text__: a pointer to a null terminated array of utf8 encoded data. -- __out_ptr__: a pointer to the result variable: - A pointer to the start of the `km_core_context_item` array containing the - representation of the input string. - Terminated with a type of `KM_CORE_CT_END`. - -```c -*/ -KMN_API -km_core_status -km_core_context_items_from_utf8(char const *text, - km_core_context_item **out_ptr); - -/* -``` -### `km_core_context_items_to_utf16` -##### Description: -Convert a context item array into a UTF-16 encoded string placing it into -the supplied buffer of specified size, and return the number of code units -actually used in the conversion. If null is passed as the buffer the -number of codeunits required is returned. Any markers in the context will -not be included in the output buffer. -##### Return status: -- `KM_CORE_STATUS_OK`: On success. -- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. -- `KM_CORE_STATUS_INSUFFICENT_BUFFER`: If the buffer is not large enough. - `buf_size` will contain the space required. The contents of the buffer are - undefined. -##### Parameters: -- __context_items__: A pointer to the start of an array `km_core_context_item`. - Must be terminated with a type of `KM_CORE_CT_END`. -- __buf__: A pointer to the buffer to place the UTF-16 string into. - May be null to request size calculation. -- __buf_size__: a pointer to the result variable: - The size of the supplied buffer in codeunits if `buf` is given. - On return will be the size required if `buf` is null. - -```c -*/ -KMN_API -km_core_status -km_core_context_items_to_utf16(km_core_context_item const *item, - km_core_cp *buf, - size_t *buf_size); - -/* -``` -### `km_core_context_items_to_utf8` -##### Description: -Convert a context item array into a UTF-8 encoded string placing it into -the supplied buffer of specified size, and return the number of code units -actually used in the conversion. If null is passed as the buffer the -number of codeunits required is returned. Any markers in the context will -not be included in the output buffer. -##### Return status: -- `KM_CORE_STATUS_OK`: On success. -- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. -- `KM_CORE_STATUS_INSUFFICENT_BUFFER`: If the buffer is not large enough. - `buf_size` will contain the space required. The contents of the buffer are - undefined. -##### Parameters: -- __context_items__: A pointer to the start of an array `km_core_context_item`. - Must be terminated with a type of `KM_CORE_CT_END`. -- __buf__: A pointer to the buffer to place the UTF-8 string into. - May be null to request size calculation. -- __buf_size__: a pointer to the result variable: - The size of the supplied buffer in codeunits if `buf` is given. - On return will be the size required if `buf` is null. - -```c -*/ -KMN_API -km_core_status -km_core_context_items_to_utf8(km_core_context_item const *item, - char *buf, - size_t *buf_size); - -/* -``` -### `km_core_context_items_to_utf32` -##### Description: -Convert a context item array into a UTF-32 encoded string placing it into -the supplied buffer of specified size, and return the number of codepoints -actually used in the conversion. If null is passed as the buffer the -number of codepoints required is returned. Any markers in the context will -not be included in the output buffer. -##### Return status: -- `KM_CORE_STATUS_OK`: On success. -- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. -- `KM_CORE_STATUS_INSUFFICENT_BUFFER`: If the buffer is not large enough. - `buf_size` will contain the space required. The contents of the buffer are - undefined. -##### Parameters: -- __context_items__: A pointer to the start of an array `km_core_context_item`. - Must be terminated with a type of `KM_CORE_CT_END`. -- __buf__: A pointer to the buffer to place the UTF-32 string into. - May be null to request size calculation. -- __buf_size__: a pointer to the result variable: - The size of the supplied buffer in codepoints if `buf` is given. - On return will be the size required if `buf` is null. - -```c -*/ -KMN_API -km_core_status -km_core_context_items_to_utf32(km_core_context_item const *item, - km_core_usv *buf, - size_t *buf_size); - -/* -``` -### `km_core_context_items_dispose` -##### Description: -Free the allocated memory belonging to a `km_core_context_item` array previously -returned by `km_core_context_items_from_utf16` or `km_core_context_get` -##### Parameters: -- __context_items__: A pointer to the start of the `km_core_context_item` array - to be disposed of. - -```c -*/ -KMN_API -void -km_core_context_items_dispose(km_core_context_item *context_items); - -/* -``` -### `km_core_context_set` -##### Description: -Replace the contents of the current context with a new sequence of -`km_core_context_item` entries. -##### Return status: -- `KM_CORE_STATUS_OK`: On success. -- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. -- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated to - grow the context buffer internally. -##### Parameters: -- __context__: A pointer to an opaque context object -- __context_items__: A pointer to the start of the `km_core_context_item` - array containing the new context. It must be terminated with an item - of type `KM_CORE_CT_END`. - -```c -*/ -KMN_API -km_core_status -km_core_context_set(km_core_context *context, - km_core_context_item const *context_items); - -/* -``` -### `km_core_context_get` -##### Description: -Copies all items in the context into a new array and returns the new array. -This must be disposed of by caller using `km_core_context_items_dispose`. -##### Return status: -- `KM_CORE_STATUS_OK`: On success. -- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. -- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated for the - output buffer. -##### Parameters: -- __context_items__: A pointer to the start of an array `km_core_context_item`. -- __out__: a pointer to the result variable: - A pointer to the start of the `km_core_context_item` array containing a - copy of the context. Terminated with a type of `KM_CORE_CT_END`. Must be - disposed of with `km_core_context_items_dispose`. - -```c -*/ -KMN_API -km_core_status -km_core_context_get(km_core_context const *context_items, - km_core_context_item **out); - -/* -``` -### `km_core_context_clear` -##### Description: -Removes all context_items from the internal array. If `context` is -null, has no effect. -##### Parameters: -- __context__: A pointer to an opaque context object - -```c -*/ -KMN_API -void -km_core_context_clear(km_core_context *); - -/* -``` -### `km_core_context_length` -##### Description: -Return the number of items in the context. -##### Return: -The number of items in the context, and will return 0 if passed a null `context` -pointer. -##### Parameters: -- __context__: A pointer to an opaque context object - -```c -*/ -KMN_API -size_t -km_core_context_length(km_core_context *); - -/* -``` -### `km_core_context_append` -##### Description: -Add more items to the end (insertion point) of the context. If these exceed the -maximum context length the same number of items will be dropped from the -beginning of the context. -##### Return status: -- `KM_CORE_STATUS_OK`: On success. -- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. -- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated to - grow the context buffer internally. -##### Parameters: -- __context__: A pointer to an opaque context object. -- __context_items__: A pointer to the start of the `KM_CORE_CT_END` terminated - array of `km_core_context_item` to append. - -```c -*/ -KMN_API -km_core_status -km_core_context_append(km_core_context *context, - km_core_context_item const *context_items); - -/* -``` -### `km_core_context_shrink` -##### Description: -Remove a specified number of items from the end of the context, optionally -add up to the same number of the supplied items to the front of the context. -##### Return status: -- `KM_CORE_STATUS_OK`: On success. -- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. -- `KM_CORE_STATUS_NO_MEM`: in the event it cannot allocated enough memory to grow - the context internally. -##### Parameters: -- __context__: A pointer to an opaque context object. -- __num__: The number of items to remove from the end of context. -- __context_items__: Pointer to the start of the `KM_CORE_CT_END` terminated - array of `km_core_context_item` to add to the front. Up to `num` items will - be prepended. This may be null if not required. - -```c -*/ -KMN_API -km_core_status -km_core_context_shrink(km_core_context *context, - size_t num, - km_core_context_item const *prefix); - -/* -``` -### `km_core_context_item_list_size` -##### Description: -Return the length of a terminated `km_core_context_item` array. -##### Return: -The number of items in the list, not including terminating item, -or 0 if `context_items` is null. -##### Parameters: -- __context_items__: A pointer to a `KM_CORE_CT_END` terminated array of - `km_core_context_item` values. - -```c -*/ -KMN_API -size_t -km_core_context_item_list_size(km_core_context_item const *context_items); /* ``` @@ -1104,46 +753,6 @@ KMN_API void km_core_state_dispose(km_core_state *state); -/* -``` -### `km_core_state_context` -##### Description: -Get access to the state object's context. -##### Return: -A pointer to an opaque state object. This pointer is valid for the lifetime -of the state object. If null is passed in, then null is returned. -##### Parameters: -- __state__: A pointer to the opaque state object to be queried. - -```c -*/ -KMN_API -km_core_context * -km_core_state_context(km_core_state *state); - - -KMN_API -km_core_context * -km_core_state_app_context(km_core_state *state); - -/* -``` -### `km_core_state_get_intermediate_context` -##### Description: -Get access to the state object's keyboard processor's intermediate context. This context -is used during an IMX callback, part way through processing a keystroke. -##### Return: -A pointer to an context item array. Must be disposed of by a call -to `km_core_context_items_dispose`. -##### Parameters: -- __state__: A pointer to the opaque state object to be queried. - -```c -*/ -KMN_API -km_core_status -km_core_state_get_intermediate_context(km_core_state *state, km_core_context_item ** context_items); - /* ``` ### Context Debug Reporting diff --git a/core/include/keyman/keyman_core_api_context.h b/core/include/keyman/keyman_core_api_context.h new file mode 100644 index 0000000000..26bd2f25bc --- /dev/null +++ b/core/include/keyman/keyman_core_api_context.h @@ -0,0 +1,478 @@ +/* + * Keyman is copyright (C) SIL International. MIT License. + * + * Keyman Keyboard Processor API - Debugger Interfaces + * + * The debugger interfaces are still very dependent on .kmx + * objects. + * + * Note: this file is subject to change; the debugger + * interfaces are not stable across versions. + * + */ + +#pragma once + +#include +#include +#include +#include + +// Currently, the Core unit tests use private context APIs defined in +// keyman_core_api_context.h, which are unused by other consumers. We are +// hoping to remove these entirely in the future, so we restrict access +// by default with this macro. Keyman Core internally uses these functions +// #define _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +#if defined(__cplusplus) +extern "C" +{ +#endif + +typedef struct km_core_context km_core_context; + +// ---------------------------------------------------------------------------------- +// Context APIs are now available only to the keyboard debugger and Core unit +// tests (17.0) +// ---------------------------------------------------------------------------------- + +/* +### Context +The context is the text prior to the insertion point (caret, cursor). +The context is constructed by the Platform layer, typically by interrogating the +Client Application. The context will be updated by the engine for keystroke +events. If the Platform layer code caches the context, the context should be +reset when a context state change is detected. Context state changes can occur +when the user uses the mouse to move the insertion point, uses cursor keys, +switches applications or input fields, or presses hotkeys such as Ctrl+N to +start a new document. The full set of context state change triggers is up to the +Platform layer. + +Context can also contain positional Markers (also known as 'deadkeys' in kmn +keyboards), which are transitory state flags that are erased whenever a context +state change is detected. Markers are always controlled by the Engine. + +Contexts are always owned by their state. They may be set to a list of +context_items or interrogated for their current list of context items. +```c +*/ +enum km_core_context_type { + KM_CORE_CT_END, + KM_CORE_CT_CHAR, + KM_CORE_CT_MARKER +}; + +typedef struct { + uint8_t type; + uint8_t _reserved[3]; + union { + km_core_usv character; + uint32_t marker; + }; +} km_core_context_item; + +#define KM_CORE_CONTEXT_ITEM_END {KM_CORE_CT_END, {0,}, {0,}} + +/* +``` +### `km_core_state_get_intermediate_context` +##### Description: +Get access to the state object's keyboard processor's intermediate context. This context +is used during an IMX callback, part way through processing a keystroke. +##### Return: +A pointer to an context item array. Must be disposed of by a call +to `km_core_context_items_dispose`. +##### Parameters: +- __state__: A pointer to the opaque state object to be queried. + +```c +*/ +KMN_API +km_core_status +km_core_state_get_intermediate_context(km_core_state *state, km_core_context_item ** context_items); + +/* +``` +### `km_core_context_items_dispose` +##### Description: +Free the allocated memory belonging to a `km_core_context_item` array previously +returned by `km_core_context_items_from_utf16` or `km_core_context_get` +##### Parameters: +- __context_items__: A pointer to the start of the `km_core_context_item` array + to be disposed of. + +```c +*/ +KMN_API +void +km_core_context_items_dispose(km_core_context_item *context_items); + +/** + * Get access to the state object's cached context. + * @param state A pointer to the opaque state object to be queried. + * @returns A pointer to an opaque context object. This pointer is valid for the + * lifetime of the state object. If null is passed in, then null is + * returned. + */ +KMN_API +km_core_context * +km_core_state_context(km_core_state *state); + +/** + * Get access to the state object's application context. + * @param state A pointer to the opaque state object to be queried. + * @returns A pointer to an opaque context object. This pointer is valid for the + * lifetime of the state object. If null is passed in, then null is + * returned. + */ +KMN_API +km_core_context * +km_core_state_app_context(km_core_state *state); + +/* +``` +### `km_core_context_items_from_utf16` +##### Description: +Convert a UTF16 encoded Unicode string into an array of `km_core_context_item` +structures. Allocates memory as needed. +##### Return status: +- `KM_CORE_STATUS_OK`: On success. +- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. +- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated for the + output buffer. +- `KM_CORE_STATUS_INVALID_UTF`: In the event the UTF16 string cannot be decoded + because it contains unpaired surrogate codeunits. +##### Parameters: +- __text__: a pointer to a null terminated array of utf16 encoded data. +- __out_ptr__: a pointer to the result variable: + A pointer to the start of the `km_core_context_item` array containing the + representation of the input string. + Terminated with a type of `KM_CORE_CT_END`. Must be disposed of with + `km_core_context_items_dispose`. + +```c +*/ + +#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +KMN_API +km_core_status +km_core_context_items_from_utf16(km_core_cp const *text, + km_core_context_item **out_ptr); + +#endif + +/* +``` +### `km_core_context_items_from_utf8` +##### Description: +Convert an UTF8 encoded Unicode string into an array of `km_core_context_item` +structures. Allocates memory as needed. +##### Status: +- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. +- `KM_CORE_STATUS_NO_MEM`: In the event it cannot allocate enough memory for the + output buffer. +- `KM_CORE_STATUS_INVALID_UTF`: In the event the UTF8 string cannot be +decoded. +##### Parameters: +- __text__: a pointer to a null terminated array of utf8 encoded data. +- __out_ptr__: a pointer to the result variable: + A pointer to the start of the `km_core_context_item` array containing the + representation of the input string. + Terminated with a type of `KM_CORE_CT_END`. + +```c +*/ + +#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +KMN_API +km_core_status +km_core_context_items_from_utf8(char const *text, + km_core_context_item **out_ptr); + +#endif + +/* +``` +### `km_core_context_items_to_utf16` +##### Description: +Convert a context item array into a UTF-16 encoded string placing it into +the supplied buffer of specified size, and return the number of code units +actually used in the conversion. If null is passed as the buffer the +number of codeunits required is returned. Any markers in the context will +not be included in the output buffer. +##### Return status: +- `KM_CORE_STATUS_OK`: On success. +- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. +- `KM_CORE_STATUS_INSUFFICENT_BUFFER`: If the buffer is not large enough. + `buf_size` will contain the space required. The contents of the buffer are + undefined. +##### Parameters: +- __context_items__: A pointer to the start of an array `km_core_context_item`. + Must be terminated with a type of `KM_CORE_CT_END`. +- __buf__: A pointer to the buffer to place the UTF-16 string into. + May be null to request size calculation. +- __buf_size__: a pointer to the result variable: + The size of the supplied buffer in codeunits if `buf` is given. + On return will be the size required if `buf` is null. + +```c +*/ + +#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +KMN_API +km_core_status +km_core_context_items_to_utf16(km_core_context_item const *item, + km_core_cp *buf, + size_t *buf_size); + +#endif + +/* +``` +### `km_core_context_items_to_utf8` +##### Description: +Convert a context item array into a UTF-8 encoded string placing it into +the supplied buffer of specified size, and return the number of code units +actually used in the conversion. If null is passed as the buffer the +number of codeunits required is returned. Any markers in the context will +not be included in the output buffer. +##### Return status: +- `KM_CORE_STATUS_OK`: On success. +- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. +- `KM_CORE_STATUS_INSUFFICENT_BUFFER`: If the buffer is not large enough. + `buf_size` will contain the space required. The contents of the buffer are + undefined. +##### Parameters: +- __context_items__: A pointer to the start of an array `km_core_context_item`. + Must be terminated with a type of `KM_CORE_CT_END`. +- __buf__: A pointer to the buffer to place the UTF-8 string into. + May be null to request size calculation. +- __buf_size__: a pointer to the result variable: + The size of the supplied buffer in codeunits if `buf` is given. + On return will be the size required if `buf` is null. + +```c +*/ + +#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +KMN_API +km_core_status +km_core_context_items_to_utf8(km_core_context_item const *item, + char *buf, + size_t *buf_size); + +#endif + +/* +``` +### `km_core_context_items_to_utf32` +##### Description: +Convert a context item array into a UTF-32 encoded string placing it into +the supplied buffer of specified size, and return the number of codepoints +actually used in the conversion. If null is passed as the buffer the +number of codepoints required is returned. Any markers in the context will +not be included in the output buffer. +##### Return status: +- `KM_CORE_STATUS_OK`: On success. +- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. +- `KM_CORE_STATUS_INSUFFICENT_BUFFER`: If the buffer is not large enough. + `buf_size` will contain the space required. The contents of the buffer are + undefined. +##### Parameters: +- __context_items__: A pointer to the start of an array `km_core_context_item`. + Must be terminated with a type of `KM_CORE_CT_END`. +- __buf__: A pointer to the buffer to place the UTF-32 string into. + May be null to request size calculation. +- __buf_size__: a pointer to the result variable: + The size of the supplied buffer in codepoints if `buf` is given. + On return will be the size required if `buf` is null. + +```c +*/ + +#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +KMN_API +km_core_status +km_core_context_items_to_utf32(km_core_context_item const *item, + km_core_usv *buf, + size_t *buf_size); + +#endif + +/* +``` +### `km_core_context_set` +##### Description: +Replace the contents of the current context with a new sequence of +`km_core_context_item` entries. +##### Return status: +- `KM_CORE_STATUS_OK`: On success. +- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. +- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated to + grow the context buffer internally. +##### Parameters: +- __context__: A pointer to an opaque context object +- __context_items__: A pointer to the start of the `km_core_context_item` + array containing the new context. It must be terminated with an item + of type `KM_CORE_CT_END`. + +```c +*/ +KMN_API +km_core_status +km_core_context_set(km_core_context *context, + km_core_context_item const *context_items); + +/* +``` +### `km_core_context_get` +##### Description: +Copies all items in the context into a new array and returns the new array. +This must be disposed of by caller using `km_core_context_items_dispose`. +##### Return status: +- `KM_CORE_STATUS_OK`: On success. +- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. +- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated for the + output buffer. +##### Parameters: +- __context_items__: A pointer to the start of an array `km_core_context_item`. +- __out__: a pointer to the result variable: + A pointer to the start of the `km_core_context_item` array containing a + copy of the context. Terminated with a type of `KM_CORE_CT_END`. Must be + disposed of with `km_core_context_items_dispose`. + +```c +*/ + +#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +KMN_API +km_core_status +km_core_context_get(km_core_context const *context_items, + km_core_context_item **out); + +#endif + +/* +``` +### `km_core_context_clear` +##### Description: +Removes all context_items from the internal array. If `context` is +null, has no effect. +##### Parameters: +- __context__: A pointer to an opaque context object + +```c +*/ +KMN_API +void +km_core_context_clear(km_core_context *); + +/* +``` +### `km_core_context_length` +##### Description: +Return the number of items in the context. +##### Return: +The number of items in the context, and will return 0 if passed a null `context` +pointer. +##### Parameters: +- __context__: A pointer to an opaque context object + +```c +*/ + +#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +KMN_API +size_t +km_core_context_length(km_core_context *); + +#endif + +/* +``` +### `km_core_context_append` +##### Description: +Add more items to the end (insertion point) of the context. If these exceed the +maximum context length the same number of items will be dropped from the +beginning of the context. +##### Return status: +- `KM_CORE_STATUS_OK`: On success. +- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. +- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated to + grow the context buffer internally. +##### Parameters: +- __context__: A pointer to an opaque context object. +- __context_items__: A pointer to the start of the `KM_CORE_CT_END` terminated + array of `km_core_context_item` to append. + +```c +*/ + +#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +KMN_API +km_core_status +km_core_context_append(km_core_context *context, + km_core_context_item const *context_items); + +#endif + +/* +``` +### `km_core_context_shrink` +##### Description: +Remove a specified number of items from the end of the context, optionally +add up to the same number of the supplied items to the front of the context. +##### Return status: +- `KM_CORE_STATUS_OK`: On success. +- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. +- `KM_CORE_STATUS_NO_MEM`: in the event it cannot allocated enough memory to grow + the context internally. +##### Parameters: +- __context__: A pointer to an opaque context object. +- __num__: The number of items to remove from the end of context. +- __context_items__: Pointer to the start of the `KM_CORE_CT_END` terminated + array of `km_core_context_item` to add to the front. Up to `num` items will + be prepended. This may be null if not required. + +```c +*/ + +#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +KMN_API +km_core_status +km_core_context_shrink(km_core_context *context, + size_t num, + km_core_context_item const *prefix); + +#endif + +/* +``` +### `km_core_context_item_list_size` +##### Description: +Return the length of a terminated `km_core_context_item` array. +##### Return: +The number of items in the list, not including terminating item, +or 0 if `context_items` is null. +##### Parameters: +- __context_items__: A pointer to a `KM_CORE_CT_END` terminated array of + `km_core_context_item` values. + +```c +*/ +KMN_API +size_t +km_core_context_item_list_size(km_core_context_item const *context_items); + +#if defined(__cplusplus) +} // extern "C" +#endif diff --git a/core/meson.build b/core/meson.build index 716faca2dc..e6b1fd8fa1 100644 --- a/core/meson.build +++ b/core/meson.build @@ -29,6 +29,9 @@ python = py.find_installation() # For now, we use KM_CORE_LIBRARY to inject the km::core::kmx namespace defns += ['-DKM_CORE_LIBRARY'] +# See keyman_core_api_context.h +defns += ['-D_KM_CORE_ACCESS_PRIVATE_CONTEXT_API'] + # #define DEBUG when we are on a debug build if get_option('buildtype') == 'debug' add_global_arguments('-DDEBUG', language : 'cpp') diff --git a/core/src/action.cpp b/core/src/action.cpp index f4565844d7..d2e0613b14 100644 --- a/core/src/action.cpp +++ b/core/src/action.cpp @@ -11,7 +11,6 @@ #include #include -#include #include "action.hpp" #include "state.hpp" diff --git a/core/src/action.hpp b/core/src/action.hpp index 06038c3122..d8ee401e97 100644 --- a/core/src/action.hpp +++ b/core/src/action.hpp @@ -8,7 +8,7 @@ #pragma once -#include +#include "keyman_core.h" #include namespace km { diff --git a/core/src/context.hpp b/core/src/context.hpp index dba7bec060..36831f6d14 100644 --- a/core/src/context.hpp +++ b/core/src/context.hpp @@ -9,7 +9,7 @@ #pragma once #include #include -#include +#include "keyman_core.h" // Forward declarations class json; diff --git a/core/src/debug.hpp b/core/src/debug.hpp index a68d9f1e57..33de3dd324 100644 --- a/core/src/debug.hpp +++ b/core/src/debug.hpp @@ -9,8 +9,7 @@ #include #include -#include -#include +#include "keyman_core.h" namespace km { namespace core diff --git a/core/src/keyboard.hpp b/core/src/keyboard.hpp index 6dd8517ce1..142bf40e86 100644 --- a/core/src/keyboard.hpp +++ b/core/src/keyboard.hpp @@ -11,7 +11,7 @@ #include #include -#include +#include "keyman_core.h" #include "option.hpp" #include "path.hpp" diff --git a/core/src/keyman_core.h b/core/src/keyman_core.h new file mode 100644 index 0000000000..8750b7b93e --- /dev/null +++ b/core/src/keyman_core.h @@ -0,0 +1,5 @@ +#pragma once + +#include +#include +#include \ No newline at end of file diff --git a/core/src/km_core_action_api.cpp b/core/src/km_core_action_api.cpp index 1177ec71fc..db8a5c8820 100644 --- a/core/src/km_core_action_api.cpp +++ b/core/src/km_core_action_api.cpp @@ -10,7 +10,8 @@ #include #include -#include +#include "keyman_core.h" + #include "jsonpp.hpp" #include "processor.hpp" diff --git a/core/src/km_core_context_api.cpp b/core/src/km_core_context_api.cpp index 46522710c0..ddf49e21e1 100644 --- a/core/src/km_core_context_api.cpp +++ b/core/src/km_core_context_api.cpp @@ -12,7 +12,7 @@ #include #include -#include +#include "keyman_core.h" #include "context.hpp" #include "jsonpp.hpp" diff --git a/core/src/km_core_debug_api.cpp b/core/src/km_core_debug_api.cpp index 2c6433a522..75e34b0b0c 100644 --- a/core/src/km_core_debug_api.cpp +++ b/core/src/km_core_debug_api.cpp @@ -8,7 +8,7 @@ #include #include -#include +#include "keyman_core.h" #include "processor.hpp" #include "state.hpp" diff --git a/core/src/km_core_keyboard_api.cpp b/core/src/km_core_keyboard_api.cpp index ae825ca402..964f73ca07 100644 --- a/core/src/km_core_keyboard_api.cpp +++ b/core/src/km_core_keyboard_api.cpp @@ -11,7 +11,8 @@ #include #include -#include +#include "keyman_core.h" + #include "keyboard.hpp" #include "processor.hpp" #include "kmx/kmx_processor.hpp" diff --git a/core/src/km_core_options_api.cpp b/core/src/km_core_options_api.cpp index 3cb0745fa9..4ea5c74062 100644 --- a/core/src/km_core_options_api.cpp +++ b/core/src/km_core_options_api.cpp @@ -11,7 +11,8 @@ #include #include -#include +#include "keyman_core.h" + #include "processor.hpp" #include "jsonpp.hpp" diff --git a/core/src/km_core_processevent_api.cpp b/core/src/km_core_processevent_api.cpp index 5295d39af7..7d5109a6b2 100644 --- a/core/src/km_core_processevent_api.cpp +++ b/core/src/km_core_processevent_api.cpp @@ -8,7 +8,8 @@ History: 17 Oct 2018 - TSE - Initial implementation. */ -#include +#include "keyman_core.h" + #include "processor.hpp" #include "state.hpp" diff --git a/core/src/km_core_state_api.cpp b/core/src/km_core_state_api.cpp index eb430ba5ac..7b7e74f634 100644 --- a/core/src/km_core_state_api.cpp +++ b/core/src/km_core_state_api.cpp @@ -15,7 +15,7 @@ #include #include -#include +#include "keyman_core.h" #include "jsonpp.hpp" #include "processor.hpp" diff --git a/core/src/km_core_state_context_set_if_needed.cpp b/core/src/km_core_state_context_set_if_needed.cpp index b79044455e..72b80f6f51 100644 --- a/core/src/km_core_state_context_set_if_needed.cpp +++ b/core/src/km_core_state_context_set_if_needed.cpp @@ -9,7 +9,7 @@ */ #include -#include +#include "keyman_core.h" #include "processor.hpp" #include "state.hpp" diff --git a/core/src/kmx/kmx_base.h b/core/src/kmx/kmx_base.h index fed03bc645..e798ff56b7 100644 --- a/core/src/kmx/kmx_base.h +++ b/core/src/kmx/kmx_base.h @@ -1,7 +1,6 @@ #pragma once -#include -#include +#include "keyman_core.h" #if defined(_WIN32) || defined(_WIN64) #define snprintf _snprintf diff --git a/core/src/kmx/kmx_options.h b/core/src/kmx/kmx_options.h index 14c280e411..cf038ac7dc 100644 --- a/core/src/kmx/kmx_options.h +++ b/core/src/kmx/kmx_options.h @@ -4,7 +4,7 @@ #include #include -#include +#include "keyman_core.h" #include "option.hpp" #include "kmx_base.h" diff --git a/core/src/kmx/kmx_processor.cpp b/core/src/kmx/kmx_processor.cpp index 1fa0734e07..95a9a09456 100644 --- a/core/src/kmx/kmx_processor.cpp +++ b/core/src/kmx/kmx_processor.cpp @@ -1,4 +1,4 @@ -#include +#include "keyman_core.h" #include "state.hpp" #include "kmx/kmx_processor.hpp" #include diff --git a/core/src/kmx/kmx_processor.hpp b/core/src/kmx/kmx_processor.hpp index 490bda53c6..08b2c1d242 100644 --- a/core/src/kmx/kmx_processor.hpp +++ b/core/src/kmx/kmx_processor.hpp @@ -9,7 +9,7 @@ #pragma once #include -#include +#include "keyman_core.h" #include "kmx/kmx_processevent.h" #include "keyboard.hpp" #include "processor.hpp" diff --git a/core/src/ldml/ldml_processor.hpp b/core/src/ldml/ldml_processor.hpp index a66d151fde..238b332215 100644 --- a/core/src/ldml/ldml_processor.hpp +++ b/core/src/ldml/ldml_processor.hpp @@ -13,7 +13,7 @@ #include #include #include -#include +#include "keyman_core.h" #include "processor.hpp" #include "option.hpp" #include "ldml_vkeys.hpp" diff --git a/core/src/ldml/ldml_vkeys.hpp b/core/src/ldml/ldml_vkeys.hpp index 22dea817ef..ed3368dc7e 100644 --- a/core/src/ldml/ldml_vkeys.hpp +++ b/core/src/ldml/ldml_vkeys.hpp @@ -13,7 +13,7 @@ #include #include -#include +#include "keyman_core.h" namespace km { namespace core { diff --git a/core/src/mock/mock_processor.hpp b/core/src/mock/mock_processor.hpp index a3ec3c1caa..87a99a49df 100644 --- a/core/src/mock/mock_processor.hpp +++ b/core/src/mock/mock_processor.hpp @@ -10,7 +10,7 @@ #include #include -#include +#include "keyman_core.h" #include "processor.hpp" #include "option.hpp" diff --git a/core/src/option.hpp b/core/src/option.hpp index 43f3397049..60026798cf 100644 --- a/core/src/option.hpp +++ b/core/src/option.hpp @@ -12,7 +12,8 @@ #include -#include +#include "keyman_core.h" + // Forward declarations class json; diff --git a/core/src/path.hpp b/core/src/path.hpp index be6ae61422..aee70ad8ba 100644 --- a/core/src/path.hpp +++ b/core/src/path.hpp @@ -12,7 +12,8 @@ #include #include -#include +#include "keyman_core.h" + #include "jsonpp.hpp" #include "utfcodec.hpp" diff --git a/core/src/processor.hpp b/core/src/processor.hpp index 7a6ed5aa33..8925ad65d1 100644 --- a/core/src/processor.hpp +++ b/core/src/processor.hpp @@ -11,7 +11,7 @@ #include #include -#include +#include "keyman_core.h" #include "keyboard.hpp" diff --git a/core/src/state.hpp b/core/src/state.hpp index 0b98cc9e24..c3afff5029 100644 --- a/core/src/state.hpp +++ b/core/src/state.hpp @@ -11,7 +11,7 @@ #include #include -#include +#include "keyman_core.h" #include "context.hpp" #include "option.hpp" diff --git a/core/tests/unit/kmnkbd/action_api.cpp b/core/tests/unit/kmnkbd/action_api.cpp index 6a84eafa2d..a9e0058b56 100644 --- a/core/tests/unit/kmnkbd/action_api.cpp +++ b/core/tests/unit/kmnkbd/action_api.cpp @@ -6,7 +6,7 @@ History: 23 Oct 2023 - MCD - Initial implementation. */ #include -#include +#include "keyman_core.h" #include "path.hpp" #include "action.hpp" diff --git a/core/tests/unit/kmnkbd/action_items.hpp b/core/tests/unit/kmnkbd/action_items.hpp index 5b4723fd57..dc5ac666b6 100644 --- a/core/tests/unit/kmnkbd/action_items.hpp +++ b/core/tests/unit/kmnkbd/action_items.hpp @@ -1,4 +1,4 @@ -#include +#include "keyman_core.h" #include #include diff --git a/core/tests/unit/kmnkbd/action_set_api.cpp b/core/tests/unit/kmnkbd/action_set_api.cpp index d3c06cc9c7..2f69f2baba 100644 --- a/core/tests/unit/kmnkbd/action_set_api.cpp +++ b/core/tests/unit/kmnkbd/action_set_api.cpp @@ -6,7 +6,8 @@ History: 23 Oct 2023 - MCD - Initial implementation. */ #include -#include + +#include "keyman_core.h" #include "path.hpp" #include "action.hpp" diff --git a/core/tests/unit/kmnkbd/context_api.cpp b/core/tests/unit/kmnkbd/context_api.cpp index d5225eb24f..149ebf9556 100644 --- a/core/tests/unit/kmnkbd/context_api.cpp +++ b/core/tests/unit/kmnkbd/context_api.cpp @@ -10,7 +10,8 @@ mutation functions. */ #include -#include + +#include "keyman_core.h" #include "context.hpp" #include "utfcodec.hpp" diff --git a/core/tests/unit/kmnkbd/debug_api.cpp b/core/tests/unit/kmnkbd/debug_api.cpp index 502f95cc47..3313888477 100644 --- a/core/tests/unit/kmnkbd/debug_api.cpp +++ b/core/tests/unit/kmnkbd/debug_api.cpp @@ -10,7 +10,9 @@ #include #include #include -#include + +#include "keyman_core.h" + #include "path.hpp" #include "state.hpp" #include "kmx/kmx_base.h" diff --git a/core/tests/unit/kmnkbd/debug_items.hpp b/core/tests/unit/kmnkbd/debug_items.hpp index 400f499bd2..ae5f0e8cbd 100644 --- a/core/tests/unit/kmnkbd/debug_items.hpp +++ b/core/tests/unit/kmnkbd/debug_items.hpp @@ -1,6 +1,5 @@ #include -#include -#include +#include "keyman_core.h" #include "kmx/kmx_base.h" #include "kmx/kmx_xstring.h" diff --git a/core/tests/unit/kmnkbd/keyboard_api.cpp b/core/tests/unit/kmnkbd/keyboard_api.cpp index 95365e901c..bf232d69af 100644 --- a/core/tests/unit/kmnkbd/keyboard_api.cpp +++ b/core/tests/unit/kmnkbd/keyboard_api.cpp @@ -6,7 +6,7 @@ */ #include -#include +#include "keyman_core.h" #include "path.hpp" //#include "keyboard.hpp" diff --git a/core/tests/unit/kmnkbd/options_api.cpp b/core/tests/unit/kmnkbd/options_api.cpp index a831180f41..7c04478fe7 100644 --- a/core/tests/unit/kmnkbd/options_api.cpp +++ b/core/tests/unit/kmnkbd/options_api.cpp @@ -7,7 +7,7 @@ #include #include -#include +#include "keyman_core.h" #include "option.hpp" #include "state.hpp" diff --git a/core/tests/unit/kmnkbd/state_api.cpp b/core/tests/unit/kmnkbd/state_api.cpp index e3d1717589..07acd8bdd4 100644 --- a/core/tests/unit/kmnkbd/state_api.cpp +++ b/core/tests/unit/kmnkbd/state_api.cpp @@ -9,7 +9,7 @@ #include #include -#include +#include "keyman_core.h" #include "path.hpp" #include "state.hpp" diff --git a/core/tests/unit/kmx/kmx.cpp b/core/tests/unit/kmx/kmx.cpp index 794c744a7c..1e94f4cf86 100644 --- a/core/tests/unit/kmx/kmx.cpp +++ b/core/tests/unit/kmx/kmx.cpp @@ -17,6 +17,8 @@ #include #include +#include "keyman_core.h" + #include #include diff --git a/core/tests/unit/ldml/ldml.cpp b/core/tests/unit/ldml/ldml.cpp index 453ef8ac8e..db95141eba 100644 --- a/core/tests/unit/ldml/ldml.cpp +++ b/core/tests/unit/ldml/ldml.cpp @@ -24,6 +24,8 @@ #include #include +#include "keyman_core.h" + #include // for surrogate pair macros #include "ldml_test_source.hpp" diff --git a/core/tests/unit/ldml/ldml_test_utils.hpp b/core/tests/unit/ldml/ldml_test_utils.hpp index 1d876365bb..ec180200ed 100644 --- a/core/tests/unit/ldml/ldml_test_utils.hpp +++ b/core/tests/unit/ldml/ldml_test_utils.hpp @@ -13,7 +13,8 @@ #include // for char to vk mapping tables #include // for surrogate pair macros #include -#include + +#include "keyman_core.h" namespace km { namespace tests { diff --git a/core/tests/unit/ldml/test_context_normalization.cpp b/core/tests/unit/ldml/test_context_normalization.cpp index c3215ce4a3..11a3b8ca0e 100644 --- a/core/tests/unit/ldml/test_context_normalization.cpp +++ b/core/tests/unit/ldml/test_context_normalization.cpp @@ -6,7 +6,7 @@ History: 15 Jan 2024 - MCD - Initial implementation. */ #include -#include +#include "keyman_core.h" #include "path.hpp" #include "action.hpp" diff --git a/windows/src/engine/keyman32/keymanengine.h b/windows/src/engine/keyman32/keymanengine.h index c0db193fdf..9befcb302a 100644 --- a/windows/src/engine/keyman32/keymanengine.h +++ b/windows/src/engine/keyman32/keymanengine.h @@ -40,6 +40,7 @@ #include #include "../../../../common/windows/cpp/include/legacy_kmx_file.h" #include +#include // for intermediate context #include /***************************************************************************/ From ba1f5921cfbca87d7e5920006533bcadc32d39f3 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Tue, 16 Jan 2024 12:18:29 +0700 Subject: [PATCH 24/45] chore(mac): remove legacy Core action items references Relates to #9999. --- .../CoreWrapper/CoreWrapper.m | 85 ------------------- 1 file changed, 85 deletions(-) diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m index 8472d70d39..ac011ce35e 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m @@ -225,91 +225,6 @@ const int CORE_ENVIRONMENT_ARRAY_LENGTH = 6; return capsLock; } --(NSArray*)loadActionsUsingCore { - size_t actionCount = 0; - km_core_action_item const * actionList = - km_core_state_action_items(self.coreState, &actionCount); - - NSMutableArray *eventArray = [NSMutableArray arrayWithCapacity:actionCount]; - - for (int i = 0; i < actionCount; i++) { - km_core_action_item action = actionList[i]; - CoreAction *coreAction = [self createCoreActionForActionStruct:&action]; - [eventArray insertObject:coreAction atIndex:i]; - } - - return eventArray; -} - --(CoreAction*)createCoreActionForActionStruct:(km_core_action_item*)actionStruct { - CoreAction* action = nil; - switch (actionStruct->type) - { - case KM_CORE_IT_END: { - action = [[CoreAction alloc] initWithType: EndAction actionContent:@"" backspaceCount:0 key:@"" value:@"" scope:0]; - break; - } - case KM_CORE_IT_CHAR: { - NSString *characterString = [self.coreHelper utf32ValueToString:actionStruct->character]; - action = [[CoreAction alloc] initWithType: CharacterAction actionContent:characterString backspaceCount:0 key:@"" value:@"" scope:0]; - [self.coreHelper logDebugMessage:@"createCoreActionForActionStruct actionStruct->character decimal: %u, hex: %X", actionStruct->character, actionStruct->character]; - [self.coreHelper logDebugMessage:@"createCoreActionForActionStruct converted unicode string: '%@' length=%lu", characterString, characterString.length]; - break; - } - case KM_CORE_IT_MARKER: { - action = [[CoreAction alloc] initWithType: MarkerAction actionContent:@"" backspaceCount:0 key:@"" value:@"" scope:0]; - break; - } - case KM_CORE_IT_ALERT: { - action = [[CoreAction alloc] initWithType: AlertAction actionContent:@"" backspaceCount:0 key:@"" value:@"" scope:0]; - break; - } - case KM_CORE_IT_BACK: { - km_core_backspace_item backspace = actionStruct->backspace; - - if (backspace.expected_type == KM_CORE_BT_CHAR) { - NSString *charString = [self.coreHelper utf32ValueToString:backspace.expected_value]; - [self.coreHelper logDebugMessage:@"createCoreActionForActionStruct charString = %@", charString]; - action = [[CoreAction alloc] initCharacterBackspaceAction:charString]; - [self.coreHelper logDebugMessage:@"createCoreActionForActionStruct converted character backspace, expected value =%lu, expected type =%u", backspace.expected_value, backspace.expected_type]; - } else if(backspace.expected_type == KM_CORE_BT_MARKER) { - action = [[CoreAction alloc] initMarkerBackspaceAction:actionStruct->backspace.expected_value]; - [self.coreHelper logDebugMessage:@"createCoreActionForActionStruct converted marker backspace, expected value =%lu, expected type =%u", backspace.expected_value, backspace.expected_type]; - } else { - [self.coreHelper logDebugMessage:@"createCoreActionForActionStruct did not convert unknown backspace, expected value =%lu, expected type =%u", backspace.expected_value, backspace.expected_type]; - } - break; - } - case KM_CORE_IT_PERSIST_OPT: { - [self.coreHelper logDebugMessage:@"***createCoreActionForActionStruct Persist Options encountered."]; - km_core_option_item const * option = actionStruct->option; - NSString *keyString = [self.coreHelper createNSStringFromUnicharString:option->key]; - NSString *valueString = [self.coreHelper createNSStringFromUnicharString:option->value]; - - [self.coreHelper logDebugMessage:@"***createCoreActionForActionStruct converted Persist Options, key = %@, value = %@, scope = %d", keyString, valueString, option->scope]; - - action = [[CoreAction alloc] initPersistOptionAction:keyString value:valueString scope:option->scope]; - break; - } - case KM_CORE_IT_EMIT_KEYSTROKE: { - action = [[CoreAction alloc] initWithType: EmitKeystrokeAction actionContent:@"" backspaceCount:0 key:@"" value:@"" scope:0]; - break; - } - case KM_CORE_IT_INVALIDATE_CONTEXT: { - action = [[CoreAction alloc] initWithType: InvalidateContextAction actionContent:@"" backspaceCount:0 key:@"" value:@"" scope:0]; - break; - } - case KM_CORE_IT_CAPSLOCK: { - action = [[CoreAction alloc] initWithType: CapsLockAction actionContent:@"" backspaceCount:0 key:@"" value:@"" scope:0]; - break; - } - default: { - NSLog(@"createCoreActionForActionStruct unrecognized type of km_core_action_item = %u\n", actionStruct->type); - } - } - return action; -} - -(void)clearContextUsingCore { km_core_state_context_clear(self.coreState); [self.coreHelper logDebugMessage:@"km_core_state_context_clear called"]; From e39affe93bf9a8a4a2159f8b68363adb86c7eb78 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Tue, 16 Jan 2024 13:46:46 +0700 Subject: [PATCH 25/45] feat(core): infrastructure for normalization of output Relates to #9999. Establishes functions, unit test sources, normalization entry point and an effectively no-op unit test for normalization support. --- core/include/keyman/keyman_core_api_context.h | 4 +- core/src/action.cpp | 2 +- core/src/action.hpp | 8 +- core/src/actions_normalize.cpp | 34 ++++++ core/src/km_core_action_api.cpp | 11 +- core/src/km_core_state_api.cpp | 8 +- core/src/meson.build | 1 + core/tests/unit/kmnkbd/meson.build | 1 + .../unit/kmnkbd/test_actions_normalize.cpp | 112 ++++++++++++++++++ 9 files changed, 172 insertions(+), 9 deletions(-) create mode 100644 core/src/actions_normalize.cpp create mode 100644 core/tests/unit/kmnkbd/test_actions_normalize.cpp diff --git a/core/include/keyman/keyman_core_api_context.h b/core/include/keyman/keyman_core_api_context.h index 26bd2f25bc..02bd751150 100644 --- a/core/include/keyman/keyman_core_api_context.h +++ b/core/include/keyman/keyman_core_api_context.h @@ -116,7 +116,7 @@ km_core_context_items_dispose(km_core_context_item *context_items); */ KMN_API km_core_context * -km_core_state_context(km_core_state *state); +km_core_state_context(km_core_state const *state); /** * Get access to the state object's application context. @@ -127,7 +127,7 @@ km_core_state_context(km_core_state *state); */ KMN_API km_core_context * -km_core_state_app_context(km_core_state *state); +km_core_state_app_context(km_core_state const *state); /* ``` diff --git a/core/src/action.cpp b/core/src/action.cpp index d2e0613b14..f958ca3dbe 100644 --- a/core/src/action.cpp +++ b/core/src/action.cpp @@ -16,7 +16,7 @@ #include "state.hpp" #include "option.hpp" -km_core_actions const * km::core::action_item_list_to_actions_object( +km_core_actions * km::core::action_item_list_to_actions_object( km_core_action_item const *action_items ) { assert(action_items != nullptr); diff --git a/core/src/action.hpp b/core/src/action.hpp index d8ee401e97..0d6844474f 100644 --- a/core/src/action.hpp +++ b/core/src/action.hpp @@ -14,8 +14,14 @@ namespace km { namespace core { - km_core_actions const *action_item_list_to_actions_object( + km_core_actions *action_item_list_to_actions_object( km_core_action_item const *action_items ); + + bool actions_normalize( + km_core_context const *cached_context, + km_core_context const *app_context, + km_core_actions *actions + ); } // namespace core } // namespace km diff --git a/core/src/actions_normalize.cpp b/core/src/actions_normalize.cpp new file mode 100644 index 0000000000..896350fc3a --- /dev/null +++ b/core/src/actions_normalize.cpp @@ -0,0 +1,34 @@ +/* + Copyright: © 2024 SIL International. + Description: Implementation of the action output normalization. + Create Date: 16 Jan 2024 + Authors: Marc Durdin (MCD) + History: 16 Jan 2024 - MCD - Initial implementation from #9999 +*/ +#include +#include +#include +#include + +#include "action.hpp" +#include "state.hpp" +#include "option.hpp" + +bool km::core::actions_normalize( + km_core_context const *cached_context, + km_core_context const *app_context, + km_core_actions *actions +) { + assert(actions != nullptr); + assert(cached_context != nullptr); + assert(app_context != nullptr); + if(actions == nullptr || cached_context == nullptr || app_context == nullptr) { + return false; + } + + // Normalize output to NFC + //TODO + // actions->code_points_to_delete++; + + return true; +} diff --git a/core/src/km_core_action_api.cpp b/core/src/km_core_action_api.cpp index db8a5c8820..4d9a57c371 100644 --- a/core/src/km_core_action_api.cpp +++ b/core/src/km_core_action_api.cpp @@ -33,7 +33,16 @@ km_core_actions const * km_core_state_get_actions( return nullptr; } - return action_item_list_to_actions_object(action_items); + km_core_actions * result = action_item_list_to_actions_object(action_items); + + if(state->processor().supports_normalization()) { + // Normalize to NFC for those keyboard processors that support it + if(!actions_normalize(km_core_state_context(state), km_core_state_app_context(state), result)) { + km_core_actions_dispose(result); + return nullptr; + } + } + return result; } km_core_status km_core_actions_dispose( diff --git a/core/src/km_core_state_api.cpp b/core/src/km_core_state_api.cpp index 7b7e74f634..03e041322d 100644 --- a/core/src/km_core_state_api.cpp +++ b/core/src/km_core_state_api.cpp @@ -65,20 +65,20 @@ void km_core_state_dispose(km_core_state *state) } -km_core_context *km_core_state_context(km_core_state *state) +km_core_context *km_core_state_context(km_core_state const *state) { assert(state); if (!state) return nullptr; - return static_cast(&state->context()); + return static_cast(&(const_cast(state)->context())); } -km_core_context *km_core_state_app_context(km_core_state *state) +km_core_context *km_core_state_app_context(km_core_state const *state) { assert(state); if (!state) return nullptr; - return static_cast(&state->app_context()); + return static_cast(&(const_cast(state)->app_context())); } km_core_status km_core_state_get_intermediate_context( diff --git a/core/src/meson.build b/core/src/meson.build index a80eaaa94a..6f96041dac 100644 --- a/core/src/meson.build +++ b/core/src/meson.build @@ -42,6 +42,7 @@ endif kmx_files = files( + 'actions_normalize.cpp', 'action.cpp', 'option.cpp', 'keyboard.cpp', diff --git a/core/tests/unit/kmnkbd/meson.build b/core/tests/unit/kmnkbd/meson.build index 703374bdf6..16f0c87e8d 100644 --- a/core/tests/unit/kmnkbd/meson.build +++ b/core/tests/unit/kmnkbd/meson.build @@ -26,6 +26,7 @@ tests = [ ['debug-api', 'debug_api.cpp'], ['kmx_xstring', 'test_kmx_xstring.cpp'], ['kmx_context', 'test_kmx_context.cpp'], + ['test_actions_normalize', 'test_actions_normalize.cpp'], ] test_path = join_paths(meson.current_build_dir(), '..', 'kmx') diff --git a/core/tests/unit/kmnkbd/test_actions_normalize.cpp b/core/tests/unit/kmnkbd/test_actions_normalize.cpp new file mode 100644 index 0000000000..95d300ef0c --- /dev/null +++ b/core/tests/unit/kmnkbd/test_actions_normalize.cpp @@ -0,0 +1,112 @@ +/* + Copyright: © 2018 SIL International. + Description: Tests for the context API family of functions. + Create Date: 23 Oct 2023 + Authors: Marc Durdin + History: 23 Oct 2023 - MCD - Initial implementation. +*/ +#include +#include "keyman_core.h" + +#include "path.hpp" +#include "action.hpp" + +#include +#include "../emscripten_filesystem.h" + +km_core_option_item test_env_opts[] = +{ + KM_CORE_OPTIONS_END +}; + +km_core_keyboard * test_kb = nullptr; +km_core_state * test_state = nullptr; +km_core_actions * test_actions = nullptr; +std::string arg_path; + +void teardown() { + if(test_state) { + km_core_state_dispose(test_state); + test_state = nullptr; + } + if(test_kb) { + km_core_keyboard_dispose(test_kb); + test_kb = nullptr; + } + if(test_actions) { + delete [] test_actions->output; + delete test_actions; + test_actions = nullptr; + } +} + +void setup(const km_core_cp *context, int actions_code_points_to_delete, const std::u32string actions_output) { + teardown(); + + km::core::path path = km::core::path::join(arg_path, "..", "ldml", "keyboards", "k_001_tiny.kmx"); + try_status(km_core_keyboard_load(path.native().c_str(), &test_kb)); + try_status(km_core_state_create(test_kb, test_env_opts, &test_state)); + + assert(km_core_state_context_set_if_needed(test_state, context) == KM_CORE_CONTEXT_STATUS_UPDATED); + + test_actions = new km_core_actions; + test_actions->code_points_to_delete = actions_code_points_to_delete; + test_actions->output = new km_core_usv[actions_output.length() + 1]; + actions_output.copy(test_actions->output, actions_output.length()); + test_actions->output[actions_output.length()] = 0; +} + +//------------------------------------------------------------------------------------- + +void test_no_normalization() { + setup(u"abc", 0, U"def"); + const int expected_delete = 0; + const std::u32string expected_output = U"def"; + + assert(km::core::actions_normalize(km_core_state_context(test_state), km_core_state_app_context(test_state), test_actions)); + + assert(expected_delete == test_actions->code_points_to_delete); + assert(expected_output == test_actions->output); + + teardown(); +} + +//------------------------------------------------------------------------------------- +// Launcher +//------------------------------------------------------------------------------------- + +constexpr const auto help_str = "\ +test_actions_normalize [--color] \n\ +\n\ + --color Force color output\n\ + BUILD_PATH Path where test_actions_normalize.exe is found; kmx files are\n\ + located relative to this path.\n"; + +int error_args() { + std::cerr << "test_actions_normalize: Invalid arguments." << std::endl; + std::cout << help_str; + return 1; +} + +int main(int argc, char *argv []) { + + if(argc < 2) { + return error_args(); + } + + auto arg_color = std::string(argv[1]) == "--color"; + if(arg_color && argc < 3) { + return error_args(); + } + console_color::enabled = console_color::isaterminal() || arg_color; + +#ifdef __EMSCRIPTEN__ + arg_path = get_wasm_file_path(argv[arg_color ? 2 : 1]); +#else + arg_path = argv[arg_color ? 2 : 1]; +#endif + + // actions + test_no_normalization(); +} + From 2f9a7de87ca4c62fd65458e88ea9fa6fb3cb01a3 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Tue, 16 Jan 2024 09:14:02 +0100 Subject: [PATCH 26/45] chore(core): Update symbols file --- linux/debian/libkeymancore.symbols | 1 + 1 file changed, 1 insertion(+) diff --git a/linux/debian/libkeymancore.symbols b/linux/debian/libkeymancore.symbols index 5b46e97776..7e4522850d 100644 --- a/linux/debian/libkeymancore.symbols +++ b/linux/debian/libkeymancore.symbols @@ -30,6 +30,7 @@ libkeymancore.so.1 libkeymancore #MINVER# km_core_process_event@Base 17.0.195 km_core_process_queued_actions@Base 17.0.195 km_core_state_action_items@Base 17.0.195 + km_core_state_app_context@Base 17.0.245 km_core_state_clone@Base 17.0.195 km_core_state_context@Base 17.0.195 km_core_state_context_clear@Base 17.0.197 From e6096a267c5212701a8068db024817983d5b5b0a Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 17 Jan 2024 10:41:00 +0700 Subject: [PATCH 27/45] feat(core): action output normalization Fixes #9999. Note TODO items: - [ ] Renormalize cached_context across action boundary. Blocked by #10369. - [ ] Add extra tests for surrogate pairs - [ ] Move set_context_from_string into helper module - [ ] if we don't apply normalization, we still need to fixup the app_context, to keep it coherent with cached_context (or at least we need to verify that app_context is never used in this situation) --- core/src/action.hpp | 6 +- core/src/actions_normalize.cpp | 265 +++++++++++++++++- .../unit/kmnkbd/test_actions_normalize.cpp | 222 ++++++++++++++- 3 files changed, 475 insertions(+), 18 deletions(-) diff --git a/core/src/action.hpp b/core/src/action.hpp index 0d6844474f..40b9316a64 100644 --- a/core/src/action.hpp +++ b/core/src/action.hpp @@ -19,9 +19,9 @@ namespace core ); bool actions_normalize( - km_core_context const *cached_context, - km_core_context const *app_context, - km_core_actions *actions + /* in */ km_core_context const *cached_context, + /* in, out */ km_core_context *app_context, + /* in, out */ km_core_actions *actions ); } // namespace core } // namespace km diff --git a/core/src/actions_normalize.cpp b/core/src/actions_normalize.cpp index 896350fc3a..09cd857d80 100644 --- a/core/src/actions_normalize.cpp +++ b/core/src/actions_normalize.cpp @@ -5,19 +5,52 @@ Authors: Marc Durdin (MCD) History: 16 Jan 2024 - MCD - Initial implementation from #9999 */ -#include #include #include #include +#include +#include "context.hpp" #include "action.hpp" #include "state.hpp" #include "option.hpp" +#include "debuglog.h" +#if !defined(HAVE_ICU4C) +#error icu4c is required for this code +#endif + +#define U_FALLTHROUGH +#include "unicode/utypes.h" +#include "unicode/unistr.h" +#include "unicode/normalizer2.h" + +// ******************************************************************************** +// TODO: if we don't apply normalization, we should still fixup the app_context +// ******************************************************************************** + +// forward declarations + +icu::UnicodeString context_items_to_unicode_string(km_core_context const *context); +km_core_usv *unicode_string_to_usv(icu::UnicodeString& src); + +/** + * Normalize the output from an action to NFC, across the context | output + * boundary, fixing up the app_context and the output actions to take into + * account the NFU input app_context + * + * @param cached_context the cached context, in NFD, after transform has been + * applied to it by the keyboard processor + * @param app_context the app context, in NFU; transform has not been + * applied, and will be applied by this function + * @param actions transform to apply, in NFD, which will be converted + * to NFC by this function + * @return true on success, false on failure + */ bool km::core::actions_normalize( - km_core_context const *cached_context, - km_core_context const *app_context, - km_core_actions *actions + /* in */ km_core_context const *cached_context, + /* in, out */ km_core_context *app_context, + /* in, out */ km_core_actions *actions ) { assert(actions != nullptr); assert(cached_context != nullptr); @@ -26,9 +59,227 @@ bool km::core::actions_normalize( return false; } - // Normalize output to NFC - //TODO - // actions->code_points_to_delete++; + /* + The code_points_to_delete value at this point is in NFD. The cached_context + is in NFD and has already been updated by the keyboard processor to the + expected result of the action, so we need to remove the output from a copy + of the cached_context to start, in order to get it to the same position as + the app_context. + + The app_context is in NFU. We need to figure out how many characters to + remove from the end of app_context in order to correctly normalize across + the boundary, without normalizing more of the string than necessary. + + We do not need to mutate the cached_context itself, because it is already + correct. This is good, because it means we will not lose track of markers + within it. The app_context will be mutated, as it will need the new output + appended, in order to match the expected result. Remember that the + app_context does not contain markers; these are maintained only in the + cached_context. + */ + + // TODO: That description above is not quite right. cached_context is not + // guaranteed to be normalized across the transform boundary, because its + // output was simply appended to the existing context. MUCH PAIN COMING WITH + // THIS, because cached_context includes markers. Hence, blocked by #10369. + + /* + Initialization + */ + + UErrorCode icu_status = U_ZERO_ERROR; + const icu::Normalizer2 *nfc = icu::Normalizer2::getNFCInstance(icu_status); + assert(U_SUCCESS(icu_status)); + if(!U_SUCCESS(icu_status)) { + DebugLog("getNFCInstance failed with %x", icu_status); + return false; + } + + const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(icu_status); + assert(U_SUCCESS(icu_status)); + if(!U_SUCCESS(icu_status)) { + DebugLog("getNFDInstance failed with %x", icu_status); + return false; + } + + icu::UnicodeString output = icu::UnicodeString::fromUTF32(reinterpret_cast(actions->output), -1); + icu::UnicodeString cached_context_string = context_items_to_unicode_string(cached_context); + icu::UnicodeString app_context_string = context_items_to_unicode_string(app_context); + assert(!output.isBogus()); + assert(!cached_context_string.isBogus()); + assert(!app_context_string.isBogus()); + if(output.isBogus() || cached_context_string.isBogus() || app_context_string.isBogus()) { + return false; + } + int nfu_to_delete = 0; + + /* + The keyboard processor will have updated the cached_context already, + applying the transform to it, so we need to rewind this. Remove the output + from cached_context_string to start + */ + + assert(cached_context_string.length() >= output.length()); + int n = cached_context_string.length() - output.length(); + assert(cached_context_string.compare(n, output.length(), output) == 0); + cached_context_string.remove(n); + + /* + Now, look for a normalization boundary at the intersection of the + cached_context and the output + */ + + while(n > 0 && output[0] && !nfd->hasBoundaryBefore(output[0])) { + // The output may interact with the context further in normalization. We + // need to copy characters back further until we reach a normalization + // boundary. + + // Remove last code point from the context ... + + n = cached_context_string.moveIndex32(n, -1); + UChar32 chr = cached_context_string.char32At(n); + cached_context_string.remove(n); + + // And prepend it to the output ... + + output.insert(0, chr); + + // And finally remember that we now need to delete an additional NFD codepoint + + actions->code_points_to_delete++; + } + + /* + At this point, our output and cached_context are coherent and normalization + will be complete at the edit boundary. + + Now, we need to adjust the delete_back to match the number of characters + that must actually be deleted from the applications's NFU context + + To adjust, we remove one character at a time from the app_context until + its normalized form matches the cached_context normalized form. + */ + + + while(app_context_string.length()) { + icu::UnicodeString app_context_nfd; + nfd->normalize(app_context_string, app_context_nfd, icu_status); + assert(U_SUCCESS(icu_status)); + if(!U_SUCCESS(icu_status)) { + DebugLog("nfd->normalize failed with %x", icu_status); + return false; + } + + if(app_context_nfd.compare(cached_context_string) == 0) { + break; + } + app_context_string.remove(app_context_string.length()-1); + nfu_to_delete++; + } + + /* + Normalize our output string + */ + + icu::UnicodeString output_nfc; + nfc->normalize(output, output_nfc, icu_status); + assert(U_SUCCESS(icu_status)); + if(!U_SUCCESS(icu_status)) { + DebugLog("nfc->normalize failed with %x", icu_status); + return false; + } + + auto new_output = unicode_string_to_usv(output_nfc); + if(!new_output) { + // error logging handled in unicode_string_to_usv + return false; + } + + /* + Final steps -- set our outputs + */ + + // Append the new NFC output to our reduced app_context + + app_context_string.append(output_nfc); + km_core_context_item *app_context_items = nullptr; + km_core_status status = KM_CORE_STATUS_OK; + if((status = km_core_context_items_from_utf16(app_context_string.getTerminatedBuffer(), &app_context_items)) != KM_CORE_STATUS_OK) { + DebugLog("km_core_context_items_from_utf16 failed with %x", status); + delete [] new_output; + return false; + } + + if((status = km_core_context_set(app_context, app_context_items)) != KM_CORE_STATUS_OK) { + DebugLog("km_core_context_set failed with %x", status); + km_core_context_items_dispose(app_context_items); + delete [] new_output; + return false; + } + + km_core_context_items_dispose(app_context_items); + + // Update actions with new NFC output + count of NFU code points to delete + + delete [] actions->output; + actions->output = new_output; + actions->code_points_to_delete = nfu_to_delete; return true; } + +/** + * Helper to convert km_core_context list into a icu::UnicodeString + */ +icu::UnicodeString context_items_to_unicode_string(km_core_context const *context) { + icu::UnicodeString nullString; + nullString.setToBogus(); + + km_core_context_item *items = nullptr; + km_core_status status; + if((status = km_core_context_get(context, &items)) != KM_CORE_STATUS_OK) { + DebugLog("Failed to retrieve context with %s", status); + return nullString; + } + size_t buf_size = 0; + if((status = km_core_context_items_to_utf32(items, nullptr, &buf_size)) != KM_CORE_STATUS_OK) { + DebugLog("Failed to retrieve context size with %s", status); + km_core_context_items_dispose(items); + return nullString; + } + + km_core_usv *buf = new km_core_usv[buf_size]; + if((status = km_core_context_items_to_utf32(items, buf, &buf_size)) != KM_CORE_STATUS_OK) { + DebugLog("Failed to retrieve context with %s", status); + km_core_context_items_dispose(items); + delete [] buf; + return nullString; + } + + auto result = icu::UnicodeString::fromUTF32(reinterpret_cast(buf), -1); + km_core_context_items_dispose(items); + delete [] buf; + return result; +} + +/** + * Helper to convert icu::UnicodeString to a UTF-32 km_core_usv buffer, + * nul-terminated + */ +km_core_usv *unicode_string_to_usv(icu::UnicodeString& src) { + UErrorCode icu_status = U_ZERO_ERROR; + + km_core_usv *dst = new km_core_usv[src.length() + 1]; + + src.toUTF32(reinterpret_cast(dst), src.length(), icu_status); + + assert(U_SUCCESS(icu_status)); + if(!U_SUCCESS(icu_status)) { + DebugLog("toUTF32 failed with %x", icu_status); + delete[] dst; + return nullptr; + } + + dst[src.length()] = 0; + return dst; +} diff --git a/core/tests/unit/kmnkbd/test_actions_normalize.cpp b/core/tests/unit/kmnkbd/test_actions_normalize.cpp index 95d300ef0c..21428331af 100644 --- a/core/tests/unit/kmnkbd/test_actions_normalize.cpp +++ b/core/tests/unit/kmnkbd/test_actions_normalize.cpp @@ -14,6 +14,9 @@ #include #include "../emscripten_filesystem.h" +km_core_status set_context_from_string(km_core_context *context, km_core_cp const *new_context); +km_core_cp* get_context_as_string(km_core_context *context); + km_core_option_item test_env_opts[] = { KM_CORE_OPTIONS_END @@ -40,14 +43,17 @@ void teardown() { } } -void setup(const km_core_cp *context, int actions_code_points_to_delete, const std::u32string actions_output) { + + +void setup(const km_core_cp *app_context, const km_core_cp *cached_context, int actions_code_points_to_delete, const std::u32string actions_output) { teardown(); km::core::path path = km::core::path::join(arg_path, "..", "ldml", "keyboards", "k_001_tiny.kmx"); try_status(km_core_keyboard_load(path.native().c_str(), &test_kb)); try_status(km_core_state_create(test_kb, test_env_opts, &test_state)); - assert(km_core_state_context_set_if_needed(test_state, context) == KM_CORE_CONTEXT_STATUS_UPDATED); + try_status(set_context_from_string(km_core_state_context(test_state), cached_context)); + try_status(set_context_from_string(km_core_state_app_context(test_state), app_context)); test_actions = new km_core_actions; test_actions->code_points_to_delete = actions_code_points_to_delete; @@ -58,19 +64,200 @@ void setup(const km_core_cp *context, int actions_code_points_to_delete, const s //------------------------------------------------------------------------------------- -void test_no_normalization() { - setup(u"abc", 0, U"def"); - const int expected_delete = 0; - const std::u32string expected_output = U"def"; +/** + * Run a single test on actions_normalize. This is quite nuanced, because the + * input state is more than a little complex. We have inputs in NFU and NFD, and + * outputs counting NFU and inserting NFC. Be careful! + * + * @param name descriptive name for the test + * @param initial_app_context the app context stored in the state, + * _before_ transform is applied -- NFU + * @param final_cached_context cached context _after_ actions have been + * applied -- NFD except on the boundary of + * the transform, where reorder may be + * required (essentially, this is + * initial_cached_context - + * actions_code_points_to_delete + + * actions_output) + * @param actions_code_points_to_delete number of NFD code points that the + * keyboard processor has asked to remove + * in its actions + * @param actions_output NFD string that the keyboard processor + * has asked to insert in its actions + * @param expected_delete expected: NFU code points to ask app to + * remove + * @param expected_output expected: adjusted NFC output to insert + * into the app + * @param expected_final_app_context expected: NFU adjusted final app + * context, which will be NFC from the + * boundary of the transform, but will not + * have been modified prior to that. Should + * match char-for-char what the app ends up + * with in its text buffer. + */ +void test( + const char *name, + const km_core_cp *initial_app_context, + const km_core_cp *final_cached_context, + int actions_code_points_to_delete, + const std::u32string actions_output, + + const int expected_delete, + const std::u32string expected_output, + const km_core_cp *expected_final_app_context +) { + std::cout << "test: " << name << std::endl; + + setup(initial_app_context, final_cached_context, actions_code_points_to_delete, actions_output); assert(km::core::actions_normalize(km_core_state_context(test_state), km_core_state_app_context(test_state), test_actions)); + std::cout << " (" << name << "): delete: " << expected_delete << " output: |" << std::u32string(test_actions->output) << "|" << std::endl; + std::u32string o(test_actions->output); + for(auto i = o.begin(); i < o.end(); i++) { + std::cout << "U+" << std::hex << (int)(*i) << " "; + } + std::cout << std::endl; + assert(expected_delete == test_actions->code_points_to_delete); assert(expected_output == test_actions->output); + auto actual_final_app_context = get_context_as_string(km_core_state_app_context(test_state)); + auto actual_final_app_context_string = std::u16string(actual_final_app_context); + auto expected_final_app_context_string = std::u16string(final_app_context); + std::cout << " final app context: actual: |" << actual_final_app_context_string << "| expected: |" << expected_final_app_context_string << "|" << std::endl; + assert(actual_final_app_context_string == expected_final_app_context_string); + delete [] actual_final_app_context; + teardown(); } +void run_tests() { + + // Null boundary tests + + test( + "noop", + /* app context pre transform: */ u"", + /* cached context post transform: */ u"", + /* action del, output: */ 0, U"", + // ---- results ---- + /* action del, output: */ 0, U"", + /* app_context: */ u"" + ); + + test( + "no_output", + /* app context pre transform: */ u"abc", + /* cached context post transform: */ u"abc", + /* action del, output: */ 0, U"", + // ---- results ---- + /* action del, output: */ 0, U"", + /* app_context: */ u"abc" + ); + + test( + "no_context", + /* app context pre transform: */ u"", + /* cached context post transform: */ u"def", + /* action del, output: */ 0, U"def", + // ---- results ---- + /* action del, output: */ 0, U"def", + /* app_context: */ u"def" + ); + + // Simple tests -- no deletions involved + + test( + "no_normalization", + /* app context pre transform: */ u"abc", + /* cached context post transform: */ u"abcdef", + /* action del, output: */ 0, U"def", + // ---- results ---- + /* action del, output: */ 0, U"def", + /* app_context: */ u"abcdef" + ); + + test( + "output_to_nfc_basic", + /* app context pre transform: */ u"abc", + /* cached context post transform: */ u"abcde\u0300f", + /* action del, output: */ 0, U"de\u0300f", + // ---- results ---- + /* action del, output: */ 0, U"dèf", + /* app_context: */ u"abcdèf" + ); + + test( + "output_to_nfc_hefty", + /* app context pre transform: */ u"abc", + /* cached context post transform: */ u"abcA\u0300" u"e\u0316\u0301" u"\u0073\u0323\u0307" u"\u0041\u030a" u"\U000114B9\U000114B0", + /* action del, output: */ 0, U"A\u0300" U"e\u0316\u0301" U"\u0073\u0323\u0307" U"\u0041\u030a" U"\U000114B9\U000114B0", + // ---- results ---- + /* action del, output: */ 0, U"À" U"é̖" U"\u1e69" U"\u00c5" U"\U000114BC", + /* app_context: */ u"abcÀé̖\u1e69\u00c5\U000114BC" + ); + + // Interaction with input context when not on normalization boundary + + test( + "Backtrack one character to combine as NFC", + /* app context pre transform: */ u"XYZA", + /* cached context post transform: */ u"XYZA\u0300abc", + /* action del, output: */ 0, U"\u0300abc", + // ---- results ---- + /* action del, output: */ 1, U"Àabc", + /* app_context: */ u"XYZÀabc" + ); + + test( + "Backtrack e+comb circ (2 chars) to combine as NFC", + /* app context pre transform: */ u"abce\u0302", + /* cached context post transform: */ u"abce\u0323\u0302", + /* action del, output: */ 1, U"\u0323\u0302", + // ---- results ---- + /* action del, output: */ 2, U"ệ", + /* app_context: */ u"abcệ" + ); + + test( + "Two backspaces for NFD converts into one char in NFC (ê) and recombine", + /* app context pre transform: */ u"abcê", + /* cached context post transform: */ u"abce\u0323\u0302", + /* action del, output: */ 2, U"e\u0323\u0302", // NFD input; delete 2: e and \u0302 + // ---- results ---- + /* action del, output: */ 1, U"ệ", // NFC output; delete 1: ê + /* app_context: */ u"abcệ" + ); + + // a\u0300 should not be normalized because it is not otherwise impacted by + // the action. + test( + "Avoid editing too far back in context when finding normalization boundary", + /* app context pre transform: */ u"a\u0300bcê", + /* cached context post transform: */ u"a\u0300bce\u0323\u0302", + /* action del, output: */ 2, U"e\u0323\u0302", // NFD input; delete 2: e and \u0302 + // ---- results ---- + /* action del, output: */ 1, U"ệ", // NFC output; delete 1: ê + /* app_context: */ u"a\u0300bcệ" + ); + + // If we don't reach a normalization boundary, we still should continue to work + test( + "Normalizable letters at start of context", + /* app context pre transform: */ u"\u0300", + /* cached context post transform: */ u"\u0300\u0323\u0302", + /* action del, output: */ 0, U"\u0323\u0302", // NFD input; + // ---- results ---- + /* action del, output: */ 1, U"\u0323\u0300\u0302", // NFC output is still decomposed because there is no base + /* app_context: */ u"\u0323\u0300\u0302" + ); + + // TODO: surrogate pair tests + // TODO: add check for updated cached_context once #10369 lands and we apply + // it here +} + //------------------------------------------------------------------------------------- // Launcher //------------------------------------------------------------------------------------- @@ -106,7 +293,26 @@ int main(int argc, char *argv []) { arg_path = argv[arg_color ? 2 : 1]; #endif - // actions - test_no_normalization(); + run_tests(); } + +// TODO: These helpers are copied from km_core_state_context_set_if_needed and +// should be refactored into a helper module + +/** + * Updates the context from the new_context km_core_cp string + */ +km_core_status set_context_from_string(km_core_context *context, km_core_cp const *new_context) { + km_core_context_item* new_context_items = nullptr; + + km_core_status status = km_core_context_items_from_utf16(new_context, &new_context_items); + if (status != KM_CORE_STATUS_OK) { + return status; + } + + km_core_context_set(context, new_context_items); + km_core_context_items_dispose(new_context_items); + + return KM_CORE_STATUS_OK; +} From 823b7722a0e8a2a76335c6a49772d08c26e2c35b Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Tue, 16 Jan 2024 11:12:01 +0700 Subject: [PATCH 28/45] refactor(mac): remove legacy Core context API calls Relates to #9999. --- .../KeymanTests/TestAppDelegate.m | 6 +++--- .../CoreWrapper/CoreWrapper.h | 1 - .../CoreWrapper/CoreWrapper.m | 19 ------------------- .../KeymanEngine4Mac/KME/KMEngine.h | 1 - .../KeymanEngine4Mac/KME/KMEngine.m | 6 +----- .../KeymanEngine4MacTests/CoreWrapperTests.m | 2 +- 6 files changed, 5 insertions(+), 30 deletions(-) diff --git a/mac/Keyman4MacIM/KeymanTests/TestAppDelegate.m b/mac/Keyman4MacIM/KeymanTests/TestAppDelegate.m index a17e1f78da..e646e33905 100644 --- a/mac/Keyman4MacIM/KeymanTests/TestAppDelegate.m +++ b/mac/Keyman4MacIM/KeymanTests/TestAppDelegate.m @@ -18,7 +18,7 @@ if (_kme == nil) { _kme = [[KMEngine alloc] initWithKMX:nil context:self.contextBuffer verboseLogging:self.debugMode]; } - + return _kme; } @@ -31,7 +31,7 @@ if (_contextBuffer == nil) { _contextBuffer = [[NSMutableString alloc] initWithString:@""]; } - + return _contextBuffer; } @@ -39,7 +39,7 @@ _contextBuffer = [contextBuffer mutableCopy]; if (_contextBuffer.length) [_contextBuffer replaceOccurrencesOfString:@"\0" withString:[NSString nullChar] options:0 range:NSMakeRange(0, 1)]; - [self.kme setCoreContext:self.contextBuffer]; + [self.kme setCoreContextIfNeeded:self.contextBuffer]; } - (BOOL)debugMode { diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.h b/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.h index 8cc1c68c9d..1109b1130f 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.h +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.h @@ -28,7 +28,6 @@ NS_ASSUME_NONNULL_BEGIN withModifiers:(NSEventModifierFlags)modifierState withKeyDown:(BOOL)isKeyDown; -(void)setContextIfNeeded:(NSString*)context; --(void)setContext:(NSString*)context; -(NSString*)contextDebug; -(void)clearCoreContext; -(void)dealloc; diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m index d736d80279..8472d70d39 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m @@ -321,25 +321,6 @@ const int CORE_ENVIRONMENT_ARRAY_LENGTH = 6; [self.coreHelper logDebugMessage:@"CoreWrapper setContextIfNeeded, context=%@, km_core_state_context_set_if_needed result=%i", context, result]; } --(void)setContext:(NSString*)context { - if (context.length == 0) { - [self clearContextUsingCore]; - } else { - char const *coreString = [context cStringUsingEncoding:NSUTF8StringEncoding]; - km_core_context_item *contextItemArray; - - // create array of context items - km_core_status result = km_core_context_items_from_utf8(coreString, &contextItemArray); - [self.coreHelper logDebugMessage:@"km_core_context_items_from_utf8, result=%i", result]; - - // set the context in core using the array - km_core_context * coreContext = km_core_state_context(self.coreState); - km_core_context_set(coreContext, contextItemArray); - // dispose - km_core_context_items_dispose(contextItemArray); - } -} - -(NSString*)contextDebug { km_core_cp * context = km_core_state_context_debug(self.coreState, KM_CORE_DEBUG_CONTEXT_CACHED); NSString *debugString = [self.coreHelper createNSStringFromUnicharString:context]; diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.h b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.h index 711c34b9ba..c79b02ba32 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.h +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.h @@ -24,7 +24,6 @@ - (NSString *)getCoreContextDebug; - (void)clearCoreContext; - (void)setCoreContextIfNeeded:(NSString *)context; -- (void)setCoreContext:(NSString *)context; - (void)setCoreOptions:(NSString *)key withValue:(NSString *)value; - (CoreKeyOutput *)processEvent:(NSEvent *)event; diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m index 7afd3d40b5..ea59fb950d 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m @@ -39,7 +39,7 @@ const NSString* kEasterEggKmxName = @"EnglishSpanish.kmx"; if (kmx) { [self loadCoreWrapperFromKmxFile:self.kmx.filePath]; - [self.coreWrapper setContext:contextString]; + [self.coreWrapper setContextIfNeeded:contextString]; } } @@ -101,10 +101,6 @@ const NSString* kEasterEggKmxName = @"EnglishSpanish.kmx"; [self.coreWrapper setContextIfNeeded:context]; } -- (void)setCoreContext:(NSString *)context { - [self.coreWrapper setContext:context]; -} - - (void)setCoreOptions:(NSString *)key withValue:(NSString *)value { BOOL success = [self.coreWrapper setOptionsForCore:key value:value]; [self.coreHelper logDebugMessage:@"setCoreOptions for key: %@, value: %@ succeeded = %@", key, value, success ? @"YES" : @"NO"]; diff --git a/mac/KeymanEngine4Mac/KeymanEngine4MacTests/CoreWrapperTests.m b/mac/KeymanEngine4Mac/KeymanEngine4MacTests/CoreWrapperTests.m index 826d803328..2748695a81 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4MacTests/CoreWrapperTests.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4MacTests/CoreWrapperTests.m @@ -66,7 +66,7 @@ CoreWrapper *mockWrapper; - (void)testgetContextAsString_ContextContainsEmojis_ReturnsSameContext { NSString *kmxPath = [CoreTestStaticHelperMethods getKmxFilePathTestMacEngine]; CoreWrapper *core = [[CoreWrapper alloc] initWithHelper: [CoreTestStaticHelperMethods helper] kmxFilePath:kmxPath]; - [core setContext:@"🤔?👍🏻✅"]; + [core setContextIfNeeded:@"🤔?👍🏻✅"]; NSString *finalContext = core.contextDebug; // Note: relying on km_core_state_context_debug output format is just barely // acceptable for a unit test From 671973baabfadc4bcd51abd7a6115e8edf860d53 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Tue, 16 Jan 2024 12:04:41 +0700 Subject: [PATCH 29/45] refactor(core): split context API from Core primary API Relates to #9999. Fixes #10384. The context API endpoints should no longer be considered as part of the standard Core API. The only consumers that have a need to access these APIs are the IMX integration in Engine for Windows, and the Keyman Developer Debugger. These symbols are currently used by Developer: * `km_core_context` struct * `km_core_context_type` enum * `km_core_context_item` struct * `KM_CORE_CONTEXT_ITEM_END` macro * `km_core_state_context()` * `km_core_context_set()` * `km_core_context_clear()` These symbols are currently used by Windows IMX: * `km_core_context` struct * `km_core_context_type` enum * `km_core_context_item` struct * `KM_CORE_CONTEXT_ITEM_END` macro * `km_core_context_items_dispose()` * `km_core_context_item_list_size()` * `km_core_state_get_intermediate_context()` The following functions and symbols are moving to keyman_core_api_context.h: * `km_core_context` struct * `km_core_context_type` enum * `km_core_context_item` struct * `KM_CORE_CONTEXT_ITEM_END` macro * `km_core_state_context()` function * `km_core_state_get_intermediate_context()` function * `km_core_context_set()` function * `km_core_context_clear()` function * `km_core_context_get()` function * `km_core_context_items_from_utf16()` function * `km_core_context_items_from_utf8()` function * `km_core_context_items_to_utf8()` function * `km_core_context_items_to_utf16()` function * `km_core_context_items_to_utf32()` function * `km_core_context_items_dispose()` function * `km_core_context_length()` function * `km_core_context_append()` function * `km_core_context_shrink()` function * `km_core_context_item_list_size()` function --- core/include/keyman/keyman_core_api.h | 391 -------------- core/include/keyman/keyman_core_api_context.h | 478 ++++++++++++++++++ core/meson.build | 3 + core/src/action.cpp | 1 - core/src/action.hpp | 2 +- core/src/context.hpp | 2 +- core/src/debug.hpp | 3 +- core/src/keyboard.hpp | 2 +- core/src/keyman_core.h | 5 + core/src/km_core_action_api.cpp | 3 +- core/src/km_core_context_api.cpp | 2 +- core/src/km_core_debug_api.cpp | 2 +- core/src/km_core_keyboard_api.cpp | 3 +- core/src/km_core_options_api.cpp | 3 +- core/src/km_core_processevent_api.cpp | 3 +- core/src/km_core_state_api.cpp | 2 +- .../km_core_state_context_set_if_needed.cpp | 2 +- core/src/kmx/kmx_base.h | 3 +- core/src/kmx/kmx_options.h | 2 +- core/src/kmx/kmx_processor.cpp | 2 +- core/src/kmx/kmx_processor.hpp | 2 +- core/src/ldml/ldml_processor.hpp | 2 +- core/src/ldml/ldml_vkeys.hpp | 2 +- core/src/mock/mock_processor.hpp | 2 +- core/src/option.hpp | 3 +- core/src/path.hpp | 3 +- core/src/processor.hpp | 2 +- core/src/state.hpp | 2 +- core/tests/unit/kmnkbd/action_api.cpp | 2 +- core/tests/unit/kmnkbd/action_items.hpp | 2 +- core/tests/unit/kmnkbd/action_set_api.cpp | 3 +- core/tests/unit/kmnkbd/context_api.cpp | 3 +- core/tests/unit/kmnkbd/debug_api.cpp | 4 +- core/tests/unit/kmnkbd/debug_items.hpp | 3 +- core/tests/unit/kmnkbd/keyboard_api.cpp | 2 +- core/tests/unit/kmnkbd/options_api.cpp | 2 +- core/tests/unit/kmnkbd/state_api.cpp | 2 +- core/tests/unit/kmx/kmx.cpp | 2 + core/tests/unit/ldml/ldml.cpp | 2 + core/tests/unit/ldml/ldml_test_utils.hpp | 3 +- .../unit/ldml/test_context_normalization.cpp | 2 +- windows/src/engine/keyman32/keymanengine.h | 1 + 42 files changed, 536 insertions(+), 429 deletions(-) create mode 100644 core/include/keyman/keyman_core_api_context.h create mode 100644 core/src/keyman_core.h diff --git a/core/include/keyman/keyman_core_api.h b/core/include/keyman/keyman_core_api.h index 3e4451f612..e744e6bf60 100644 --- a/core/include/keyman/keyman_core_api.h +++ b/core/include/keyman/keyman_core_api.h @@ -115,7 +115,6 @@ typedef uint32_t km_core_status; // Status return code. // Opaque object types. // -typedef struct km_core_context km_core_context; typedef struct km_core_keyboard km_core_keyboard; typedef struct km_core_state km_core_state; typedef struct km_core_options km_core_options; @@ -155,357 +154,7 @@ The final status code KM_CORE_STATUS_OS_ERROR is intended to allow encapsulating a platform error code; the remaining 31 low bits are the error code returned by the OS for cases where the failure mode is platform specific. For HRESULT codes this only permits failure codes to be passed. - - -### Context -The context is the text prior to the insertion point (caret, cursor). -The context is constructed by the Platform layer, typically by interrogating the -Client Application. The context will be updated by the engine for keystroke -events. If the Platform layer code caches the context, the context should be -reset when a context state change is detected. Context state changes can occur -when the user uses the mouse to move the insertion point, uses cursor keys, -switches applications or input fields, or presses hotkeys such as Ctrl+N to -start a new document. The full set of context state change triggers is up to the -Platform layer. - -Context can also contain positional Markers (also known as 'deadkeys' in kmn -keyboards), which are transitory state flags that are erased whenever a context -state change is detected. Markers are always controlled by the Engine. - -Contexts are always owned by their state. They may be set to a list of -context_items or interrogated for their current list of context items. -```c */ -enum km_core_context_type { - KM_CORE_CT_END, - KM_CORE_CT_CHAR, - KM_CORE_CT_MARKER -}; - -typedef struct { - uint8_t type; - uint8_t _reserved[3]; - union { - km_core_usv character; - uint32_t marker; - }; -} km_core_context_item; - -#define KM_CORE_CONTEXT_ITEM_END {KM_CORE_CT_END, {0,}, {0,}} -/* -``` -### `km_core_context_items_from_utf16` -##### Description: -Convert a UTF16 encoded Unicode string into an array of `km_core_context_item` -structures. Allocates memory as needed. -##### Return status: -- `KM_CORE_STATUS_OK`: On success. -- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. -- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated for the - output buffer. -- `KM_CORE_STATUS_INVALID_UTF`: In the event the UTF16 string cannot be decoded - because it contains unpaired surrogate codeunits. -##### Parameters: -- __text__: a pointer to a null terminated array of utf16 encoded data. -- __out_ptr__: a pointer to the result variable: - A pointer to the start of the `km_core_context_item` array containing the - representation of the input string. - Terminated with a type of `KM_CORE_CT_END`. Must be disposed of with - `km_core_context_items_dispose`. - -```c -*/ -KMN_API -km_core_status -km_core_context_items_from_utf16(km_core_cp const *text, - km_core_context_item **out_ptr); - -/* -``` -### `km_core_context_items_from_utf8` -##### Description: -Convert an UTF8 encoded Unicode string into an array of `km_core_context_item` -structures. Allocates memory as needed. -##### Status: -- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. -- `KM_CORE_STATUS_NO_MEM`: In the event it cannot allocate enough memory for the - output buffer. -- `KM_CORE_STATUS_INVALID_UTF`: In the event the UTF8 string cannot be -decoded. -##### Parameters: -- __text__: a pointer to a null terminated array of utf8 encoded data. -- __out_ptr__: a pointer to the result variable: - A pointer to the start of the `km_core_context_item` array containing the - representation of the input string. - Terminated with a type of `KM_CORE_CT_END`. - -```c -*/ -KMN_API -km_core_status -km_core_context_items_from_utf8(char const *text, - km_core_context_item **out_ptr); - -/* -``` -### `km_core_context_items_to_utf16` -##### Description: -Convert a context item array into a UTF-16 encoded string placing it into -the supplied buffer of specified size, and return the number of code units -actually used in the conversion. If null is passed as the buffer the -number of codeunits required is returned. Any markers in the context will -not be included in the output buffer. -##### Return status: -- `KM_CORE_STATUS_OK`: On success. -- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. -- `KM_CORE_STATUS_INSUFFICENT_BUFFER`: If the buffer is not large enough. - `buf_size` will contain the space required. The contents of the buffer are - undefined. -##### Parameters: -- __context_items__: A pointer to the start of an array `km_core_context_item`. - Must be terminated with a type of `KM_CORE_CT_END`. -- __buf__: A pointer to the buffer to place the UTF-16 string into. - May be null to request size calculation. -- __buf_size__: a pointer to the result variable: - The size of the supplied buffer in codeunits if `buf` is given. - On return will be the size required if `buf` is null. - -```c -*/ -KMN_API -km_core_status -km_core_context_items_to_utf16(km_core_context_item const *item, - km_core_cp *buf, - size_t *buf_size); - -/* -``` -### `km_core_context_items_to_utf8` -##### Description: -Convert a context item array into a UTF-8 encoded string placing it into -the supplied buffer of specified size, and return the number of code units -actually used in the conversion. If null is passed as the buffer the -number of codeunits required is returned. Any markers in the context will -not be included in the output buffer. -##### Return status: -- `KM_CORE_STATUS_OK`: On success. -- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. -- `KM_CORE_STATUS_INSUFFICENT_BUFFER`: If the buffer is not large enough. - `buf_size` will contain the space required. The contents of the buffer are - undefined. -##### Parameters: -- __context_items__: A pointer to the start of an array `km_core_context_item`. - Must be terminated with a type of `KM_CORE_CT_END`. -- __buf__: A pointer to the buffer to place the UTF-8 string into. - May be null to request size calculation. -- __buf_size__: a pointer to the result variable: - The size of the supplied buffer in codeunits if `buf` is given. - On return will be the size required if `buf` is null. - -```c -*/ -KMN_API -km_core_status -km_core_context_items_to_utf8(km_core_context_item const *item, - char *buf, - size_t *buf_size); - -/* -``` -### `km_core_context_items_to_utf32` -##### Description: -Convert a context item array into a UTF-32 encoded string placing it into -the supplied buffer of specified size, and return the number of codepoints -actually used in the conversion. If null is passed as the buffer the -number of codepoints required is returned. Any markers in the context will -not be included in the output buffer. -##### Return status: -- `KM_CORE_STATUS_OK`: On success. -- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. -- `KM_CORE_STATUS_INSUFFICENT_BUFFER`: If the buffer is not large enough. - `buf_size` will contain the space required. The contents of the buffer are - undefined. -##### Parameters: -- __context_items__: A pointer to the start of an array `km_core_context_item`. - Must be terminated with a type of `KM_CORE_CT_END`. -- __buf__: A pointer to the buffer to place the UTF-32 string into. - May be null to request size calculation. -- __buf_size__: a pointer to the result variable: - The size of the supplied buffer in codepoints if `buf` is given. - On return will be the size required if `buf` is null. - -```c -*/ -KMN_API -km_core_status -km_core_context_items_to_utf32(km_core_context_item const *item, - km_core_usv *buf, - size_t *buf_size); - -/* -``` -### `km_core_context_items_dispose` -##### Description: -Free the allocated memory belonging to a `km_core_context_item` array previously -returned by `km_core_context_items_from_utf16` or `km_core_context_get` -##### Parameters: -- __context_items__: A pointer to the start of the `km_core_context_item` array - to be disposed of. - -```c -*/ -KMN_API -void -km_core_context_items_dispose(km_core_context_item *context_items); - -/* -``` -### `km_core_context_set` -##### Description: -Replace the contents of the current context with a new sequence of -`km_core_context_item` entries. -##### Return status: -- `KM_CORE_STATUS_OK`: On success. -- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. -- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated to - grow the context buffer internally. -##### Parameters: -- __context__: A pointer to an opaque context object -- __context_items__: A pointer to the start of the `km_core_context_item` - array containing the new context. It must be terminated with an item - of type `KM_CORE_CT_END`. - -```c -*/ -KMN_API -km_core_status -km_core_context_set(km_core_context *context, - km_core_context_item const *context_items); - -/* -``` -### `km_core_context_get` -##### Description: -Copies all items in the context into a new array and returns the new array. -This must be disposed of by caller using `km_core_context_items_dispose`. -##### Return status: -- `KM_CORE_STATUS_OK`: On success. -- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. -- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated for the - output buffer. -##### Parameters: -- __context_items__: A pointer to the start of an array `km_core_context_item`. -- __out__: a pointer to the result variable: - A pointer to the start of the `km_core_context_item` array containing a - copy of the context. Terminated with a type of `KM_CORE_CT_END`. Must be - disposed of with `km_core_context_items_dispose`. - -```c -*/ -KMN_API -km_core_status -km_core_context_get(km_core_context const *context_items, - km_core_context_item **out); - -/* -``` -### `km_core_context_clear` -##### Description: -Removes all context_items from the internal array. If `context` is -null, has no effect. -##### Parameters: -- __context__: A pointer to an opaque context object - -```c -*/ -KMN_API -void -km_core_context_clear(km_core_context *); - -/* -``` -### `km_core_context_length` -##### Description: -Return the number of items in the context. -##### Return: -The number of items in the context, and will return 0 if passed a null `context` -pointer. -##### Parameters: -- __context__: A pointer to an opaque context object - -```c -*/ -KMN_API -size_t -km_core_context_length(km_core_context *); - -/* -``` -### `km_core_context_append` -##### Description: -Add more items to the end (insertion point) of the context. If these exceed the -maximum context length the same number of items will be dropped from the -beginning of the context. -##### Return status: -- `KM_CORE_STATUS_OK`: On success. -- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. -- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated to - grow the context buffer internally. -##### Parameters: -- __context__: A pointer to an opaque context object. -- __context_items__: A pointer to the start of the `KM_CORE_CT_END` terminated - array of `km_core_context_item` to append. - -```c -*/ -KMN_API -km_core_status -km_core_context_append(km_core_context *context, - km_core_context_item const *context_items); - -/* -``` -### `km_core_context_shrink` -##### Description: -Remove a specified number of items from the end of the context, optionally -add up to the same number of the supplied items to the front of the context. -##### Return status: -- `KM_CORE_STATUS_OK`: On success. -- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. -- `KM_CORE_STATUS_NO_MEM`: in the event it cannot allocated enough memory to grow - the context internally. -##### Parameters: -- __context__: A pointer to an opaque context object. -- __num__: The number of items to remove from the end of context. -- __context_items__: Pointer to the start of the `KM_CORE_CT_END` terminated - array of `km_core_context_item` to add to the front. Up to `num` items will - be prepended. This may be null if not required. - -```c -*/ -KMN_API -km_core_status -km_core_context_shrink(km_core_context *context, - size_t num, - km_core_context_item const *prefix); - -/* -``` -### `km_core_context_item_list_size` -##### Description: -Return the length of a terminated `km_core_context_item` array. -##### Return: -The number of items in the list, not including terminating item, -or 0 if `context_items` is null. -##### Parameters: -- __context_items__: A pointer to a `KM_CORE_CT_END` terminated array of - `km_core_context_item` values. - -```c -*/ -KMN_API -size_t -km_core_context_item_list_size(km_core_context_item const *context_items); /* ``` @@ -1104,46 +753,6 @@ KMN_API void km_core_state_dispose(km_core_state *state); -/* -``` -### `km_core_state_context` -##### Description: -Get access to the state object's context. -##### Return: -A pointer to an opaque state object. This pointer is valid for the lifetime -of the state object. If null is passed in, then null is returned. -##### Parameters: -- __state__: A pointer to the opaque state object to be queried. - -```c -*/ -KMN_API -km_core_context * -km_core_state_context(km_core_state *state); - - -KMN_API -km_core_context * -km_core_state_app_context(km_core_state *state); - -/* -``` -### `km_core_state_get_intermediate_context` -##### Description: -Get access to the state object's keyboard processor's intermediate context. This context -is used during an IMX callback, part way through processing a keystroke. -##### Return: -A pointer to an context item array. Must be disposed of by a call -to `km_core_context_items_dispose`. -##### Parameters: -- __state__: A pointer to the opaque state object to be queried. - -```c -*/ -KMN_API -km_core_status -km_core_state_get_intermediate_context(km_core_state *state, km_core_context_item ** context_items); - /* ``` ### Context Debug Reporting diff --git a/core/include/keyman/keyman_core_api_context.h b/core/include/keyman/keyman_core_api_context.h new file mode 100644 index 0000000000..26bd2f25bc --- /dev/null +++ b/core/include/keyman/keyman_core_api_context.h @@ -0,0 +1,478 @@ +/* + * Keyman is copyright (C) SIL International. MIT License. + * + * Keyman Keyboard Processor API - Debugger Interfaces + * + * The debugger interfaces are still very dependent on .kmx + * objects. + * + * Note: this file is subject to change; the debugger + * interfaces are not stable across versions. + * + */ + +#pragma once + +#include +#include +#include +#include + +// Currently, the Core unit tests use private context APIs defined in +// keyman_core_api_context.h, which are unused by other consumers. We are +// hoping to remove these entirely in the future, so we restrict access +// by default with this macro. Keyman Core internally uses these functions +// #define _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +#if defined(__cplusplus) +extern "C" +{ +#endif + +typedef struct km_core_context km_core_context; + +// ---------------------------------------------------------------------------------- +// Context APIs are now available only to the keyboard debugger and Core unit +// tests (17.0) +// ---------------------------------------------------------------------------------- + +/* +### Context +The context is the text prior to the insertion point (caret, cursor). +The context is constructed by the Platform layer, typically by interrogating the +Client Application. The context will be updated by the engine for keystroke +events. If the Platform layer code caches the context, the context should be +reset when a context state change is detected. Context state changes can occur +when the user uses the mouse to move the insertion point, uses cursor keys, +switches applications or input fields, or presses hotkeys such as Ctrl+N to +start a new document. The full set of context state change triggers is up to the +Platform layer. + +Context can also contain positional Markers (also known as 'deadkeys' in kmn +keyboards), which are transitory state flags that are erased whenever a context +state change is detected. Markers are always controlled by the Engine. + +Contexts are always owned by their state. They may be set to a list of +context_items or interrogated for their current list of context items. +```c +*/ +enum km_core_context_type { + KM_CORE_CT_END, + KM_CORE_CT_CHAR, + KM_CORE_CT_MARKER +}; + +typedef struct { + uint8_t type; + uint8_t _reserved[3]; + union { + km_core_usv character; + uint32_t marker; + }; +} km_core_context_item; + +#define KM_CORE_CONTEXT_ITEM_END {KM_CORE_CT_END, {0,}, {0,}} + +/* +``` +### `km_core_state_get_intermediate_context` +##### Description: +Get access to the state object's keyboard processor's intermediate context. This context +is used during an IMX callback, part way through processing a keystroke. +##### Return: +A pointer to an context item array. Must be disposed of by a call +to `km_core_context_items_dispose`. +##### Parameters: +- __state__: A pointer to the opaque state object to be queried. + +```c +*/ +KMN_API +km_core_status +km_core_state_get_intermediate_context(km_core_state *state, km_core_context_item ** context_items); + +/* +``` +### `km_core_context_items_dispose` +##### Description: +Free the allocated memory belonging to a `km_core_context_item` array previously +returned by `km_core_context_items_from_utf16` or `km_core_context_get` +##### Parameters: +- __context_items__: A pointer to the start of the `km_core_context_item` array + to be disposed of. + +```c +*/ +KMN_API +void +km_core_context_items_dispose(km_core_context_item *context_items); + +/** + * Get access to the state object's cached context. + * @param state A pointer to the opaque state object to be queried. + * @returns A pointer to an opaque context object. This pointer is valid for the + * lifetime of the state object. If null is passed in, then null is + * returned. + */ +KMN_API +km_core_context * +km_core_state_context(km_core_state *state); + +/** + * Get access to the state object's application context. + * @param state A pointer to the opaque state object to be queried. + * @returns A pointer to an opaque context object. This pointer is valid for the + * lifetime of the state object. If null is passed in, then null is + * returned. + */ +KMN_API +km_core_context * +km_core_state_app_context(km_core_state *state); + +/* +``` +### `km_core_context_items_from_utf16` +##### Description: +Convert a UTF16 encoded Unicode string into an array of `km_core_context_item` +structures. Allocates memory as needed. +##### Return status: +- `KM_CORE_STATUS_OK`: On success. +- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. +- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated for the + output buffer. +- `KM_CORE_STATUS_INVALID_UTF`: In the event the UTF16 string cannot be decoded + because it contains unpaired surrogate codeunits. +##### Parameters: +- __text__: a pointer to a null terminated array of utf16 encoded data. +- __out_ptr__: a pointer to the result variable: + A pointer to the start of the `km_core_context_item` array containing the + representation of the input string. + Terminated with a type of `KM_CORE_CT_END`. Must be disposed of with + `km_core_context_items_dispose`. + +```c +*/ + +#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +KMN_API +km_core_status +km_core_context_items_from_utf16(km_core_cp const *text, + km_core_context_item **out_ptr); + +#endif + +/* +``` +### `km_core_context_items_from_utf8` +##### Description: +Convert an UTF8 encoded Unicode string into an array of `km_core_context_item` +structures. Allocates memory as needed. +##### Status: +- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. +- `KM_CORE_STATUS_NO_MEM`: In the event it cannot allocate enough memory for the + output buffer. +- `KM_CORE_STATUS_INVALID_UTF`: In the event the UTF8 string cannot be +decoded. +##### Parameters: +- __text__: a pointer to a null terminated array of utf8 encoded data. +- __out_ptr__: a pointer to the result variable: + A pointer to the start of the `km_core_context_item` array containing the + representation of the input string. + Terminated with a type of `KM_CORE_CT_END`. + +```c +*/ + +#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +KMN_API +km_core_status +km_core_context_items_from_utf8(char const *text, + km_core_context_item **out_ptr); + +#endif + +/* +``` +### `km_core_context_items_to_utf16` +##### Description: +Convert a context item array into a UTF-16 encoded string placing it into +the supplied buffer of specified size, and return the number of code units +actually used in the conversion. If null is passed as the buffer the +number of codeunits required is returned. Any markers in the context will +not be included in the output buffer. +##### Return status: +- `KM_CORE_STATUS_OK`: On success. +- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. +- `KM_CORE_STATUS_INSUFFICENT_BUFFER`: If the buffer is not large enough. + `buf_size` will contain the space required. The contents of the buffer are + undefined. +##### Parameters: +- __context_items__: A pointer to the start of an array `km_core_context_item`. + Must be terminated with a type of `KM_CORE_CT_END`. +- __buf__: A pointer to the buffer to place the UTF-16 string into. + May be null to request size calculation. +- __buf_size__: a pointer to the result variable: + The size of the supplied buffer in codeunits if `buf` is given. + On return will be the size required if `buf` is null. + +```c +*/ + +#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +KMN_API +km_core_status +km_core_context_items_to_utf16(km_core_context_item const *item, + km_core_cp *buf, + size_t *buf_size); + +#endif + +/* +``` +### `km_core_context_items_to_utf8` +##### Description: +Convert a context item array into a UTF-8 encoded string placing it into +the supplied buffer of specified size, and return the number of code units +actually used in the conversion. If null is passed as the buffer the +number of codeunits required is returned. Any markers in the context will +not be included in the output buffer. +##### Return status: +- `KM_CORE_STATUS_OK`: On success. +- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. +- `KM_CORE_STATUS_INSUFFICENT_BUFFER`: If the buffer is not large enough. + `buf_size` will contain the space required. The contents of the buffer are + undefined. +##### Parameters: +- __context_items__: A pointer to the start of an array `km_core_context_item`. + Must be terminated with a type of `KM_CORE_CT_END`. +- __buf__: A pointer to the buffer to place the UTF-8 string into. + May be null to request size calculation. +- __buf_size__: a pointer to the result variable: + The size of the supplied buffer in codeunits if `buf` is given. + On return will be the size required if `buf` is null. + +```c +*/ + +#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +KMN_API +km_core_status +km_core_context_items_to_utf8(km_core_context_item const *item, + char *buf, + size_t *buf_size); + +#endif + +/* +``` +### `km_core_context_items_to_utf32` +##### Description: +Convert a context item array into a UTF-32 encoded string placing it into +the supplied buffer of specified size, and return the number of codepoints +actually used in the conversion. If null is passed as the buffer the +number of codepoints required is returned. Any markers in the context will +not be included in the output buffer. +##### Return status: +- `KM_CORE_STATUS_OK`: On success. +- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. +- `KM_CORE_STATUS_INSUFFICENT_BUFFER`: If the buffer is not large enough. + `buf_size` will contain the space required. The contents of the buffer are + undefined. +##### Parameters: +- __context_items__: A pointer to the start of an array `km_core_context_item`. + Must be terminated with a type of `KM_CORE_CT_END`. +- __buf__: A pointer to the buffer to place the UTF-32 string into. + May be null to request size calculation. +- __buf_size__: a pointer to the result variable: + The size of the supplied buffer in codepoints if `buf` is given. + On return will be the size required if `buf` is null. + +```c +*/ + +#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +KMN_API +km_core_status +km_core_context_items_to_utf32(km_core_context_item const *item, + km_core_usv *buf, + size_t *buf_size); + +#endif + +/* +``` +### `km_core_context_set` +##### Description: +Replace the contents of the current context with a new sequence of +`km_core_context_item` entries. +##### Return status: +- `KM_CORE_STATUS_OK`: On success. +- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. +- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated to + grow the context buffer internally. +##### Parameters: +- __context__: A pointer to an opaque context object +- __context_items__: A pointer to the start of the `km_core_context_item` + array containing the new context. It must be terminated with an item + of type `KM_CORE_CT_END`. + +```c +*/ +KMN_API +km_core_status +km_core_context_set(km_core_context *context, + km_core_context_item const *context_items); + +/* +``` +### `km_core_context_get` +##### Description: +Copies all items in the context into a new array and returns the new array. +This must be disposed of by caller using `km_core_context_items_dispose`. +##### Return status: +- `KM_CORE_STATUS_OK`: On success. +- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. +- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated for the + output buffer. +##### Parameters: +- __context_items__: A pointer to the start of an array `km_core_context_item`. +- __out__: a pointer to the result variable: + A pointer to the start of the `km_core_context_item` array containing a + copy of the context. Terminated with a type of `KM_CORE_CT_END`. Must be + disposed of with `km_core_context_items_dispose`. + +```c +*/ + +#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +KMN_API +km_core_status +km_core_context_get(km_core_context const *context_items, + km_core_context_item **out); + +#endif + +/* +``` +### `km_core_context_clear` +##### Description: +Removes all context_items from the internal array. If `context` is +null, has no effect. +##### Parameters: +- __context__: A pointer to an opaque context object + +```c +*/ +KMN_API +void +km_core_context_clear(km_core_context *); + +/* +``` +### `km_core_context_length` +##### Description: +Return the number of items in the context. +##### Return: +The number of items in the context, and will return 0 if passed a null `context` +pointer. +##### Parameters: +- __context__: A pointer to an opaque context object + +```c +*/ + +#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +KMN_API +size_t +km_core_context_length(km_core_context *); + +#endif + +/* +``` +### `km_core_context_append` +##### Description: +Add more items to the end (insertion point) of the context. If these exceed the +maximum context length the same number of items will be dropped from the +beginning of the context. +##### Return status: +- `KM_CORE_STATUS_OK`: On success. +- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. +- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated to + grow the context buffer internally. +##### Parameters: +- __context__: A pointer to an opaque context object. +- __context_items__: A pointer to the start of the `KM_CORE_CT_END` terminated + array of `km_core_context_item` to append. + +```c +*/ + +#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +KMN_API +km_core_status +km_core_context_append(km_core_context *context, + km_core_context_item const *context_items); + +#endif + +/* +``` +### `km_core_context_shrink` +##### Description: +Remove a specified number of items from the end of the context, optionally +add up to the same number of the supplied items to the front of the context. +##### Return status: +- `KM_CORE_STATUS_OK`: On success. +- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. +- `KM_CORE_STATUS_NO_MEM`: in the event it cannot allocated enough memory to grow + the context internally. +##### Parameters: +- __context__: A pointer to an opaque context object. +- __num__: The number of items to remove from the end of context. +- __context_items__: Pointer to the start of the `KM_CORE_CT_END` terminated + array of `km_core_context_item` to add to the front. Up to `num` items will + be prepended. This may be null if not required. + +```c +*/ + +#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +KMN_API +km_core_status +km_core_context_shrink(km_core_context *context, + size_t num, + km_core_context_item const *prefix); + +#endif + +/* +``` +### `km_core_context_item_list_size` +##### Description: +Return the length of a terminated `km_core_context_item` array. +##### Return: +The number of items in the list, not including terminating item, +or 0 if `context_items` is null. +##### Parameters: +- __context_items__: A pointer to a `KM_CORE_CT_END` terminated array of + `km_core_context_item` values. + +```c +*/ +KMN_API +size_t +km_core_context_item_list_size(km_core_context_item const *context_items); + +#if defined(__cplusplus) +} // extern "C" +#endif diff --git a/core/meson.build b/core/meson.build index 716faca2dc..e6b1fd8fa1 100644 --- a/core/meson.build +++ b/core/meson.build @@ -29,6 +29,9 @@ python = py.find_installation() # For now, we use KM_CORE_LIBRARY to inject the km::core::kmx namespace defns += ['-DKM_CORE_LIBRARY'] +# See keyman_core_api_context.h +defns += ['-D_KM_CORE_ACCESS_PRIVATE_CONTEXT_API'] + # #define DEBUG when we are on a debug build if get_option('buildtype') == 'debug' add_global_arguments('-DDEBUG', language : 'cpp') diff --git a/core/src/action.cpp b/core/src/action.cpp index f4565844d7..d2e0613b14 100644 --- a/core/src/action.cpp +++ b/core/src/action.cpp @@ -11,7 +11,6 @@ #include #include -#include #include "action.hpp" #include "state.hpp" diff --git a/core/src/action.hpp b/core/src/action.hpp index 06038c3122..d8ee401e97 100644 --- a/core/src/action.hpp +++ b/core/src/action.hpp @@ -8,7 +8,7 @@ #pragma once -#include +#include "keyman_core.h" #include namespace km { diff --git a/core/src/context.hpp b/core/src/context.hpp index dba7bec060..36831f6d14 100644 --- a/core/src/context.hpp +++ b/core/src/context.hpp @@ -9,7 +9,7 @@ #pragma once #include #include -#include +#include "keyman_core.h" // Forward declarations class json; diff --git a/core/src/debug.hpp b/core/src/debug.hpp index a68d9f1e57..33de3dd324 100644 --- a/core/src/debug.hpp +++ b/core/src/debug.hpp @@ -9,8 +9,7 @@ #include #include -#include -#include +#include "keyman_core.h" namespace km { namespace core diff --git a/core/src/keyboard.hpp b/core/src/keyboard.hpp index 6dd8517ce1..142bf40e86 100644 --- a/core/src/keyboard.hpp +++ b/core/src/keyboard.hpp @@ -11,7 +11,7 @@ #include #include -#include +#include "keyman_core.h" #include "option.hpp" #include "path.hpp" diff --git a/core/src/keyman_core.h b/core/src/keyman_core.h new file mode 100644 index 0000000000..8750b7b93e --- /dev/null +++ b/core/src/keyman_core.h @@ -0,0 +1,5 @@ +#pragma once + +#include +#include +#include \ No newline at end of file diff --git a/core/src/km_core_action_api.cpp b/core/src/km_core_action_api.cpp index 1177ec71fc..db8a5c8820 100644 --- a/core/src/km_core_action_api.cpp +++ b/core/src/km_core_action_api.cpp @@ -10,7 +10,8 @@ #include #include -#include +#include "keyman_core.h" + #include "jsonpp.hpp" #include "processor.hpp" diff --git a/core/src/km_core_context_api.cpp b/core/src/km_core_context_api.cpp index 46522710c0..ddf49e21e1 100644 --- a/core/src/km_core_context_api.cpp +++ b/core/src/km_core_context_api.cpp @@ -12,7 +12,7 @@ #include #include -#include +#include "keyman_core.h" #include "context.hpp" #include "jsonpp.hpp" diff --git a/core/src/km_core_debug_api.cpp b/core/src/km_core_debug_api.cpp index 2c6433a522..75e34b0b0c 100644 --- a/core/src/km_core_debug_api.cpp +++ b/core/src/km_core_debug_api.cpp @@ -8,7 +8,7 @@ #include #include -#include +#include "keyman_core.h" #include "processor.hpp" #include "state.hpp" diff --git a/core/src/km_core_keyboard_api.cpp b/core/src/km_core_keyboard_api.cpp index ae825ca402..964f73ca07 100644 --- a/core/src/km_core_keyboard_api.cpp +++ b/core/src/km_core_keyboard_api.cpp @@ -11,7 +11,8 @@ #include #include -#include +#include "keyman_core.h" + #include "keyboard.hpp" #include "processor.hpp" #include "kmx/kmx_processor.hpp" diff --git a/core/src/km_core_options_api.cpp b/core/src/km_core_options_api.cpp index 3cb0745fa9..4ea5c74062 100644 --- a/core/src/km_core_options_api.cpp +++ b/core/src/km_core_options_api.cpp @@ -11,7 +11,8 @@ #include #include -#include +#include "keyman_core.h" + #include "processor.hpp" #include "jsonpp.hpp" diff --git a/core/src/km_core_processevent_api.cpp b/core/src/km_core_processevent_api.cpp index 5295d39af7..7d5109a6b2 100644 --- a/core/src/km_core_processevent_api.cpp +++ b/core/src/km_core_processevent_api.cpp @@ -8,7 +8,8 @@ History: 17 Oct 2018 - TSE - Initial implementation. */ -#include +#include "keyman_core.h" + #include "processor.hpp" #include "state.hpp" diff --git a/core/src/km_core_state_api.cpp b/core/src/km_core_state_api.cpp index eb430ba5ac..7b7e74f634 100644 --- a/core/src/km_core_state_api.cpp +++ b/core/src/km_core_state_api.cpp @@ -15,7 +15,7 @@ #include #include -#include +#include "keyman_core.h" #include "jsonpp.hpp" #include "processor.hpp" diff --git a/core/src/km_core_state_context_set_if_needed.cpp b/core/src/km_core_state_context_set_if_needed.cpp index b79044455e..72b80f6f51 100644 --- a/core/src/km_core_state_context_set_if_needed.cpp +++ b/core/src/km_core_state_context_set_if_needed.cpp @@ -9,7 +9,7 @@ */ #include -#include +#include "keyman_core.h" #include "processor.hpp" #include "state.hpp" diff --git a/core/src/kmx/kmx_base.h b/core/src/kmx/kmx_base.h index fed03bc645..e798ff56b7 100644 --- a/core/src/kmx/kmx_base.h +++ b/core/src/kmx/kmx_base.h @@ -1,7 +1,6 @@ #pragma once -#include -#include +#include "keyman_core.h" #if defined(_WIN32) || defined(_WIN64) #define snprintf _snprintf diff --git a/core/src/kmx/kmx_options.h b/core/src/kmx/kmx_options.h index 14c280e411..cf038ac7dc 100644 --- a/core/src/kmx/kmx_options.h +++ b/core/src/kmx/kmx_options.h @@ -4,7 +4,7 @@ #include #include -#include +#include "keyman_core.h" #include "option.hpp" #include "kmx_base.h" diff --git a/core/src/kmx/kmx_processor.cpp b/core/src/kmx/kmx_processor.cpp index 1fa0734e07..95a9a09456 100644 --- a/core/src/kmx/kmx_processor.cpp +++ b/core/src/kmx/kmx_processor.cpp @@ -1,4 +1,4 @@ -#include +#include "keyman_core.h" #include "state.hpp" #include "kmx/kmx_processor.hpp" #include diff --git a/core/src/kmx/kmx_processor.hpp b/core/src/kmx/kmx_processor.hpp index 490bda53c6..08b2c1d242 100644 --- a/core/src/kmx/kmx_processor.hpp +++ b/core/src/kmx/kmx_processor.hpp @@ -9,7 +9,7 @@ #pragma once #include -#include +#include "keyman_core.h" #include "kmx/kmx_processevent.h" #include "keyboard.hpp" #include "processor.hpp" diff --git a/core/src/ldml/ldml_processor.hpp b/core/src/ldml/ldml_processor.hpp index a66d151fde..238b332215 100644 --- a/core/src/ldml/ldml_processor.hpp +++ b/core/src/ldml/ldml_processor.hpp @@ -13,7 +13,7 @@ #include #include #include -#include +#include "keyman_core.h" #include "processor.hpp" #include "option.hpp" #include "ldml_vkeys.hpp" diff --git a/core/src/ldml/ldml_vkeys.hpp b/core/src/ldml/ldml_vkeys.hpp index 22dea817ef..ed3368dc7e 100644 --- a/core/src/ldml/ldml_vkeys.hpp +++ b/core/src/ldml/ldml_vkeys.hpp @@ -13,7 +13,7 @@ #include #include -#include +#include "keyman_core.h" namespace km { namespace core { diff --git a/core/src/mock/mock_processor.hpp b/core/src/mock/mock_processor.hpp index a3ec3c1caa..87a99a49df 100644 --- a/core/src/mock/mock_processor.hpp +++ b/core/src/mock/mock_processor.hpp @@ -10,7 +10,7 @@ #include #include -#include +#include "keyman_core.h" #include "processor.hpp" #include "option.hpp" diff --git a/core/src/option.hpp b/core/src/option.hpp index 43f3397049..60026798cf 100644 --- a/core/src/option.hpp +++ b/core/src/option.hpp @@ -12,7 +12,8 @@ #include -#include +#include "keyman_core.h" + // Forward declarations class json; diff --git a/core/src/path.hpp b/core/src/path.hpp index be6ae61422..aee70ad8ba 100644 --- a/core/src/path.hpp +++ b/core/src/path.hpp @@ -12,7 +12,8 @@ #include #include -#include +#include "keyman_core.h" + #include "jsonpp.hpp" #include "utfcodec.hpp" diff --git a/core/src/processor.hpp b/core/src/processor.hpp index 7a6ed5aa33..8925ad65d1 100644 --- a/core/src/processor.hpp +++ b/core/src/processor.hpp @@ -11,7 +11,7 @@ #include #include -#include +#include "keyman_core.h" #include "keyboard.hpp" diff --git a/core/src/state.hpp b/core/src/state.hpp index 0b98cc9e24..c3afff5029 100644 --- a/core/src/state.hpp +++ b/core/src/state.hpp @@ -11,7 +11,7 @@ #include #include -#include +#include "keyman_core.h" #include "context.hpp" #include "option.hpp" diff --git a/core/tests/unit/kmnkbd/action_api.cpp b/core/tests/unit/kmnkbd/action_api.cpp index 6a84eafa2d..a9e0058b56 100644 --- a/core/tests/unit/kmnkbd/action_api.cpp +++ b/core/tests/unit/kmnkbd/action_api.cpp @@ -6,7 +6,7 @@ History: 23 Oct 2023 - MCD - Initial implementation. */ #include -#include +#include "keyman_core.h" #include "path.hpp" #include "action.hpp" diff --git a/core/tests/unit/kmnkbd/action_items.hpp b/core/tests/unit/kmnkbd/action_items.hpp index 5b4723fd57..dc5ac666b6 100644 --- a/core/tests/unit/kmnkbd/action_items.hpp +++ b/core/tests/unit/kmnkbd/action_items.hpp @@ -1,4 +1,4 @@ -#include +#include "keyman_core.h" #include #include diff --git a/core/tests/unit/kmnkbd/action_set_api.cpp b/core/tests/unit/kmnkbd/action_set_api.cpp index d3c06cc9c7..2f69f2baba 100644 --- a/core/tests/unit/kmnkbd/action_set_api.cpp +++ b/core/tests/unit/kmnkbd/action_set_api.cpp @@ -6,7 +6,8 @@ History: 23 Oct 2023 - MCD - Initial implementation. */ #include -#include + +#include "keyman_core.h" #include "path.hpp" #include "action.hpp" diff --git a/core/tests/unit/kmnkbd/context_api.cpp b/core/tests/unit/kmnkbd/context_api.cpp index d5225eb24f..149ebf9556 100644 --- a/core/tests/unit/kmnkbd/context_api.cpp +++ b/core/tests/unit/kmnkbd/context_api.cpp @@ -10,7 +10,8 @@ mutation functions. */ #include -#include + +#include "keyman_core.h" #include "context.hpp" #include "utfcodec.hpp" diff --git a/core/tests/unit/kmnkbd/debug_api.cpp b/core/tests/unit/kmnkbd/debug_api.cpp index 502f95cc47..3313888477 100644 --- a/core/tests/unit/kmnkbd/debug_api.cpp +++ b/core/tests/unit/kmnkbd/debug_api.cpp @@ -10,7 +10,9 @@ #include #include #include -#include + +#include "keyman_core.h" + #include "path.hpp" #include "state.hpp" #include "kmx/kmx_base.h" diff --git a/core/tests/unit/kmnkbd/debug_items.hpp b/core/tests/unit/kmnkbd/debug_items.hpp index 400f499bd2..ae5f0e8cbd 100644 --- a/core/tests/unit/kmnkbd/debug_items.hpp +++ b/core/tests/unit/kmnkbd/debug_items.hpp @@ -1,6 +1,5 @@ #include -#include -#include +#include "keyman_core.h" #include "kmx/kmx_base.h" #include "kmx/kmx_xstring.h" diff --git a/core/tests/unit/kmnkbd/keyboard_api.cpp b/core/tests/unit/kmnkbd/keyboard_api.cpp index 95365e901c..bf232d69af 100644 --- a/core/tests/unit/kmnkbd/keyboard_api.cpp +++ b/core/tests/unit/kmnkbd/keyboard_api.cpp @@ -6,7 +6,7 @@ */ #include -#include +#include "keyman_core.h" #include "path.hpp" //#include "keyboard.hpp" diff --git a/core/tests/unit/kmnkbd/options_api.cpp b/core/tests/unit/kmnkbd/options_api.cpp index a831180f41..7c04478fe7 100644 --- a/core/tests/unit/kmnkbd/options_api.cpp +++ b/core/tests/unit/kmnkbd/options_api.cpp @@ -7,7 +7,7 @@ #include #include -#include +#include "keyman_core.h" #include "option.hpp" #include "state.hpp" diff --git a/core/tests/unit/kmnkbd/state_api.cpp b/core/tests/unit/kmnkbd/state_api.cpp index e3d1717589..07acd8bdd4 100644 --- a/core/tests/unit/kmnkbd/state_api.cpp +++ b/core/tests/unit/kmnkbd/state_api.cpp @@ -9,7 +9,7 @@ #include #include -#include +#include "keyman_core.h" #include "path.hpp" #include "state.hpp" diff --git a/core/tests/unit/kmx/kmx.cpp b/core/tests/unit/kmx/kmx.cpp index 794c744a7c..1e94f4cf86 100644 --- a/core/tests/unit/kmx/kmx.cpp +++ b/core/tests/unit/kmx/kmx.cpp @@ -17,6 +17,8 @@ #include #include +#include "keyman_core.h" + #include #include diff --git a/core/tests/unit/ldml/ldml.cpp b/core/tests/unit/ldml/ldml.cpp index 453ef8ac8e..db95141eba 100644 --- a/core/tests/unit/ldml/ldml.cpp +++ b/core/tests/unit/ldml/ldml.cpp @@ -24,6 +24,8 @@ #include #include +#include "keyman_core.h" + #include // for surrogate pair macros #include "ldml_test_source.hpp" diff --git a/core/tests/unit/ldml/ldml_test_utils.hpp b/core/tests/unit/ldml/ldml_test_utils.hpp index 1d876365bb..ec180200ed 100644 --- a/core/tests/unit/ldml/ldml_test_utils.hpp +++ b/core/tests/unit/ldml/ldml_test_utils.hpp @@ -13,7 +13,8 @@ #include // for char to vk mapping tables #include // for surrogate pair macros #include -#include + +#include "keyman_core.h" namespace km { namespace tests { diff --git a/core/tests/unit/ldml/test_context_normalization.cpp b/core/tests/unit/ldml/test_context_normalization.cpp index c3215ce4a3..11a3b8ca0e 100644 --- a/core/tests/unit/ldml/test_context_normalization.cpp +++ b/core/tests/unit/ldml/test_context_normalization.cpp @@ -6,7 +6,7 @@ History: 15 Jan 2024 - MCD - Initial implementation. */ #include -#include +#include "keyman_core.h" #include "path.hpp" #include "action.hpp" diff --git a/windows/src/engine/keyman32/keymanengine.h b/windows/src/engine/keyman32/keymanengine.h index c0db193fdf..9befcb302a 100644 --- a/windows/src/engine/keyman32/keymanengine.h +++ b/windows/src/engine/keyman32/keymanengine.h @@ -40,6 +40,7 @@ #include #include "../../../../common/windows/cpp/include/legacy_kmx_file.h" #include +#include // for intermediate context #include /***************************************************************************/ From cf2d6c42445baa692129a3686d4c9624c0a71283 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Tue, 16 Jan 2024 12:18:29 +0700 Subject: [PATCH 30/45] chore(mac): remove legacy Core action items references Relates to #9999. --- .../CoreWrapper/CoreWrapper.m | 85 ------------------- 1 file changed, 85 deletions(-) diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m index 8472d70d39..ac011ce35e 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m @@ -225,91 +225,6 @@ const int CORE_ENVIRONMENT_ARRAY_LENGTH = 6; return capsLock; } --(NSArray*)loadActionsUsingCore { - size_t actionCount = 0; - km_core_action_item const * actionList = - km_core_state_action_items(self.coreState, &actionCount); - - NSMutableArray *eventArray = [NSMutableArray arrayWithCapacity:actionCount]; - - for (int i = 0; i < actionCount; i++) { - km_core_action_item action = actionList[i]; - CoreAction *coreAction = [self createCoreActionForActionStruct:&action]; - [eventArray insertObject:coreAction atIndex:i]; - } - - return eventArray; -} - --(CoreAction*)createCoreActionForActionStruct:(km_core_action_item*)actionStruct { - CoreAction* action = nil; - switch (actionStruct->type) - { - case KM_CORE_IT_END: { - action = [[CoreAction alloc] initWithType: EndAction actionContent:@"" backspaceCount:0 key:@"" value:@"" scope:0]; - break; - } - case KM_CORE_IT_CHAR: { - NSString *characterString = [self.coreHelper utf32ValueToString:actionStruct->character]; - action = [[CoreAction alloc] initWithType: CharacterAction actionContent:characterString backspaceCount:0 key:@"" value:@"" scope:0]; - [self.coreHelper logDebugMessage:@"createCoreActionForActionStruct actionStruct->character decimal: %u, hex: %X", actionStruct->character, actionStruct->character]; - [self.coreHelper logDebugMessage:@"createCoreActionForActionStruct converted unicode string: '%@' length=%lu", characterString, characterString.length]; - break; - } - case KM_CORE_IT_MARKER: { - action = [[CoreAction alloc] initWithType: MarkerAction actionContent:@"" backspaceCount:0 key:@"" value:@"" scope:0]; - break; - } - case KM_CORE_IT_ALERT: { - action = [[CoreAction alloc] initWithType: AlertAction actionContent:@"" backspaceCount:0 key:@"" value:@"" scope:0]; - break; - } - case KM_CORE_IT_BACK: { - km_core_backspace_item backspace = actionStruct->backspace; - - if (backspace.expected_type == KM_CORE_BT_CHAR) { - NSString *charString = [self.coreHelper utf32ValueToString:backspace.expected_value]; - [self.coreHelper logDebugMessage:@"createCoreActionForActionStruct charString = %@", charString]; - action = [[CoreAction alloc] initCharacterBackspaceAction:charString]; - [self.coreHelper logDebugMessage:@"createCoreActionForActionStruct converted character backspace, expected value =%lu, expected type =%u", backspace.expected_value, backspace.expected_type]; - } else if(backspace.expected_type == KM_CORE_BT_MARKER) { - action = [[CoreAction alloc] initMarkerBackspaceAction:actionStruct->backspace.expected_value]; - [self.coreHelper logDebugMessage:@"createCoreActionForActionStruct converted marker backspace, expected value =%lu, expected type =%u", backspace.expected_value, backspace.expected_type]; - } else { - [self.coreHelper logDebugMessage:@"createCoreActionForActionStruct did not convert unknown backspace, expected value =%lu, expected type =%u", backspace.expected_value, backspace.expected_type]; - } - break; - } - case KM_CORE_IT_PERSIST_OPT: { - [self.coreHelper logDebugMessage:@"***createCoreActionForActionStruct Persist Options encountered."]; - km_core_option_item const * option = actionStruct->option; - NSString *keyString = [self.coreHelper createNSStringFromUnicharString:option->key]; - NSString *valueString = [self.coreHelper createNSStringFromUnicharString:option->value]; - - [self.coreHelper logDebugMessage:@"***createCoreActionForActionStruct converted Persist Options, key = %@, value = %@, scope = %d", keyString, valueString, option->scope]; - - action = [[CoreAction alloc] initPersistOptionAction:keyString value:valueString scope:option->scope]; - break; - } - case KM_CORE_IT_EMIT_KEYSTROKE: { - action = [[CoreAction alloc] initWithType: EmitKeystrokeAction actionContent:@"" backspaceCount:0 key:@"" value:@"" scope:0]; - break; - } - case KM_CORE_IT_INVALIDATE_CONTEXT: { - action = [[CoreAction alloc] initWithType: InvalidateContextAction actionContent:@"" backspaceCount:0 key:@"" value:@"" scope:0]; - break; - } - case KM_CORE_IT_CAPSLOCK: { - action = [[CoreAction alloc] initWithType: CapsLockAction actionContent:@"" backspaceCount:0 key:@"" value:@"" scope:0]; - break; - } - default: { - NSLog(@"createCoreActionForActionStruct unrecognized type of km_core_action_item = %u\n", actionStruct->type); - } - } - return action; -} - -(void)clearContextUsingCore { km_core_state_context_clear(self.coreState); [self.coreHelper logDebugMessage:@"km_core_state_context_clear called"]; From 40f0839a4afe6c75377842b4ab66d408b319b153 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 17 Jan 2024 11:03:42 +0700 Subject: [PATCH 31/45] chore(core): remove bogus test_color.h inclusion --- common/include/test_color.h | 2 +- core/src/actions_normalize.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/include/test_color.h b/common/include/test_color.h index 6837b89327..fdf2f69586 100644 --- a/common/include/test_color.h +++ b/common/include/test_color.h @@ -23,7 +23,7 @@ enum ansi_code { BRIGHT_RED = 196 }; -bool enabled = false; +bool enabled = false; // TODO: move to test_color.c because test_color.h cannot be #included in more than 1 file in a project otherwise. class fg { ansi_code code; diff --git a/core/src/actions_normalize.cpp b/core/src/actions_normalize.cpp index 09cd857d80..ecb9077bd1 100644 --- a/core/src/actions_normalize.cpp +++ b/core/src/actions_normalize.cpp @@ -9,7 +9,7 @@ #include #include -#include +#include #include "context.hpp" #include "action.hpp" #include "state.hpp" From 7218f93ca9707d920cbae337e636be27b2ce428d Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 17 Jan 2024 12:07:38 +0700 Subject: [PATCH 32/45] chore(core): test typos --- core/tests/unit/kmnkbd/test_actions_normalize.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/tests/unit/kmnkbd/test_actions_normalize.cpp b/core/tests/unit/kmnkbd/test_actions_normalize.cpp index 21428331af..2b11b1f9fa 100644 --- a/core/tests/unit/kmnkbd/test_actions_normalize.cpp +++ b/core/tests/unit/kmnkbd/test_actions_normalize.cpp @@ -102,7 +102,7 @@ void test( int actions_code_points_to_delete, const std::u32string actions_output, - const int expected_delete, + const unsigned int expected_delete, const std::u32string expected_output, const km_core_cp *expected_final_app_context ) { @@ -124,7 +124,7 @@ void test( auto actual_final_app_context = get_context_as_string(km_core_state_app_context(test_state)); auto actual_final_app_context_string = std::u16string(actual_final_app_context); - auto expected_final_app_context_string = std::u16string(final_app_context); + auto expected_final_app_context_string = std::u16string(expected_final_app_context); std::cout << " final app context: actual: |" << actual_final_app_context_string << "| expected: |" << expected_final_app_context_string << "|" << std::endl; assert(actual_final_app_context_string == expected_final_app_context_string); delete [] actual_final_app_context; From 39f081391e1b57bb40cfc28c54910edc48b54846 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 17 Jan 2024 13:05:11 +0700 Subject: [PATCH 33/45] chore(core): dedup set_context_from_string --- .../km_core_state_context_set_if_needed.cpp | 12 +++++----- .../unit/kmnkbd/test_actions_normalize.cpp | 23 ++----------------- 2 files changed, 8 insertions(+), 27 deletions(-) diff --git a/core/src/km_core_state_context_set_if_needed.cpp b/core/src/km_core_state_context_set_if_needed.cpp index 72b80f6f51..a6f6fde9c0 100644 --- a/core/src/km_core_state_context_set_if_needed.cpp +++ b/core/src/km_core_state_context_set_if_needed.cpp @@ -31,7 +31,7 @@ using namespace km::core; bool should_normalize(km_core_state *state); bool is_context_valid(km_core_cp const * context, km_core_cp const * cached_context); km_core_cp* get_context_as_string(km_core_context *context); -bool set_context_from_string(km_core_context *context, km_core_cp const *new_context); +km_core_status set_context_from_string(km_core_context *context, km_core_cp const *new_context); bool do_normalize_nfd(km_core_cp const * src, std::u16string &dst); km_core_context_status do_fail(km_core_context *app_context, km_core_context *cached_context, const char* error); @@ -63,7 +63,7 @@ km_core_context_status km_core_state_context_set_if_needed( // We replace the internal app context with the passed-in application context - if(!set_context_from_string(app_context, new_app_context)) { + if(set_context_from_string(app_context, new_app_context) != KM_CORE_STATUS_OK) { return do_fail(app_context, cached_context, "could not set new app context"); } @@ -84,7 +84,7 @@ km_core_context_status km_core_state_context_set_if_needed( // TODO: #10100 will alter how we replace the cached context here -- maintaining // markers as far as possible - if(!set_context_from_string(cached_context, new_cached_context)) { + if(set_context_from_string(cached_context, new_cached_context) != KM_CORE_STATUS_OK) { return do_fail(app_context, cached_context, "could not set new cached context"); } @@ -170,18 +170,18 @@ km_core_cp* get_context_as_string(km_core_context *context) { /** * Updates the context from the new_context km_core_cp string */ -bool set_context_from_string(km_core_context *context, km_core_cp const *new_context) { +km_core_status set_context_from_string(km_core_context *context, km_core_cp const *new_context) { km_core_context_item* new_context_items = nullptr; km_core_status status = km_core_context_items_from_utf16(new_context, &new_context_items); if (status != KM_CORE_STATUS_OK) { - return false; + return status; } km_core_context_set(context, new_context_items); km_core_context_items_dispose(new_context_items); - return true; + return KM_CORE_STATUS_OK; } /** diff --git a/core/tests/unit/kmnkbd/test_actions_normalize.cpp b/core/tests/unit/kmnkbd/test_actions_normalize.cpp index 2b11b1f9fa..6434aa4381 100644 --- a/core/tests/unit/kmnkbd/test_actions_normalize.cpp +++ b/core/tests/unit/kmnkbd/test_actions_normalize.cpp @@ -14,6 +14,8 @@ #include #include "../emscripten_filesystem.h" +// TODO: These helpers are in km_core_state_context_set_if_needed.cpp and +// should be refactored into a helper module km_core_status set_context_from_string(km_core_context *context, km_core_cp const *new_context); km_core_cp* get_context_as_string(km_core_context *context); @@ -295,24 +297,3 @@ int main(int argc, char *argv []) { run_tests(); } - - -// TODO: These helpers are copied from km_core_state_context_set_if_needed and -// should be refactored into a helper module - -/** - * Updates the context from the new_context km_core_cp string - */ -km_core_status set_context_from_string(km_core_context *context, km_core_cp const *new_context) { - km_core_context_item* new_context_items = nullptr; - - km_core_status status = km_core_context_items_from_utf16(new_context, &new_context_items); - if (status != KM_CORE_STATUS_OK) { - return status; - } - - km_core_context_set(context, new_context_items); - km_core_context_items_dispose(new_context_items); - - return KM_CORE_STATUS_OK; -} From f48461545004dd2ad3cac035802cf6c73196bfb5 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Wed, 17 Jan 2024 16:12:51 -0600 Subject: [PATCH 34/45] =?UTF-8?q?chore(core):=20ldml=20don't=20use=20queue?= =?UTF-8?q?=20action=20functions=20=F0=9F=8C=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/include/keyman/keyman_core_api.h | 1 + core/src/ldml/ldml_processor.cpp | 14 +++++--------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/core/include/keyman/keyman_core_api.h b/core/include/keyman/keyman_core_api.h index 3e4451f612..7d1feb078c 100644 --- a/core/include/keyman/keyman_core_api.h +++ b/core/include/keyman/keyman_core_api.h @@ -146,6 +146,7 @@ enum km_core_status_codes { KM_CORE_STATUS_INSUFFICENT_BUFFER = 5, KM_CORE_STATUS_INVALID_UTF = 6, KM_CORE_STATUS_INVALID_KEYBOARD = 7, + KM_CORE_STATUS_NOT_IMPLEMENTED = 8, KM_CORE_STATUS_OS_ERROR = 0x80000000 }; diff --git a/core/src/ldml/ldml_processor.cpp b/core/src/ldml/ldml_processor.cpp index 42baaed0a4..90dc4bcb9e 100644 --- a/core/src/ldml/ldml_processor.cpp +++ b/core/src/ldml/ldml_processor.cpp @@ -153,11 +153,9 @@ km_core_status ldml_processor::process_queued_actions( km_core_state *state ) { - assert(state); - if (!state) - return KM_CORE_STATUS_INVALID_ARGUMENT; - // TODO Implement - return KM_CORE_STATUS_OK; + /* Not implemented. Only used by IMX. */ + assert(false); + return KM_CORE_STATUS_NOT_IMPLEMENTED; } bool ldml_processor::queue_action( @@ -165,10 +163,8 @@ bool ldml_processor::queue_action( km_core_action_item const* action_item ) { - assert(state); - assert(action_item); - if ((!state) || (!action_item)) - return false; + /* Not implemented. Only used by IMX. */ + assert(false); return false; } From 792329fba0f55a31334a8f53930afb5693f275e4 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Wed, 17 Jan 2024 17:30:48 -0600 Subject: [PATCH 35/45] chore(core): tag unused args --- core/src/ldml/ldml_processor.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/ldml/ldml_processor.cpp b/core/src/ldml/ldml_processor.cpp index 90dc4bcb9e..cfe0b1a85f 100644 --- a/core/src/ldml/ldml_processor.cpp +++ b/core/src/ldml/ldml_processor.cpp @@ -151,7 +151,7 @@ bool ldml_processor::is_kmxplus_file(path const & kb_path, std::vector& km_core_status ldml_processor::process_queued_actions( - km_core_state *state + km_core_state *_kmn_unused(state) ) { /* Not implemented. Only used by IMX. */ assert(false); @@ -159,8 +159,8 @@ ldml_processor::process_queued_actions( } bool ldml_processor::queue_action( - km_core_state * state, - km_core_action_item const* action_item + km_core_state * _kmn_unused(state), + km_core_action_item const* _kmn_unused(action_item) ) { /* Not implemented. Only used by IMX. */ From 823120b0c45d47e60775ad897703482c63d7c961 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Thu, 18 Jan 2024 08:25:15 +0700 Subject: [PATCH 36/45] chore(core): simplify input assumptions for actions_normalize Per discussion in #10422, we can assume that input cached_context is always NFD. However input actions->output may not start at a normalization boundary, so we still need to backtrack to a normalization boundary in order to get our NFC output. But cached_context never need change. This makes no change to the algorithm, but tweaks some of the unit tests to adhere to this input assumption. Note: we could consider adding a debug assertion that cached_context is NFD. --- core/src/actions_normalize.cpp | 18 +++++------- .../unit/kmnkbd/test_actions_normalize.cpp | 29 +++++++++++++------ 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/core/src/actions_normalize.cpp b/core/src/actions_normalize.cpp index ecb9077bd1..389aefa4ca 100644 --- a/core/src/actions_normalize.cpp +++ b/core/src/actions_normalize.cpp @@ -62,9 +62,9 @@ bool km::core::actions_normalize( /* The code_points_to_delete value at this point is in NFD. The cached_context is in NFD and has already been updated by the keyboard processor to the - expected result of the action, so we need to remove the output from a copy - of the cached_context to start, in order to get it to the same position as - the app_context. + expected result of the action, so we need to remove the output from a string + copy of the cached_context to start, in order to get it to the same position + as the app_context. The app_context is in NFU. We need to figure out how many characters to remove from the end of app_context in order to correctly normalize across @@ -78,11 +78,6 @@ bool km::core::actions_normalize( cached_context. */ - // TODO: That description above is not quite right. cached_context is not - // guaranteed to be normalized across the transform boundary, because its - // output was simply appended to the existing context. MUCH PAIN COMING WITH - // THIS, because cached_context includes markers. Hence, blocked by #10369. - /* Initialization */ @@ -125,8 +120,10 @@ bool km::core::actions_normalize( cached_context_string.remove(n); /* - Now, look for a normalization boundary at the intersection of the - cached_context and the output + While cached_context is guaranteed to be normalized, actions->output may not + start at a normalization boundary. In order to achieve the correct NFC + normalization in our output, we now need to look for a normalization + boundary prior to the intersection of the cached_context and the output. */ while(n > 0 && output[0] && !nfd->hasBoundaryBefore(output[0])) { @@ -160,7 +157,6 @@ bool km::core::actions_normalize( its normalized form matches the cached_context normalized form. */ - while(app_context_string.length()) { icu::UnicodeString app_context_nfd; nfd->normalize(app_context_string, app_context_nfd, icu_status); diff --git a/core/tests/unit/kmnkbd/test_actions_normalize.cpp b/core/tests/unit/kmnkbd/test_actions_normalize.cpp index 6434aa4381..dada129cfa 100644 --- a/core/tests/unit/kmnkbd/test_actions_normalize.cpp +++ b/core/tests/unit/kmnkbd/test_actions_normalize.cpp @@ -75,10 +75,8 @@ void setup(const km_core_cp *app_context, const km_core_cp *cached_context, int * @param initial_app_context the app context stored in the state, * _before_ transform is applied -- NFU * @param final_cached_context cached context _after_ actions have been - * applied -- NFD except on the boundary of - * the transform, where reorder may be - * required (essentially, this is - * initial_cached_context - + * applied -- guaranteed NFD (essentially, + * this is initial_cached_context - * actions_code_points_to_delete + * actions_output) * @param actions_code_points_to_delete number of NFD code points that the @@ -223,10 +221,10 @@ void run_tests() { ); test( - "Two backspaces for NFD converts into one char in NFC (ê) and recombine", + "One backspace for NFD converts into one char in NFC (ê) and recombine", /* app context pre transform: */ u"abcê", /* cached context post transform: */ u"abce\u0323\u0302", - /* action del, output: */ 2, U"e\u0323\u0302", // NFD input; delete 2: e and \u0302 + /* action del, output: */ 1, U"\u0323\u0302", // NFD input; delete 1: \u0302 // ---- results ---- /* action del, output: */ 1, U"ệ", // NFC output; delete 1: ê /* app_context: */ u"abcệ" @@ -238,7 +236,7 @@ void run_tests() { "Avoid editing too far back in context when finding normalization boundary", /* app context pre transform: */ u"a\u0300bcê", /* cached context post transform: */ u"a\u0300bce\u0323\u0302", - /* action del, output: */ 2, U"e\u0323\u0302", // NFD input; delete 2: e and \u0302 + /* action del, output: */ 1, U"\u0323\u0302", // NFD input; delete 1: \u0302 // ---- results ---- /* action del, output: */ 1, U"ệ", // NFC output; delete 1: ê /* app_context: */ u"a\u0300bcệ" @@ -248,13 +246,26 @@ void run_tests() { test( "Normalizable letters at start of context", /* app context pre transform: */ u"\u0300", - /* cached context post transform: */ u"\u0300\u0323\u0302", - /* action del, output: */ 0, U"\u0323\u0302", // NFD input; + /* cached context post transform: */ u"\u0323\u0300\u0302", + /* action del, output: */ 1, U"\u0323\u0300\u0302", // NFD input; // ---- results ---- /* action del, output: */ 1, U"\u0323\u0300\u0302", // NFC output is still decomposed because there is no base /* app_context: */ u"\u0323\u0300\u0302" ); + // Modifies the base as well as diacritic + + test( + "Two backspaces for NFD converts into one char in NFC (ê) and recombine", + /* app context pre transform: */ u"abcê", + /* cached context post transform: */ u"abca\u0323\u0302", + /* action del, output: */ 2, U"a\u0323\u0302", // NFD input; delete 2: e\u0302 + // ---- results ---- + /* action del, output: */ 1, U"ậ", // NFC output; delete 1: ê + /* app_context: */ u"abcậ" + ); + + // TODO: surrogate pair tests // TODO: add check for updated cached_context once #10369 lands and we apply // it here From 4ab47ea435d14f3e1ad820b7edd3d755f7b7073c Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Thu, 18 Jan 2024 08:39:53 +0700 Subject: [PATCH 37/45] chore(core): further debug assertions for actions_normalize --- core/src/actions_normalize.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/src/actions_normalize.cpp b/core/src/actions_normalize.cpp index 389aefa4ca..bb6eeab917 100644 --- a/core/src/actions_normalize.cpp +++ b/core/src/actions_normalize.cpp @@ -108,6 +108,13 @@ bool km::core::actions_normalize( } int nfu_to_delete = 0; + /* + Further debug assertion of inputs + */ + + assert(nfd->isNormalized(output, icu_status) && U_SUCCESS(icu_status)); + assert(nfd->isNormalized(cached_context_string, icu_status) && U_SUCCESS(icu_status)); + /* The keyboard processor will have updated the cached_context already, applying the transform to it, so we need to rewind this. Remove the output From 425ebe36b0253107a3ff7287394c84e34f8533d7 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Thu, 18 Jan 2024 08:52:59 +0700 Subject: [PATCH 38/45] feat(core): synchronize app_context for non-normalizing keyboardprocessors Fixes #10421. --- core/src/action.hpp | 6 ++++++ core/src/actions_normalize.cpp | 34 +++++++++++++++++++++++++++++++++ core/src/km_core_action_api.cpp | 6 ++++++ 3 files changed, 46 insertions(+) diff --git a/core/src/action.hpp b/core/src/action.hpp index 40b9316a64..e7bee85db0 100644 --- a/core/src/action.hpp +++ b/core/src/action.hpp @@ -23,5 +23,11 @@ namespace core /* in, out */ km_core_context *app_context, /* in, out */ km_core_actions *actions ); + + bool actions_update_app_context( + /* in */ km_core_context const *cached_context, + /* in, out */ km_core_context *app_context + ); + } // namespace core } // namespace km diff --git a/core/src/actions_normalize.cpp b/core/src/actions_normalize.cpp index bb6eeab917..cf9567640a 100644 --- a/core/src/actions_normalize.cpp +++ b/core/src/actions_normalize.cpp @@ -286,3 +286,37 @@ km_core_usv *unicode_string_to_usv(icu::UnicodeString& src) { dst[src.length()] = 0; return dst; } + + + +/** + * Refresh app_context to match the cached_context + * + * @param cached_context the cached context, in NFU, after transform has been + * applied to it by the keyboard processor + * @param app_context the app context, in NFU; transform has not been + * applied, and will effectively be applied by this + * function + * @return true on success, false on failure + */ +bool km::core::actions_update_app_context( + /* in */ km_core_context const *cached_context, + /* in, out */ km_core_context *app_context +) { + // We simply copy the cached_context to the app_context + km_core_status status = KM_CORE_STATUS_OK; + km_core_context_item *items = nullptr; + + if((status = km_core_context_get(cached_context, &items)) != KM_CORE_STATUS_OK) { + DebugLog("km_core_context_get failed with %d", status); + return false; + } + + if((status = km_core_context_set(app_context, items)) != KM_CORE_STATUS_OK) { + DebugLog("km_core_context_set failed with %d", status); + } + + delete [] items; + + return status == KM_CORE_STATUS_OK; +} \ No newline at end of file diff --git a/core/src/km_core_action_api.cpp b/core/src/km_core_action_api.cpp index 4d9a57c371..097f5a3e2b 100644 --- a/core/src/km_core_action_api.cpp +++ b/core/src/km_core_action_api.cpp @@ -41,6 +41,12 @@ km_core_actions const * km_core_state_get_actions( km_core_actions_dispose(result); return nullptr; } + } else { + // For all other keyboard processors, we just copy the cached_context to the app_context + if(!actions_update_app_context(km_core_state_context(state), km_core_state_app_context(state))) { + km_core_actions_dispose(result); + return nullptr; + } } return result; } From 38783947878d099056469119125dc18d232ac8fe Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Thu, 18 Jan 2024 08:53:55 +0700 Subject: [PATCH 39/45] chore(core): remove todo comment --- core/src/actions_normalize.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/core/src/actions_normalize.cpp b/core/src/actions_normalize.cpp index cf9567640a..a83e76c590 100644 --- a/core/src/actions_normalize.cpp +++ b/core/src/actions_normalize.cpp @@ -25,10 +25,6 @@ #include "unicode/unistr.h" #include "unicode/normalizer2.h" -// ******************************************************************************** -// TODO: if we don't apply normalization, we should still fixup the app_context -// ******************************************************************************** - // forward declarations icu::UnicodeString context_items_to_unicode_string(km_core_context const *context); From b3261b8dc49746eb2fedd785e20c92c7bf2d7ebf Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Thu, 18 Jan 2024 09:26:58 +0700 Subject: [PATCH 40/45] refactor(core): move context helpers to a new module Fixes #10423. --- core/src/context.hpp | 5 ++ core/src/context_helpers.cpp | 58 +++++++++++++++++++ .../km_core_state_context_set_if_needed.cpp | 47 --------------- core/src/meson.build | 1 + .../unit/kmnkbd/test_actions_normalize.cpp | 6 +- 5 files changed, 65 insertions(+), 52 deletions(-) create mode 100644 core/src/context_helpers.cpp diff --git a/core/src/context.hpp b/core/src/context.hpp index 36831f6d14..10ad878e9e 100644 --- a/core/src/context.hpp +++ b/core/src/context.hpp @@ -39,6 +39,11 @@ void context::push_marker(uint32_t marker) { emplace_back(km_core_context_item { KM_CORE_CT_MARKER, {0,}, {marker} }); } +// Context helper functions + +km_core_cp* get_context_as_string(km_core_context *context); +km_core_status set_context_from_string(km_core_context *context, km_core_cp const *new_context); + } // namespace core } // namespace km diff --git a/core/src/context_helpers.cpp b/core/src/context_helpers.cpp new file mode 100644 index 0000000000..8e6d2a1920 --- /dev/null +++ b/core/src/context_helpers.cpp @@ -0,0 +1,58 @@ +/* + Copyright: © 2018-2024 SIL International. + Description: Helper functions for context data type conversions + Create Date: 18 Jan 2024 + Authors: Marc Durdin + History: 18 Jan 2024 - MCD - Refactor from km_core_state_context_set_if_needed.cpp +*/ +#include + +#include "keyman_core.h" +#include "context.hpp" + +using namespace km::core; + +/** + * Retrieves the context as a km_core_cp string, dropping markers + */ +km_core_cp* km::core::get_context_as_string(km_core_context *context) { + size_t buf_size = 0; + km_core_context_item* context_items = nullptr; + + if(km_core_context_get(context, &context_items) != KM_CORE_STATUS_OK) { + return nullptr; + } + + if(km_core_context_items_to_utf16(context_items, nullptr, &buf_size) != KM_CORE_STATUS_OK) { + km_core_context_items_dispose(context_items); + return nullptr; + } + + km_core_cp *app_context_string = new km_core_cp[buf_size]; + + km_core_status status = km_core_context_items_to_utf16(context_items, app_context_string, &buf_size); + km_core_context_items_dispose(context_items); + + if(status != KM_CORE_STATUS_OK) { + return nullptr; + } + + return app_context_string; +} + +/** + * Updates the context from the new_context km_core_cp string + */ +km_core_status km::core::set_context_from_string(km_core_context *context, km_core_cp const *new_context) { + km_core_context_item* new_context_items = nullptr; + + km_core_status status = km_core_context_items_from_utf16(new_context, &new_context_items); + if (status != KM_CORE_STATUS_OK) { + return status; + } + + km_core_context_set(context, new_context_items); + km_core_context_items_dispose(new_context_items); + + return KM_CORE_STATUS_OK; +} diff --git a/core/src/km_core_state_context_set_if_needed.cpp b/core/src/km_core_state_context_set_if_needed.cpp index a6f6fde9c0..b5ac9f7a44 100644 --- a/core/src/km_core_state_context_set_if_needed.cpp +++ b/core/src/km_core_state_context_set_if_needed.cpp @@ -30,8 +30,6 @@ using namespace km::core; bool should_normalize(km_core_state *state); bool is_context_valid(km_core_cp const * context, km_core_cp const * cached_context); -km_core_cp* get_context_as_string(km_core_context *context); -km_core_status set_context_from_string(km_core_context *context, km_core_cp const *new_context); bool do_normalize_nfd(km_core_cp const * src, std::u16string &dst); km_core_context_status do_fail(km_core_context *app_context, km_core_context *cached_context, const char* error); @@ -139,51 +137,6 @@ bool is_context_valid(km_core_cp const * new_app_context, km_core_cp const * app return true; } -/** - * Retrieves the context as a km_core_cp string, dropping markers - */ -km_core_cp* get_context_as_string(km_core_context *context) { - size_t buf_size = 0; - km_core_context_item* context_items = nullptr; - - if(km_core_context_get(context, &context_items) != KM_CORE_STATUS_OK) { - return nullptr; - } - - if(km_core_context_items_to_utf16(context_items, nullptr, &buf_size) != KM_CORE_STATUS_OK) { - km_core_context_items_dispose(context_items); - return nullptr; - } - - km_core_cp *app_context_string = new km_core_cp[buf_size]; - - km_core_status status = km_core_context_items_to_utf16(context_items, app_context_string, &buf_size); - km_core_context_items_dispose(context_items); - - if(status != KM_CORE_STATUS_OK) { - return nullptr; - } - - return app_context_string; -} - -/** - * Updates the context from the new_context km_core_cp string - */ -km_core_status set_context_from_string(km_core_context *context, km_core_cp const *new_context) { - km_core_context_item* new_context_items = nullptr; - - km_core_status status = km_core_context_items_from_utf16(new_context, &new_context_items); - if (status != KM_CORE_STATUS_OK) { - return status; - } - - km_core_context_set(context, new_context_items); - km_core_context_items_dispose(new_context_items); - - return KM_CORE_STATUS_OK; -} - /** * Normalize the input string using ICU */ diff --git a/core/src/meson.build b/core/src/meson.build index 6f96041dac..9943c2fd07 100644 --- a/core/src/meson.build +++ b/core/src/meson.build @@ -44,6 +44,7 @@ endif kmx_files = files( 'actions_normalize.cpp', 'action.cpp', + 'context_helpers.cpp', 'option.cpp', 'keyboard.cpp', 'state.cpp', diff --git a/core/tests/unit/kmnkbd/test_actions_normalize.cpp b/core/tests/unit/kmnkbd/test_actions_normalize.cpp index dada129cfa..741b55a9c5 100644 --- a/core/tests/unit/kmnkbd/test_actions_normalize.cpp +++ b/core/tests/unit/kmnkbd/test_actions_normalize.cpp @@ -10,15 +10,11 @@ #include "path.hpp" #include "action.hpp" +#include "context.hpp" #include #include "../emscripten_filesystem.h" -// TODO: These helpers are in km_core_state_context_set_if_needed.cpp and -// should be refactored into a helper module -km_core_status set_context_from_string(km_core_context *context, km_core_cp const *new_context); -km_core_cp* get_context_as_string(km_core_context *context); - km_core_option_item test_env_opts[] = { KM_CORE_OPTIONS_END From 63574f85137896fe8d3dba8aab0155e503a56c7a Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Thu, 18 Jan 2024 09:31:13 +0700 Subject: [PATCH 41/45] chore(core): add assertions to context helpers --- core/src/context_helpers.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/core/src/context_helpers.cpp b/core/src/context_helpers.cpp index 8e6d2a1920..936f68be52 100644 --- a/core/src/context_helpers.cpp +++ b/core/src/context_helpers.cpp @@ -16,6 +16,11 @@ using namespace km::core; * Retrieves the context as a km_core_cp string, dropping markers */ km_core_cp* km::core::get_context_as_string(km_core_context *context) { + assert(context != nullptr); + if(context == nullptr) { + return nullptr; + } + size_t buf_size = 0; km_core_context_item* context_items = nullptr; @@ -44,6 +49,12 @@ km_core_cp* km::core::get_context_as_string(km_core_context *context) { * Updates the context from the new_context km_core_cp string */ km_core_status km::core::set_context_from_string(km_core_context *context, km_core_cp const *new_context) { + assert(context != nullptr); + assert(new_context != nullptr); + if(context == nullptr || new_context == nullptr) { + return KM_CORE_STATUS_INVALID_ARGUMENT; + } + km_core_context_item* new_context_items = nullptr; km_core_status status = km_core_context_items_from_utf16(new_context, &new_context_items); From 8b3c2475203a4da616173d516958a361f6626a85 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Thu, 18 Jan 2024 09:48:29 +0700 Subject: [PATCH 42/45] chore(core): add surrogate pair tests to test_actions_normalize Fixes #10424. --- .../unit/kmnkbd/test_actions_normalize.cpp | 33 +++++++++++++++++-- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/core/tests/unit/kmnkbd/test_actions_normalize.cpp b/core/tests/unit/kmnkbd/test_actions_normalize.cpp index 741b55a9c5..dbe298e08c 100644 --- a/core/tests/unit/kmnkbd/test_actions_normalize.cpp +++ b/core/tests/unit/kmnkbd/test_actions_normalize.cpp @@ -261,10 +261,37 @@ void run_tests() { /* app_context: */ u"abcậ" ); + // surrogate pair tests - // TODO: surrogate pair tests - // TODO: add check for updated cached_context once #10369 lands and we apply - // it here + test( + "Surrogate pair in context", + /* app context pre transform: */ u"abc\U0001F607ê", + /* cached context post transform: */ u"abc\U0001F607a\u0323\u0302", + /* action del, output: */ 2, U"a\u0323\u0302", + // ---- results ---- + /* action del, output: */ 1, U"ậ", + /* app_context: */ u"abc\U0001F607ậ" + ); + + test( + "Surrogate pair in output", + /* app context pre transform: */ u"abc", + /* cached context post transform: */ u"abc\U0001F607", + /* action del, output: */ 0, U"\U0001F607", + // ---- results ---- + /* action del, output: */ 0, U"\U0001F607", + /* app_context: */ u"abc\U0001F607" + ); + + test( + "Surrogate pairs in both context and output", + /* app context pre transform: */ u"a\U0001F607bcê", + /* cached context post transform: */ u"a\U0001F607bca\U0001F60E", + /* action del, output: */ 2, U"a\U0001F60E", + // ---- results ---- + /* action del, output: */ 1, U"a\U0001F60E", + /* app_context: */ u"a\U0001F607bca\U0001F60E" + ); } //------------------------------------------------------------------------------------- From eece9716b7dd2fab64956a46f969e3c463f33ba9 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Thu, 18 Jan 2024 10:42:18 +0700 Subject: [PATCH 43/45] refactor(core): additional comments --- core/include/keyman/keyman_core_api_context.h | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/core/include/keyman/keyman_core_api_context.h b/core/include/keyman/keyman_core_api_context.h index 26bd2f25bc..5934740214 100644 --- a/core/include/keyman/keyman_core_api_context.h +++ b/core/include/keyman/keyman_core_api_context.h @@ -32,14 +32,14 @@ extern "C" typedef struct km_core_context km_core_context; // ---------------------------------------------------------------------------------- -// Context APIs are now available only to the keyboard debugger and Core unit +// Context APIs are now available only to the keyboard debugger, IMX, and Core unit // tests (17.0) // ---------------------------------------------------------------------------------- /* ### Context -The context is the text prior to the insertion point (caret, cursor). -The context is constructed by the Platform layer, typically by interrogating the +The context is the text prior to the insertion point (caret, cursor). The +context is constructed by the Platform layer, typically by interrogating the Client Application. The context will be updated by the engine for keystroke events. If the Platform layer code caches the context, the context should be reset when a context state change is detected. Context state changes can occur @@ -54,6 +54,14 @@ state change is detected. Markers are always controlled by the Engine. Contexts are always owned by their state. They may be set to a list of context_items or interrogated for their current list of context items. + +Core maintains and caches the context. Engine can update the context with +`km_core_state_context_set_if_needed` and `km_core_state_context_clear`. These +two functions are available in keyman_core_api.h. + +The Keyboard Debugger in Keyman Developer, and IMX in Keyman for Windows, make +use of the context functionality in this header, but these functions should not +be used in other places. ```c */ enum km_core_context_type { From ae00eda4a0766f78ddd7cecec42e8d028067d9c1 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 18 Jan 2024 15:47:15 -0600 Subject: [PATCH 44/45] =?UTF-8?q?chore(core):=20ldml=20shuffle=20some=20de?= =?UTF-8?q?ck=20chairs=20=F0=9F=8C=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - move the emit_ and context functions into an ldml_event_state object For #10410 --- core/src/ldml/ldml_processor.cpp | 106 ++++++++++++++-------- core/src/ldml/ldml_processor.hpp | 145 ++++++++++++++++++++----------- 2 files changed, 164 insertions(+), 87 deletions(-) diff --git a/core/src/ldml/ldml_processor.cpp b/core/src/ldml/ldml_processor.cpp index cfe0b1a85f..49cee6f561 100644 --- a/core/src/ldml/ldml_processor.cpp +++ b/core/src/ldml/ldml_processor.cpp @@ -174,59 +174,68 @@ ldml_processor::process_event( km_core_virtual_key vk, uint16_t modifier_state, uint8_t is_key_down, - uint16_t _kmn_unused(event_flags) + uint16_t event_flags ) { assert(state); if (!state) return KM_CORE_STATUS_INVALID_ARGUMENT; - try { - // At the start of every process_event always clear the action_items - state->actions().clear(); + // this will hold the parameters, and our response + ldml_event_state ldml_state(state, vk, modifier_state, is_key_down, event_flags); + ldml_state.clear(); + try { if (!is_key_down) { - process_key_up(state, vk, modifier_state); + process_key_up(ldml_state); } else { switch (vk) { // Currently, only one VK gets spoecial treatment. // Special handling for backspace VK case KM_CORE_VKEY_BKSP: - process_backspace(state); + process_backspace(ldml_state); break; default: // all other VKs - process_key_down(state, vk, modifier_state); + process_key_down(ldml_state); } // end of switch } // end of normal processing // all key-up and key-down events end up here. - state->actions().commit(); // always commit + // commit the ldml state into the core state + ldml_state.commit(); return KM_CORE_STATUS_OK; } catch (std::bad_alloc &) { // out of memory, clean up and get out - state->actions().clear(); + ldml_state.clear(); + // no actions will be set return KM_CORE_STATUS_NO_MEM; } } void -ldml_processor::process_key_up(km_core_state *state, km_core_virtual_key _kmn_unused(vk), uint16_t _kmn_unused(modifier_state)) +ldml_processor::process_key_up(ldml_event_state &ldml_state) const { // TODO-LDML: Implement caps lock handling - state->actions().clear(); // TODO-LDML: Why is clear here? + ldml_state.clear(); } void -ldml_processor::process_backspace(km_core_state *state) const { +ldml_processor::process_backspace(ldml_event_state &ldml_state) const { if (!!bksp_transforms) { - // process with an empty string - auto matchedContext = process_output(state, std::u32string(), bksp_transforms.get()); + // process with an empty string voa the bksp transforms + auto matchedContext = process_output(ldml_state, std::u32string(), bksp_transforms.get()); if (matchedContext > 0) { return; // The transform took care of the backspacing. } // else, fall through to default processing below. } + ldml_state.emit_backspace(); +} + +void ldml_event_state::emit_backspace() { + // this is called from user-initiated backspace, not internal backspacing. + // Find out what the last actual character was and remove it. // attempt to get the last char // TODO-LDML: emoji backspace @@ -239,6 +248,7 @@ ldml_processor::process_backspace(km_core_state *state) const { } else if ((*end).type == KM_CORE_BT_MARKER) { state->actions().push_backspace(KM_CORE_BT_MARKER, (*end).marker); state->context().pop_back(); + // TODO-LDML: fall through here? } } /* @@ -253,41 +263,41 @@ ldml_processor::process_backspace(km_core_state *state) const { } void -ldml_processor::process_key_down(km_core_state *state, km_core_virtual_key vk, uint16_t modifier_state) const { +ldml_processor::process_key_down(ldml_event_state &ldml_state) const { // Look up the key bool found = false; - const std::u16string key_str = keys.lookup(vk, modifier_state, found); + const std::u16string key_str = keys.lookup(ldml_state.get_vk(), ldml_state.get_modifier_state(), found); if (!found) { // no key was found, so pass the keystroke on to the Engine - emit_invalidate_passthrough_keystroke(state, vk, modifier_state); + ldml_state.emit_invalidate_passthrough_keystroke(); } else if (!key_str.empty()) { - process_key_string(state, key_str); + process_key_string(ldml_state, key_str); } // else no action: It's a gap or gap-like key. } void -ldml_processor::process_key_string(km_core_state *state, const std::u16string &key_str) const { +ldml_processor::process_key_string(ldml_event_state &ldml_state, const std::u16string &key_str) const { // We know that key_str is not empty per the caller. assert(!key_str.empty()); // we convert the keys str to UTF-32 here instead of using the emit_text() overload // so that we don't have to reconvert it inside the transform code. std::u32string key_str32 = kmx::u16string_to_u32string(key_str); - (void)process_output(state, key_str32, transforms.get()); + (void)process_output(ldml_state, key_str32, transforms.get()); } -size_t ldml_processor::process_output(km_core_state *state, const std::u32string &str, ldml::transforms *with_transforms) const { +size_t ldml_processor::process_output(ldml_event_state &ldml_state, const std::u32string &str, ldml::transforms *with_transforms) const { std::u32string nfd_str = str; assert(ldml::normalize_nfd_markers(nfd_str)); // TODO-LDML: else fail? // extract context string, in NFD std::u32string old_ctxtstr_nfd; - (void)context_to_string(state, old_ctxtstr_nfd, true); + (void)ldml_state.context_to_string(old_ctxtstr_nfd, true); assert(ldml::normalize_nfd_markers(old_ctxtstr_nfd)); // TODO-LDML: else fail? // context string in NFD std::u32string ctxtstr; - (void)context_to_string(state, ctxtstr, true); // with markers + (void)ldml_state.context_to_string(ctxtstr, true); // TODO-LDML: remove this second call // add the newly added key output to ctxtstr ctxtstr.append(nfd_str); assert(ldml::normalize_nfd_markers(ctxtstr)); // TODO-LDML: else fail? @@ -349,20 +359,21 @@ size_t ldml_processor::process_output(km_core_state *state, const std::u32string // FIRST drop the old suffix. Note: this mutates old_ctxtstr_changed. // see remove_text() docs, this PUSHes actions, POPs context items, and TRIMS the string. - remove_text(state, old_ctxtstr_changed, old_ctxtstr_changed.length()); + ldml_state.remove_text(old_ctxtstr_changed, old_ctxtstr_changed.length()); assert(old_ctxtstr_changed.length() == 0); // old_ctxtstr_changed is now empty because it's been removed. // context is "aa" in the above example. // THEN add the new suffix, "CCC" in the above example - emit_text(state, new_ctxtstr_changed); + ldml_state.emit_text(new_ctxtstr_changed); // context is now "aaCCC" return matchedContext; } + void -ldml_processor::remove_text(km_core_state *state, std::u32string &str, size_t length) { +ldml_event_state::remove_text(std::u32string &str, size_t length) { /** track how many context items have been removed, via push_backspace() */ size_t contextRemoved = 0; for (auto c = state->context().rbegin(); length > 0 && c != state->context().rend(); c++, contextRemoved++) { @@ -426,13 +437,13 @@ km_core_status ldml_processor::validate() const { } void -ldml_processor::emit_text(km_core_state *state, const std::u16string &str) { +ldml_event_state::emit_text(const std::u16string &str) { const std::u32string str32 = kmx::u16string_to_u32string(str); - emit_text(state, str32); + emit_text(str32); } void -ldml_processor::emit_text(km_core_state *state, const std::u32string &str) { +ldml_event_state::emit_text(const std::u32string &str) { for (auto it = str.begin(); it < str.end(); it++) { const auto ch = *it; // If we are at the start of a sequence: @@ -445,28 +456,28 @@ ldml_processor::emit_text(km_core_state *state, const std::u32string &str) { it++; // consume LDML_MARKER_CODE assert(it < str.end()); const auto marker_no = *it; - emit_marker(state, marker_no); + emit_marker(marker_no); } else { - emit_text(state, ch); + emit_text(ch); } } } void -ldml_processor::emit_text(km_core_state *state, km_core_usv ch) { +ldml_event_state::emit_text( km_core_usv ch) { assert(ch != LDML_UC_SENTINEL); state->context().push_character(ch); state->actions().push_character(ch); } void -ldml_processor::emit_marker(km_core_state *state, KMX_DWORD marker_no) { +ldml_event_state::emit_marker( KMX_DWORD marker_no) { assert(km::core::kmx::is_valid_marker(marker_no)); state->actions().push_marker(marker_no); state->context().push_marker(marker_no); } -void ldml_processor::emit_invalidate_passthrough_keystroke(km_core_state *state, km_core_virtual_key vk, uint16_t _kmn_unused(modifier_state)) { +void ldml_event_state::emit_invalidate_passthrough_keystroke() { if ((vk < 0x100) && km::core::kmx::vkey_to_contextreset[vk]) { state->actions().push_invalidate_context(); } else { @@ -476,10 +487,10 @@ void ldml_processor::emit_invalidate_passthrough_keystroke(km_core_state *state, } size_t -ldml_processor::context_to_string(km_core_state *state, std::u32string &str, bool include_markers) { +ldml_event_state::context_to_string(std::u32string &str, bool include_markers) { str.clear(); - auto &cp = state->context(); - size_t ctxlen = 0; // TODO-LDML: not used by callers? + auto &cp = state->context(); + size_t ctxlen = 0; // TODO-LDML: not used by callers? uint8_t last_type = KM_CORE_BT_UNKNOWN; for (auto c = cp.rbegin(); c != cp.rend(); c++, ctxlen++) { last_type = c->type; @@ -497,5 +508,26 @@ ldml_processor::context_to_string(km_core_state *state, std::u32string &str, boo return ctxlen; // consumed the entire context buffer. } +ldml_event_state::ldml_event_state( + km_core_state *s, + km_core_virtual_key v, + uint16_t m, + uint8_t i, + uint16_t e) { + this->state = s; + this->vk = v; + this->modifier_state = m; + this->is_key_down = i; + this->event_flags = e; +} + +void ldml_event_state::commit() { + state->actions().commit(); +} + +void ldml_event_state::clear() { + state->actions().clear(); +} + } // namespace core } // namespace km diff --git a/core/src/ldml/ldml_processor.hpp b/core/src/ldml/ldml_processor.hpp index 238b332215..5701de02a0 100644 --- a/core/src/ldml/ldml_processor.hpp +++ b/core/src/ldml/ldml_processor.hpp @@ -24,19 +24,16 @@ namespace core { #define KM_CORE_LMDL_PROCESSOR_VERSION u"1.0" - class ldml_processor : public abstract_processor { - private: - bool _valid; - std::unique_ptr transforms, bksp_transforms; - ldml::vkeys keys; +class ldml_event_state; + +/** our actual processor */ +class ldml_processor : public abstract_processor { public: ldml_processor( path const & kb_path, const std::vector & data ); -// ~ldml_processor() override; - static bool is_kmxplus_file( path const & kb_path, std::vector& data @@ -90,56 +87,104 @@ namespace core { } private: - /** emit text to context and actions */ - static void emit_text(km_core_state *state, const std::u16string &str); - /** emit text to context and actions */ - static void emit_text(km_core_state *state, const std::u32string &str); - /** emit char to context and actions */ - static void emit_text(km_core_state *state, km_core_usv ch); - /** emit a marker */ - static void emit_marker(km_core_state *state, KMX_DWORD marker); - /** emit a pass-through and invalidate */ - static void emit_invalidate_passthrough_keystroke(km_core_state *state, km_core_virtual_key vk, uint16_t modifier_state); + /** process a key-up */ + void process_key_up(ldml_event_state &ldml_state) const; - /** - * Delete text from the state, by: - * 1. calling actions().push_backspace() to push the appropriate backspaces - * 2. popping the same items from the context items - * 3. mutating 'str' by removing the same number of items. - * This function handles marker strings correctly. - * @param str string with text to remove, from the end - * @param length number of chars from the end of str to drop - */ - static void remove_text(km_core_state *state, std::u32string &str, size_t length); + /** process a key-down (if it wasn't handled exceptionally) */ + void process_key_down(ldml_event_state &ldml_state) const; - /** process a key-up */ - void process_key_up(km_core_state *state, km_core_virtual_key vk, uint16_t modifier_state) const; + /** process a typed key */ + void process_key_string(ldml_event_state &ldml_state, const std::u16string &key_str) const; - /** process a key-down (if it wasn't handled exceptionally) */ - void process_key_down(km_core_state *state, km_core_virtual_key vk, uint16_t modifier_state) const; + /** process a backspace */ + void process_backspace(ldml_event_state &ldml_state) const; - /** process a typed key */ - void process_key_string(km_core_state *state, const std::u16string &key_str) const; + /** + * common function for outputting a string with transforms/normalization applied. + * @param str string to output (such as from a key), or empty + * @param with_transforms transforms to use or nullptr + * @returns length of matched input context + */ + size_t process_output(ldml_event_state &ldml_state, const std::u32string &str, ldml::transforms *with_transforms) const; - /** process a backspace */ - void process_backspace(km_core_state *state) const; + private: + bool _valid; + std::unique_ptr transforms, bksp_transforms; + ldml::vkeys keys; +}; - /** - * common function for outputting a string with transforms/normalization applied. - * @param str string to output (such as from a key), or empty - * @param with_transforms transforms to use or nullptr - * @returns length of matched input context - */ - size_t process_output(km_core_state *state, const std::u32string &str, ldml::transforms *with_transforms) const; - /** - * add the string+marker portion of the context to the beginning of str. - * Stop when a non-string and non-marker is hit. - * Convert markers into the UC_SENTINEL format. - * @return the number of context items consumed - */ - static size_t context_to_string(km_core_state *state, std::u32string &str, bool include_markers = true); +/** class holding state as we process an event. mirrors process_event args. */ +class ldml_event_state { +public: + ldml_event_state( + km_core_state *state, + km_core_virtual_key vk, + uint16_t modifier_state, + uint8_t is_key_down, + uint16_t event_flags); + /** done with this, copy it into core state */ + void commit(); + /** clear this object out */ + void clear(); + + // getters + inline km_core_virtual_key get_vk() const; + inline uint16_t get_modifier_state() const; + + // actions + + /** emit text to context and actions */ + void emit_text(const std::u16string &str); + /** emit text to context and actions */ + void emit_text(const std::u32string &str); + /** emit char to context and actions */ + void emit_text(km_core_usv ch); + /** emit a marker */ + void emit_marker(KMX_DWORD marker); + /** emit a pass-through and invalidate */ + void emit_invalidate_passthrough_keystroke(); + /** emit a backspace (for a user-initiated deletion) */ + void emit_backspace(); + + /** + * Delete text from the state, by: + * 1. calling actions().push_backspace() to push the appropriate backspaces + * 2. popping the same items from the context items + * 3. mutating 'str' by removing the same number of items. + * This function handles marker strings correctly. + * @param str string with text to remove, from the end + * @param length number of chars from the end of str to drop + */ + void remove_text(std::u32string &str, size_t length); + + /** + * add the string+marker portion of the context to the beginning of str. + * Stop when a non-string and non-marker is hit. + * Convert markers into the UC_SENTINEL format. + * @return the number of context items consumed + */ + size_t context_to_string(std::u32string &str, bool include_markers = true); + + private: + km_core_virtual_key vk; + uint16_t modifier_state; + uint8_t is_key_down; + uint16_t event_flags; + km_core_state *state; +}; + + +// implementation +km_core_virtual_key +ldml_event_state::get_vk() const { + return vk; +} + +uint16_t +ldml_event_state::get_modifier_state() const { + return modifier_state; +} - }; } // namespace core } // namespace km From ba46d2127d9905ec6c6fd6b95abeb7c6eb022b28 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Fri, 19 Jan 2024 09:50:00 +0700 Subject: [PATCH 45/45] chore(core): address review comments --- core/src/action.hpp | 2 +- core/src/actions_normalize.cpp | 16 +++++----------- core/src/core_icu.h | 13 +++++++++++++ core/src/km_core_action_api.cpp | 2 +- core/src/km_core_state_context_set_if_needed.cpp | 10 +--------- core/src/ldml/ldml_markers.hpp | 9 +-------- core/src/ldml/ldml_transforms.hpp | 9 +-------- core/tests/unit/ldml/ldml_test_source.cpp | 8 +------- 8 files changed, 24 insertions(+), 45 deletions(-) create mode 100644 core/src/core_icu.h diff --git a/core/src/action.hpp b/core/src/action.hpp index e7bee85db0..f8b85c7dd3 100644 --- a/core/src/action.hpp +++ b/core/src/action.hpp @@ -24,7 +24,7 @@ namespace core /* in, out */ km_core_actions *actions ); - bool actions_update_app_context( + bool actions_update_app_context_nfu( /* in */ km_core_context const *cached_context, /* in, out */ km_core_context *app_context ); diff --git a/core/src/actions_normalize.cpp b/core/src/actions_normalize.cpp index a83e76c590..9d3ab6bc40 100644 --- a/core/src/actions_normalize.cpp +++ b/core/src/actions_normalize.cpp @@ -15,15 +15,7 @@ #include "state.hpp" #include "option.hpp" #include "debuglog.h" - -#if !defined(HAVE_ICU4C) -#error icu4c is required for this code -#endif - -#define U_FALLTHROUGH -#include "unicode/utypes.h" -#include "unicode/unistr.h" -#include "unicode/normalizer2.h" +#include "core_icu.h" // forward declarations @@ -286,7 +278,9 @@ km_core_usv *unicode_string_to_usv(icu::UnicodeString& src) { /** - * Refresh app_context to match the cached_context + * Refresh app_context to match the cached_context. Does not do normalization, + * unlike `actions_normalize`. Used in conjunction with keyboard processors that + * do not support normalization. * * @param cached_context the cached context, in NFU, after transform has been * applied to it by the keyboard processor @@ -295,7 +289,7 @@ km_core_usv *unicode_string_to_usv(icu::UnicodeString& src) { * function * @return true on success, false on failure */ -bool km::core::actions_update_app_context( +bool km::core::actions_update_app_context_nfu( /* in */ km_core_context const *cached_context, /* in, out */ km_core_context *app_context ) { diff --git a/core/src/core_icu.h b/core/src/core_icu.h new file mode 100644 index 0000000000..0461bcecfa --- /dev/null +++ b/core/src/core_icu.h @@ -0,0 +1,13 @@ +/** + * ICU modules used by Keyman Core + */ +#pragma once + +#if !defined(HAVE_ICU4C) +#error icu4c is required for this code +#endif + +#define U_FALLTHROUGH +#include "unicode/utypes.h" +#include "unicode/unistr.h" +#include "unicode/normalizer2.h" diff --git a/core/src/km_core_action_api.cpp b/core/src/km_core_action_api.cpp index 097f5a3e2b..9a57760418 100644 --- a/core/src/km_core_action_api.cpp +++ b/core/src/km_core_action_api.cpp @@ -43,7 +43,7 @@ km_core_actions const * km_core_state_get_actions( } } else { // For all other keyboard processors, we just copy the cached_context to the app_context - if(!actions_update_app_context(km_core_state_context(state), km_core_state_app_context(state))) { + if(!actions_update_app_context_nfu(km_core_state_context(state), km_core_state_app_context(state))) { km_core_actions_dispose(result); return nullptr; } diff --git a/core/src/km_core_state_context_set_if_needed.cpp b/core/src/km_core_state_context_set_if_needed.cpp index a6f6fde9c0..d6afc477d8 100644 --- a/core/src/km_core_state_context_set_if_needed.cpp +++ b/core/src/km_core_state_context_set_if_needed.cpp @@ -14,15 +14,7 @@ #include "processor.hpp" #include "state.hpp" #include "debuglog.h" - -#if !defined(HAVE_ICU4C) -#error icu4c is required for this code -#endif - -#define U_FALLTHROUGH -#include "unicode/utypes.h" -#include "unicode/unistr.h" -#include "unicode/normalizer2.h" +#include "core_icu.h" using namespace km::core; diff --git a/core/src/ldml/ldml_markers.hpp b/core/src/ldml/ldml_markers.hpp index 68c7aa2f2d..4ec9b71ae2 100644 --- a/core/src/ldml/ldml_markers.hpp +++ b/core/src/ldml/ldml_markers.hpp @@ -16,18 +16,11 @@ #include #include "debuglog.h" -#if !defined(HAVE_ICU4C) -#error icu4c is required for this code -#endif - -#define U_FALLTHROUGH -#include "unicode/utypes.h" +#include "core_icu.h" #include "unicode/uniset.h" #include "unicode/usetiter.h" -#include "unicode/unistr.h" #include "unicode/regex.h" #include "unicode/utext.h" -#include "unicode/normalizer2.h" namespace km { namespace core { diff --git a/core/src/ldml/ldml_transforms.hpp b/core/src/ldml/ldml_transforms.hpp index 72da5c68a7..e6ef4e1523 100644 --- a/core/src/ldml/ldml_transforms.hpp +++ b/core/src/ldml/ldml_transforms.hpp @@ -16,18 +16,11 @@ #include #include "debuglog.h" -#if !defined(HAVE_ICU4C) -#error icu4c is required for this code -#endif - -#define U_FALLTHROUGH -#include "unicode/utypes.h" +#include "core_icu.h" #include "unicode/uniset.h" #include "unicode/usetiter.h" -#include "unicode/unistr.h" #include "unicode/regex.h" #include "unicode/utext.h" -#include "unicode/normalizer2.h" namespace km { namespace core { diff --git a/core/tests/unit/ldml/ldml_test_source.cpp b/core/tests/unit/ldml/ldml_test_source.cpp index 4f59468dcd..0b481cd68a 100644 --- a/core/tests/unit/ldml/ldml_test_source.cpp +++ b/core/tests/unit/ldml/ldml_test_source.cpp @@ -33,15 +33,9 @@ #include "ldml_test_source.hpp" #include "ldml_test_utils.hpp" -#if defined(HAVE_ICU4C) -// TODO-LDML: Needed this for some compiler warnings -#define U_FALLTHROUGH -#include "unicode/utypes.h" +#include "core_icu.h" #include "unicode/uniset.h" #include "unicode/usetiter.h" -#else -#error icu4c is required for this test -#endif #define assert_or_return(expr) if(!(expr)) { \ std::wcerr << __FILE__ << ":" << __LINE__ << ": " << \