diff --git a/common/include/test_color.h b/common/include/test_color.h index 6837b89327..fdf2f69586 100644 --- a/common/include/test_color.h +++ b/common/include/test_color.h @@ -23,7 +23,7 @@ enum ansi_code { BRIGHT_RED = 196 }; -bool enabled = false; +bool enabled = false; // TODO: move to test_color.c because test_color.h cannot be #included in more than 1 file in a project otherwise. class fg { ansi_code code; diff --git a/core/README.md b/core/README.md new file mode 100644 index 0000000000..b1ddaa2f41 --- /dev/null +++ b/core/README.md @@ -0,0 +1,3 @@ +# Keyman Core + +Version 1.0.0 of libkeymancore released with Keyman 17.0 \ No newline at end of file diff --git a/core/include/keyman/keyman_core_api.h b/core/include/keyman/keyman_core_api.h index dc15b8b8ac..70d8dd9697 100644 --- a/core/include/keyman/keyman_core_api.h +++ b/core/include/keyman/keyman_core_api.h @@ -115,7 +115,6 @@ typedef uint32_t km_core_status; // Status return code. // Opaque object types. // -typedef struct km_core_context km_core_context; typedef struct km_core_keyboard km_core_keyboard; typedef struct km_core_state km_core_state; typedef struct km_core_options km_core_options; @@ -146,6 +145,7 @@ enum km_core_status_codes { KM_CORE_STATUS_INSUFFICENT_BUFFER = 5, KM_CORE_STATUS_INVALID_UTF = 6, KM_CORE_STATUS_INVALID_KEYBOARD = 7, + KM_CORE_STATUS_NOT_IMPLEMENTED = 8, KM_CORE_STATUS_OS_ERROR = 0x80000000 }; @@ -155,357 +155,7 @@ The final status code KM_CORE_STATUS_OS_ERROR is intended to allow encapsulating a platform error code; the remaining 31 low bits are the error code returned by the OS for cases where the failure mode is platform specific. For HRESULT codes this only permits failure codes to be passed. - - -### Context -The context is the text prior to the insertion point (caret, cursor). -The context is constructed by the Platform layer, typically by interrogating the -Client Application. The context will be updated by the engine for keystroke -events. If the Platform layer code caches the context, the context should be -reset when a context state change is detected. Context state changes can occur -when the user uses the mouse to move the insertion point, uses cursor keys, -switches applications or input fields, or presses hotkeys such as Ctrl+N to -start a new document. The full set of context state change triggers is up to the -Platform layer. - -Context can also contain positional Markers (also known as 'deadkeys' in kmn -keyboards), which are transitory state flags that are erased whenever a context -state change is detected. Markers are always controlled by the Engine. - -Contexts are always owned by their state. They may be set to a list of -context_items or interrogated for their current list of context items. -```c */ -enum km_core_context_type { - KM_CORE_CT_END, - KM_CORE_CT_CHAR, - KM_CORE_CT_MARKER -}; - -typedef struct { - uint8_t type; - uint8_t _reserved[3]; - union { - km_core_usv character; - uint32_t marker; - }; -} km_core_context_item; - -#define KM_CORE_CONTEXT_ITEM_END {KM_CORE_CT_END, {0,}, {0,}} -/* -``` -### `km_core_context_items_from_utf16` -##### Description: -Convert a UTF16 encoded Unicode string into an array of `km_core_context_item` -structures. Allocates memory as needed. -##### Return status: -- `KM_CORE_STATUS_OK`: On success. -- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. -- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated for the - output buffer. -- `KM_CORE_STATUS_INVALID_UTF`: In the event the UTF16 string cannot be decoded - because it contains unpaired surrogate codeunits. -##### Parameters: -- __text__: a pointer to a null terminated array of utf16 encoded data. -- __out_ptr__: a pointer to the result variable: - A pointer to the start of the `km_core_context_item` array containing the - representation of the input string. - Terminated with a type of `KM_CORE_CT_END`. Must be disposed of with - `km_core_context_items_dispose`. - -```c -*/ -KMN_API -km_core_status -km_core_context_items_from_utf16(km_core_cp const *text, - km_core_context_item **out_ptr); - -/* -``` -### `km_core_context_items_from_utf8` -##### Description: -Convert an UTF8 encoded Unicode string into an array of `km_core_context_item` -structures. Allocates memory as needed. -##### Status: -- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. -- `KM_CORE_STATUS_NO_MEM`: In the event it cannot allocate enough memory for the - output buffer. -- `KM_CORE_STATUS_INVALID_UTF`: In the event the UTF8 string cannot be -decoded. -##### Parameters: -- __text__: a pointer to a null terminated array of utf8 encoded data. -- __out_ptr__: a pointer to the result variable: - A pointer to the start of the `km_core_context_item` array containing the - representation of the input string. - Terminated with a type of `KM_CORE_CT_END`. - -```c -*/ -KMN_API -km_core_status -km_core_context_items_from_utf8(char const *text, - km_core_context_item **out_ptr); - -/* -``` -### `km_core_context_items_to_utf16` -##### Description: -Convert a context item array into a UTF-16 encoded string placing it into -the supplied buffer of specified size, and return the number of code units -actually used in the conversion. If null is passed as the buffer the -number of codeunits required is returned. Any markers in the context will -not be included in the output buffer. -##### Return status: -- `KM_CORE_STATUS_OK`: On success. -- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. -- `KM_CORE_STATUS_INSUFFICENT_BUFFER`: If the buffer is not large enough. - `buf_size` will contain the space required. The contents of the buffer are - undefined. -##### Parameters: -- __context_items__: A pointer to the start of an array `km_core_context_item`. - Must be terminated with a type of `KM_CORE_CT_END`. -- __buf__: A pointer to the buffer to place the UTF-16 string into. - May be null to request size calculation. -- __buf_size__: a pointer to the result variable: - The size of the supplied buffer in codeunits if `buf` is given. - On return will be the size required if `buf` is null. - -```c -*/ -KMN_API -km_core_status -km_core_context_items_to_utf16(km_core_context_item const *item, - km_core_cp *buf, - size_t *buf_size); - -/* -``` -### `km_core_context_items_to_utf8` -##### Description: -Convert a context item array into a UTF-8 encoded string placing it into -the supplied buffer of specified size, and return the number of code units -actually used in the conversion. If null is passed as the buffer the -number of codeunits required is returned. Any markers in the context will -not be included in the output buffer. -##### Return status: -- `KM_CORE_STATUS_OK`: On success. -- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. -- `KM_CORE_STATUS_INSUFFICENT_BUFFER`: If the buffer is not large enough. - `buf_size` will contain the space required. The contents of the buffer are - undefined. -##### Parameters: -- __context_items__: A pointer to the start of an array `km_core_context_item`. - Must be terminated with a type of `KM_CORE_CT_END`. -- __buf__: A pointer to the buffer to place the UTF-8 string into. - May be null to request size calculation. -- __buf_size__: a pointer to the result variable: - The size of the supplied buffer in codeunits if `buf` is given. - On return will be the size required if `buf` is null. - -```c -*/ -KMN_API -km_core_status -km_core_context_items_to_utf8(km_core_context_item const *item, - char *buf, - size_t *buf_size); - -/* -``` -### `km_core_context_items_to_utf32` -##### Description: -Convert a context item array into a UTF-32 encoded string placing it into -the supplied buffer of specified size, and return the number of codepoints -actually used in the conversion. If null is passed as the buffer the -number of codepoints required is returned. Any markers in the context will -not be included in the output buffer. -##### Return status: -- `KM_CORE_STATUS_OK`: On success. -- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. -- `KM_CORE_STATUS_INSUFFICENT_BUFFER`: If the buffer is not large enough. - `buf_size` will contain the space required. The contents of the buffer are - undefined. -##### Parameters: -- __context_items__: A pointer to the start of an array `km_core_context_item`. - Must be terminated with a type of `KM_CORE_CT_END`. -- __buf__: A pointer to the buffer to place the UTF-32 string into. - May be null to request size calculation. -- __buf_size__: a pointer to the result variable: - The size of the supplied buffer in codepoints if `buf` is given. - On return will be the size required if `buf` is null. - -```c -*/ -KMN_API -km_core_status -km_core_context_items_to_utf32(km_core_context_item const *item, - km_core_usv *buf, - size_t *buf_size); - -/* -``` -### `km_core_context_items_dispose` -##### Description: -Free the allocated memory belonging to a `km_core_context_item` array previously -returned by `km_core_context_items_from_utf16` or `km_core_context_get` -##### Parameters: -- __context_items__: A pointer to the start of the `km_core_context_item` array - to be disposed of. - -```c -*/ -KMN_API -void -km_core_context_items_dispose(km_core_context_item *context_items); - -/* -``` -### `km_core_context_set` -##### Description: -Replace the contents of the current context with a new sequence of -`km_core_context_item` entries. -##### Return status: -- `KM_CORE_STATUS_OK`: On success. -- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. -- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated to - grow the context buffer internally. -##### Parameters: -- __context__: A pointer to an opaque context object -- __context_items__: A pointer to the start of the `km_core_context_item` - array containing the new context. It must be terminated with an item - of type `KM_CORE_CT_END`. - -```c -*/ -KMN_API -km_core_status -km_core_context_set(km_core_context *context, - km_core_context_item const *context_items); - -/* -``` -### `km_core_context_get` -##### Description: -Copies all items in the context into a new array and returns the new array. -This must be disposed of by caller using `km_core_context_items_dispose`. -##### Return status: -- `KM_CORE_STATUS_OK`: On success. -- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. -- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated for the - output buffer. -##### Parameters: -- __context_items__: A pointer to the start of an array `km_core_context_item`. -- __out__: a pointer to the result variable: - A pointer to the start of the `km_core_context_item` array containing a - copy of the context. Terminated with a type of `KM_CORE_CT_END`. Must be - disposed of with `km_core_context_items_dispose`. - -```c -*/ -KMN_API -km_core_status -km_core_context_get(km_core_context const *context_items, - km_core_context_item **out); - -/* -``` -### `km_core_context_clear` -##### Description: -Removes all context_items from the internal array. If `context` is -null, has no effect. -##### Parameters: -- __context__: A pointer to an opaque context object - -```c -*/ -KMN_API -void -km_core_context_clear(km_core_context *); - -/* -``` -### `km_core_context_length` -##### Description: -Return the number of items in the context. -##### Return: -The number of items in the context, and will return 0 if passed a null `context` -pointer. -##### Parameters: -- __context__: A pointer to an opaque context object - -```c -*/ -KMN_API -size_t -km_core_context_length(km_core_context *); - -/* -``` -### `km_core_context_append` -##### Description: -Add more items to the end (insertion point) of the context. If these exceed the -maximum context length the same number of items will be dropped from the -beginning of the context. -##### Return status: -- `KM_CORE_STATUS_OK`: On success. -- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. -- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated to - grow the context buffer internally. -##### Parameters: -- __context__: A pointer to an opaque context object. -- __context_items__: A pointer to the start of the `KM_CORE_CT_END` terminated - array of `km_core_context_item` to append. - -```c -*/ -KMN_API -km_core_status -km_core_context_append(km_core_context *context, - km_core_context_item const *context_items); - -/* -``` -### `km_core_context_shrink` -##### Description: -Remove a specified number of items from the end of the context, optionally -add up to the same number of the supplied items to the front of the context. -##### Return status: -- `KM_CORE_STATUS_OK`: On success. -- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. -- `KM_CORE_STATUS_NO_MEM`: in the event it cannot allocated enough memory to grow - the context internally. -##### Parameters: -- __context__: A pointer to an opaque context object. -- __num__: The number of items to remove from the end of context. -- __context_items__: Pointer to the start of the `KM_CORE_CT_END` terminated - array of `km_core_context_item` to add to the front. Up to `num` items will - be prepended. This may be null if not required. - -```c -*/ -KMN_API -km_core_status -km_core_context_shrink(km_core_context *context, - size_t num, - km_core_context_item const *prefix); - -/* -``` -### `km_core_context_item_list_size` -##### Description: -Return the length of a terminated `km_core_context_item` array. -##### Return: -The number of items in the list, not including terminating item, -or 0 if `context_items` is null. -##### Parameters: -- __context_items__: A pointer to a `KM_CORE_CT_END` terminated array of - `km_core_context_item` values. - -```c -*/ -KMN_API -size_t -km_core_context_item_list_size(km_core_context_item const *context_items); /* ``` @@ -1106,39 +756,70 @@ km_core_state_dispose(km_core_state *state); /* ``` -### `km_core_state_context` -##### Description: -Get access to the state object's context. -##### Return: -A pointer to an opaque state object. This pointer is valid for the lifetime -of the state object. If null is passed in, then null is returned. -##### Parameters: -- __state__: A pointer to the opaque state object to be queried. +### Context Debug Reporting +As of version 17, the cached context is an internal property of the +state, not exposed to the consumer of the API -- apart from the +Keyman Developer Keyboard Debugger. However, for other debug +purposes, it is helpful to be able to examine the cached context, so +a debug-formatted version of the context is made available with +`km_core_state_context_debug`. This is not intended to be parsed for +reading the context for other purposes, and the format may change. +The three context types are: +* cached: the internal context used by Core, which may be normalized + and may contain markers. This is set via + km_core_state_context_set_if_needed, and will be modified + during keystroke event processing. +* intermediate: internal context used by IMX, only valid during + keystroke event processing. +* app: an exact copy of the current context passed in to + km_core_state_context_set_if_needed, which is used to verify + the precise text manipulations required when emitted changes. + This input context is in "NFU" -- normalization form unknown, + and may be mixed normalization so may require fixups when + it is manipulated by keyboard processors that support + normalization, such as the LDML keyboard processor. ```c */ -KMN_API -km_core_context * -km_core_state_context(km_core_state *state); - +typedef enum { + KM_CORE_DEBUG_CONTEXT_CACHED = 0, + KM_CORE_DEBUG_CONTEXT_INTERMEDIATE = 1, + KM_CORE_DEBUG_CONTEXT_APP = 2 +} km_core_debug_context_type; /* ``` -### `km_core_state_get_intermediate_context` +### `km_core_state_context_debug` ##### Description: -Get access to the state object's keyboard processor's intermediate context. This context -is used during an IMX callback, part way through processing a keystroke. +Returns a debug formatted string of the context from the state. ##### Return: -A pointer to an context item array. Must be disposed of by a call -to `km_core_context_items_dispose`. +A pointer to a km_core_cp UTF-16 string. Must be disposed of by a call +to `km_core_cp_dispose`. ##### Parameters: - __state__: A pointer to the opaque state object to be queried. +- __context_type__: the type of context to retrieve from the state ```c */ KMN_API -km_core_status -km_core_state_get_intermediate_context(km_core_state *state, km_core_context_item ** context_items); +km_core_cp * +km_core_state_context_debug(km_core_state *state, km_core_debug_context_type context_type); + +/* +``` +### `km_core_cp_dispose` +##### Description: +Free the allocated memory belonging to a `km_core_cp` array previously +returned by `km_core_state_context_debug`. May be `nullptr`. +##### Parameters: +- __cp__: A pointer to the start of the `km_core_cp` array + to be disposed of. + +```c +*/ +KMN_API +void +km_core_cp_dispose(km_core_cp *cp); /* ``` diff --git a/core/include/keyman/keyman_core_api_context.h b/core/include/keyman/keyman_core_api_context.h new file mode 100644 index 0000000000..4eac01528c --- /dev/null +++ b/core/include/keyman/keyman_core_api_context.h @@ -0,0 +1,486 @@ +/* + * Keyman is copyright (C) SIL International. MIT License. + * + * Keyman Keyboard Processor API - Debugger Interfaces + * + * The debugger interfaces are still very dependent on .kmx + * objects. + * + * Note: this file is subject to change; the debugger + * interfaces are not stable across versions. + * + */ + +#pragma once + +#include +#include +#include +#include + +// Currently, the Core unit tests use private context APIs defined in +// keyman_core_api_context.h, which are unused by other consumers. We are +// hoping to remove these entirely in the future, so we restrict access +// by default with this macro. Keyman Core internally uses these functions +// #define _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +#if defined(__cplusplus) +extern "C" +{ +#endif + +typedef struct km_core_context km_core_context; + +// ---------------------------------------------------------------------------------- +// Context APIs are now available only to the keyboard debugger, IMX, and Core unit +// tests (17.0) +// ---------------------------------------------------------------------------------- + +/* +### Context +The context is the text prior to the insertion point (caret, cursor). The +context is constructed by the Platform layer, typically by interrogating the +Client Application. The context will be updated by the engine for keystroke +events. If the Platform layer code caches the context, the context should be +reset when a context state change is detected. Context state changes can occur +when the user uses the mouse to move the insertion point, uses cursor keys, +switches applications or input fields, or presses hotkeys such as Ctrl+N to +start a new document. The full set of context state change triggers is up to the +Platform layer. + +Context can also contain positional Markers (also known as 'deadkeys' in kmn +keyboards), which are transitory state flags that are erased whenever a context +state change is detected. Markers are always controlled by the Engine. + +Contexts are always owned by their state. They may be set to a list of +context_items or interrogated for their current list of context items. + +Core maintains and caches the context. Engine can update the context with +`km_core_state_context_set_if_needed` and `km_core_state_context_clear`. These +two functions are available in keyman_core_api.h. + +The Keyboard Debugger in Keyman Developer, and IMX in Keyman for Windows, make +use of the context functionality in this header, but these functions should not +be used in other places. +```c +*/ +enum km_core_context_type { + KM_CORE_CT_END, + KM_CORE_CT_CHAR, + KM_CORE_CT_MARKER +}; + +typedef struct { + uint8_t type; + uint8_t _reserved[3]; + union { + km_core_usv character; + uint32_t marker; + }; +} km_core_context_item; + +#define KM_CORE_CONTEXT_ITEM_END {KM_CORE_CT_END, {0,}, {0,}} + +/* +``` +### `km_core_state_get_intermediate_context` +##### Description: +Get access to the state object's keyboard processor's intermediate context. This context +is used during an IMX callback, part way through processing a keystroke. +##### Return: +A pointer to an context item array. Must be disposed of by a call +to `km_core_context_items_dispose`. +##### Parameters: +- __state__: A pointer to the opaque state object to be queried. + +```c +*/ +KMN_API +km_core_status +km_core_state_get_intermediate_context(km_core_state *state, km_core_context_item ** context_items); + +/* +``` +### `km_core_context_items_dispose` +##### Description: +Free the allocated memory belonging to a `km_core_context_item` array previously +returned by `km_core_context_items_from_utf16` or `km_core_context_get` +##### Parameters: +- __context_items__: A pointer to the start of the `km_core_context_item` array + to be disposed of. + +```c +*/ +KMN_API +void +km_core_context_items_dispose(km_core_context_item *context_items); + +/** + * Get access to the state object's cached context. + * @param state A pointer to the opaque state object to be queried. + * @returns A pointer to an opaque context object. This pointer is valid for the + * lifetime of the state object. If null is passed in, then null is + * returned. + */ +KMN_API +km_core_context * +km_core_state_context(km_core_state const *state); + +/** + * Get access to the state object's application context. + * @param state A pointer to the opaque state object to be queried. + * @returns A pointer to an opaque context object. This pointer is valid for the + * lifetime of the state object. If null is passed in, then null is + * returned. + */ +KMN_API +km_core_context * +km_core_state_app_context(km_core_state const *state); + +/* +``` +### `km_core_context_items_from_utf16` +##### Description: +Convert a UTF16 encoded Unicode string into an array of `km_core_context_item` +structures. Allocates memory as needed. +##### Return status: +- `KM_CORE_STATUS_OK`: On success. +- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. +- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated for the + output buffer. +- `KM_CORE_STATUS_INVALID_UTF`: In the event the UTF16 string cannot be decoded + because it contains unpaired surrogate codeunits. +##### Parameters: +- __text__: a pointer to a null terminated array of utf16 encoded data. +- __out_ptr__: a pointer to the result variable: + A pointer to the start of the `km_core_context_item` array containing the + representation of the input string. + Terminated with a type of `KM_CORE_CT_END`. Must be disposed of with + `km_core_context_items_dispose`. + +```c +*/ + +#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +KMN_API +km_core_status +km_core_context_items_from_utf16(km_core_cp const *text, + km_core_context_item **out_ptr); + +#endif + +/* +``` +### `km_core_context_items_from_utf8` +##### Description: +Convert an UTF8 encoded Unicode string into an array of `km_core_context_item` +structures. Allocates memory as needed. +##### Status: +- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. +- `KM_CORE_STATUS_NO_MEM`: In the event it cannot allocate enough memory for the + output buffer. +- `KM_CORE_STATUS_INVALID_UTF`: In the event the UTF8 string cannot be +decoded. +##### Parameters: +- __text__: a pointer to a null terminated array of utf8 encoded data. +- __out_ptr__: a pointer to the result variable: + A pointer to the start of the `km_core_context_item` array containing the + representation of the input string. + Terminated with a type of `KM_CORE_CT_END`. + +```c +*/ + +#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +KMN_API +km_core_status +km_core_context_items_from_utf8(char const *text, + km_core_context_item **out_ptr); + +#endif + +/* +``` +### `km_core_context_items_to_utf16` +##### Description: +Convert a context item array into a UTF-16 encoded string placing it into +the supplied buffer of specified size, and return the number of code units +actually used in the conversion. If null is passed as the buffer the +number of codeunits required is returned. Any markers in the context will +not be included in the output buffer. +##### Return status: +- `KM_CORE_STATUS_OK`: On success. +- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. +- `KM_CORE_STATUS_INSUFFICENT_BUFFER`: If the buffer is not large enough. + `buf_size` will contain the space required. The contents of the buffer are + undefined. +##### Parameters: +- __context_items__: A pointer to the start of an array `km_core_context_item`. + Must be terminated with a type of `KM_CORE_CT_END`. +- __buf__: A pointer to the buffer to place the UTF-16 string into. + May be null to request size calculation. +- __buf_size__: a pointer to the result variable: + The size of the supplied buffer in codeunits if `buf` is given. + On return will be the size required if `buf` is null. + +```c +*/ + +#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +KMN_API +km_core_status +km_core_context_items_to_utf16(km_core_context_item const *item, + km_core_cp *buf, + size_t *buf_size); + +#endif + +/* +``` +### `km_core_context_items_to_utf8` +##### Description: +Convert a context item array into a UTF-8 encoded string placing it into +the supplied buffer of specified size, and return the number of code units +actually used in the conversion. If null is passed as the buffer the +number of codeunits required is returned. Any markers in the context will +not be included in the output buffer. +##### Return status: +- `KM_CORE_STATUS_OK`: On success. +- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. +- `KM_CORE_STATUS_INSUFFICENT_BUFFER`: If the buffer is not large enough. + `buf_size` will contain the space required. The contents of the buffer are + undefined. +##### Parameters: +- __context_items__: A pointer to the start of an array `km_core_context_item`. + Must be terminated with a type of `KM_CORE_CT_END`. +- __buf__: A pointer to the buffer to place the UTF-8 string into. + May be null to request size calculation. +- __buf_size__: a pointer to the result variable: + The size of the supplied buffer in codeunits if `buf` is given. + On return will be the size required if `buf` is null. + +```c +*/ + +#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +KMN_API +km_core_status +km_core_context_items_to_utf8(km_core_context_item const *item, + char *buf, + size_t *buf_size); + +#endif + +/* +``` +### `km_core_context_items_to_utf32` +##### Description: +Convert a context item array into a UTF-32 encoded string placing it into +the supplied buffer of specified size, and return the number of codepoints +actually used in the conversion. If null is passed as the buffer the +number of codepoints required is returned. Any markers in the context will +not be included in the output buffer. +##### Return status: +- `KM_CORE_STATUS_OK`: On success. +- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. +- `KM_CORE_STATUS_INSUFFICENT_BUFFER`: If the buffer is not large enough. + `buf_size` will contain the space required. The contents of the buffer are + undefined. +##### Parameters: +- __context_items__: A pointer to the start of an array `km_core_context_item`. + Must be terminated with a type of `KM_CORE_CT_END`. +- __buf__: A pointer to the buffer to place the UTF-32 string into. + May be null to request size calculation. +- __buf_size__: a pointer to the result variable: + The size of the supplied buffer in codepoints if `buf` is given. + On return will be the size required if `buf` is null. + +```c +*/ + +#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +KMN_API +km_core_status +km_core_context_items_to_utf32(km_core_context_item const *item, + km_core_usv *buf, + size_t *buf_size); + +#endif + +/* +``` +### `km_core_context_set` +##### Description: +Replace the contents of the current context with a new sequence of +`km_core_context_item` entries. +##### Return status: +- `KM_CORE_STATUS_OK`: On success. +- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. +- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated to + grow the context buffer internally. +##### Parameters: +- __context__: A pointer to an opaque context object +- __context_items__: A pointer to the start of the `km_core_context_item` + array containing the new context. It must be terminated with an item + of type `KM_CORE_CT_END`. + +```c +*/ +KMN_API +km_core_status +km_core_context_set(km_core_context *context, + km_core_context_item const *context_items); + +/* +``` +### `km_core_context_get` +##### Description: +Copies all items in the context into a new array and returns the new array. +This must be disposed of by caller using `km_core_context_items_dispose`. +##### Return status: +- `KM_CORE_STATUS_OK`: On success. +- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. +- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated for the + output buffer. +##### Parameters: +- __context_items__: A pointer to the start of an array `km_core_context_item`. +- __out__: a pointer to the result variable: + A pointer to the start of the `km_core_context_item` array containing a + copy of the context. Terminated with a type of `KM_CORE_CT_END`. Must be + disposed of with `km_core_context_items_dispose`. + +```c +*/ + +#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +KMN_API +km_core_status +km_core_context_get(km_core_context const *context_items, + km_core_context_item **out); + +#endif + +/* +``` +### `km_core_context_clear` +##### Description: +Removes all context_items from the internal array. If `context` is +null, has no effect. +##### Parameters: +- __context__: A pointer to an opaque context object + +```c +*/ +KMN_API +void +km_core_context_clear(km_core_context *); + +/* +``` +### `km_core_context_length` +##### Description: +Return the number of items in the context. +##### Return: +The number of items in the context, and will return 0 if passed a null `context` +pointer. +##### Parameters: +- __context__: A pointer to an opaque context object + +```c +*/ + +#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +KMN_API +size_t +km_core_context_length(km_core_context *); + +#endif + +/* +``` +### `km_core_context_append` +##### Description: +Add more items to the end (insertion point) of the context. If these exceed the +maximum context length the same number of items will be dropped from the +beginning of the context. +##### Return status: +- `KM_CORE_STATUS_OK`: On success. +- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. +- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated to + grow the context buffer internally. +##### Parameters: +- __context__: A pointer to an opaque context object. +- __context_items__: A pointer to the start of the `KM_CORE_CT_END` terminated + array of `km_core_context_item` to append. + +```c +*/ + +#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +KMN_API +km_core_status +km_core_context_append(km_core_context *context, + km_core_context_item const *context_items); + +#endif + +/* +``` +### `km_core_context_shrink` +##### Description: +Remove a specified number of items from the end of the context, optionally +add up to the same number of the supplied items to the front of the context. +##### Return status: +- `KM_CORE_STATUS_OK`: On success. +- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null. +- `KM_CORE_STATUS_NO_MEM`: in the event it cannot allocated enough memory to grow + the context internally. +##### Parameters: +- __context__: A pointer to an opaque context object. +- __num__: The number of items to remove from the end of context. +- __context_items__: Pointer to the start of the `KM_CORE_CT_END` terminated + array of `km_core_context_item` to add to the front. Up to `num` items will + be prepended. This may be null if not required. + +```c +*/ + +#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API + +KMN_API +km_core_status +km_core_context_shrink(km_core_context *context, + size_t num, + km_core_context_item const *prefix); + +#endif + +/* +``` +### `km_core_context_item_list_size` +##### Description: +Return the length of a terminated `km_core_context_item` array. +##### Return: +The number of items in the list, not including terminating item, +or 0 if `context_items` is null. +##### Parameters: +- __context_items__: A pointer to a `KM_CORE_CT_END` terminated array of + `km_core_context_item` values. + +```c +*/ +KMN_API +size_t +km_core_context_item_list_size(km_core_context_item const *context_items); + +#if defined(__cplusplus) +} // extern "C" +#endif diff --git a/core/meson.build b/core/meson.build index 716faca2dc..e6b1fd8fa1 100644 --- a/core/meson.build +++ b/core/meson.build @@ -29,6 +29,9 @@ python = py.find_installation() # For now, we use KM_CORE_LIBRARY to inject the km::core::kmx namespace defns += ['-DKM_CORE_LIBRARY'] +# See keyman_core_api_context.h +defns += ['-D_KM_CORE_ACCESS_PRIVATE_CONTEXT_API'] + # #define DEBUG when we are on a debug build if get_option('buildtype') == 'debug' add_global_arguments('-DDEBUG', language : 'cpp') diff --git a/core/src/action.cpp b/core/src/action.cpp index f4565844d7..f958ca3dbe 100644 --- a/core/src/action.cpp +++ b/core/src/action.cpp @@ -11,13 +11,12 @@ #include #include -#include #include "action.hpp" #include "state.hpp" #include "option.hpp" -km_core_actions const * km::core::action_item_list_to_actions_object( +km_core_actions * km::core::action_item_list_to_actions_object( km_core_action_item const *action_items ) { assert(action_items != nullptr); diff --git a/core/src/action.hpp b/core/src/action.hpp index 06038c3122..f8b85c7dd3 100644 --- a/core/src/action.hpp +++ b/core/src/action.hpp @@ -8,14 +8,26 @@ #pragma once -#include +#include "keyman_core.h" #include namespace km { namespace core { - km_core_actions const *action_item_list_to_actions_object( + km_core_actions *action_item_list_to_actions_object( km_core_action_item const *action_items ); + + bool actions_normalize( + /* in */ km_core_context const *cached_context, + /* in, out */ km_core_context *app_context, + /* in, out */ km_core_actions *actions + ); + + bool actions_update_app_context_nfu( + /* in */ km_core_context const *cached_context, + /* in, out */ km_core_context *app_context + ); + } // namespace core } // namespace km diff --git a/core/src/actions_normalize.cpp b/core/src/actions_normalize.cpp new file mode 100644 index 0000000000..9d3ab6bc40 --- /dev/null +++ b/core/src/actions_normalize.cpp @@ -0,0 +1,312 @@ +/* + Copyright: © 2024 SIL International. + Description: Implementation of the action output normalization. + Create Date: 16 Jan 2024 + Authors: Marc Durdin (MCD) + History: 16 Jan 2024 - MCD - Initial implementation from #9999 +*/ +#include +#include +#include + +#include +#include "context.hpp" +#include "action.hpp" +#include "state.hpp" +#include "option.hpp" +#include "debuglog.h" +#include "core_icu.h" + +// forward declarations + +icu::UnicodeString context_items_to_unicode_string(km_core_context const *context); +km_core_usv *unicode_string_to_usv(icu::UnicodeString& src); + +/** + * Normalize the output from an action to NFC, across the context | output + * boundary, fixing up the app_context and the output actions to take into + * account the NFU input app_context + * + * @param cached_context the cached context, in NFD, after transform has been + * applied to it by the keyboard processor + * @param app_context the app context, in NFU; transform has not been + * applied, and will be applied by this function + * @param actions transform to apply, in NFD, which will be converted + * to NFC by this function + * @return true on success, false on failure + */ +bool km::core::actions_normalize( + /* in */ km_core_context const *cached_context, + /* in, out */ km_core_context *app_context, + /* in, out */ km_core_actions *actions +) { + assert(actions != nullptr); + assert(cached_context != nullptr); + assert(app_context != nullptr); + if(actions == nullptr || cached_context == nullptr || app_context == nullptr) { + return false; + } + + /* + The code_points_to_delete value at this point is in NFD. The cached_context + is in NFD and has already been updated by the keyboard processor to the + expected result of the action, so we need to remove the output from a string + copy of the cached_context to start, in order to get it to the same position + as the app_context. + + The app_context is in NFU. We need to figure out how many characters to + remove from the end of app_context in order to correctly normalize across + the boundary, without normalizing more of the string than necessary. + + We do not need to mutate the cached_context itself, because it is already + correct. This is good, because it means we will not lose track of markers + within it. The app_context will be mutated, as it will need the new output + appended, in order to match the expected result. Remember that the + app_context does not contain markers; these are maintained only in the + cached_context. + */ + + /* + Initialization + */ + + UErrorCode icu_status = U_ZERO_ERROR; + const icu::Normalizer2 *nfc = icu::Normalizer2::getNFCInstance(icu_status); + assert(U_SUCCESS(icu_status)); + if(!U_SUCCESS(icu_status)) { + DebugLog("getNFCInstance failed with %x", icu_status); + return false; + } + + const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(icu_status); + assert(U_SUCCESS(icu_status)); + if(!U_SUCCESS(icu_status)) { + DebugLog("getNFDInstance failed with %x", icu_status); + return false; + } + + icu::UnicodeString output = icu::UnicodeString::fromUTF32(reinterpret_cast(actions->output), -1); + icu::UnicodeString cached_context_string = context_items_to_unicode_string(cached_context); + icu::UnicodeString app_context_string = context_items_to_unicode_string(app_context); + assert(!output.isBogus()); + assert(!cached_context_string.isBogus()); + assert(!app_context_string.isBogus()); + if(output.isBogus() || cached_context_string.isBogus() || app_context_string.isBogus()) { + return false; + } + int nfu_to_delete = 0; + + /* + Further debug assertion of inputs + */ + + assert(nfd->isNormalized(output, icu_status) && U_SUCCESS(icu_status)); + assert(nfd->isNormalized(cached_context_string, icu_status) && U_SUCCESS(icu_status)); + + /* + The keyboard processor will have updated the cached_context already, + applying the transform to it, so we need to rewind this. Remove the output + from cached_context_string to start + */ + + assert(cached_context_string.length() >= output.length()); + int n = cached_context_string.length() - output.length(); + assert(cached_context_string.compare(n, output.length(), output) == 0); + cached_context_string.remove(n); + + /* + While cached_context is guaranteed to be normalized, actions->output may not + start at a normalization boundary. In order to achieve the correct NFC + normalization in our output, we now need to look for a normalization + boundary prior to the intersection of the cached_context and the output. + */ + + while(n > 0 && output[0] && !nfd->hasBoundaryBefore(output[0])) { + // The output may interact with the context further in normalization. We + // need to copy characters back further until we reach a normalization + // boundary. + + // Remove last code point from the context ... + + n = cached_context_string.moveIndex32(n, -1); + UChar32 chr = cached_context_string.char32At(n); + cached_context_string.remove(n); + + // And prepend it to the output ... + + output.insert(0, chr); + + // And finally remember that we now need to delete an additional NFD codepoint + + actions->code_points_to_delete++; + } + + /* + At this point, our output and cached_context are coherent and normalization + will be complete at the edit boundary. + + Now, we need to adjust the delete_back to match the number of characters + that must actually be deleted from the applications's NFU context + + To adjust, we remove one character at a time from the app_context until + its normalized form matches the cached_context normalized form. + */ + + while(app_context_string.length()) { + icu::UnicodeString app_context_nfd; + nfd->normalize(app_context_string, app_context_nfd, icu_status); + assert(U_SUCCESS(icu_status)); + if(!U_SUCCESS(icu_status)) { + DebugLog("nfd->normalize failed with %x", icu_status); + return false; + } + + if(app_context_nfd.compare(cached_context_string) == 0) { + break; + } + app_context_string.remove(app_context_string.length()-1); + nfu_to_delete++; + } + + /* + Normalize our output string + */ + + icu::UnicodeString output_nfc; + nfc->normalize(output, output_nfc, icu_status); + assert(U_SUCCESS(icu_status)); + if(!U_SUCCESS(icu_status)) { + DebugLog("nfc->normalize failed with %x", icu_status); + return false; + } + + auto new_output = unicode_string_to_usv(output_nfc); + if(!new_output) { + // error logging handled in unicode_string_to_usv + return false; + } + + /* + Final steps -- set our outputs + */ + + // Append the new NFC output to our reduced app_context + + app_context_string.append(output_nfc); + km_core_context_item *app_context_items = nullptr; + km_core_status status = KM_CORE_STATUS_OK; + if((status = km_core_context_items_from_utf16(app_context_string.getTerminatedBuffer(), &app_context_items)) != KM_CORE_STATUS_OK) { + DebugLog("km_core_context_items_from_utf16 failed with %x", status); + delete [] new_output; + return false; + } + + if((status = km_core_context_set(app_context, app_context_items)) != KM_CORE_STATUS_OK) { + DebugLog("km_core_context_set failed with %x", status); + km_core_context_items_dispose(app_context_items); + delete [] new_output; + return false; + } + + km_core_context_items_dispose(app_context_items); + + // Update actions with new NFC output + count of NFU code points to delete + + delete [] actions->output; + actions->output = new_output; + actions->code_points_to_delete = nfu_to_delete; + + return true; +} + +/** + * Helper to convert km_core_context list into a icu::UnicodeString + */ +icu::UnicodeString context_items_to_unicode_string(km_core_context const *context) { + icu::UnicodeString nullString; + nullString.setToBogus(); + + km_core_context_item *items = nullptr; + km_core_status status; + if((status = km_core_context_get(context, &items)) != KM_CORE_STATUS_OK) { + DebugLog("Failed to retrieve context with %s", status); + return nullString; + } + size_t buf_size = 0; + if((status = km_core_context_items_to_utf32(items, nullptr, &buf_size)) != KM_CORE_STATUS_OK) { + DebugLog("Failed to retrieve context size with %s", status); + km_core_context_items_dispose(items); + return nullString; + } + + km_core_usv *buf = new km_core_usv[buf_size]; + if((status = km_core_context_items_to_utf32(items, buf, &buf_size)) != KM_CORE_STATUS_OK) { + DebugLog("Failed to retrieve context with %s", status); + km_core_context_items_dispose(items); + delete [] buf; + return nullString; + } + + auto result = icu::UnicodeString::fromUTF32(reinterpret_cast(buf), -1); + km_core_context_items_dispose(items); + delete [] buf; + return result; +} + +/** + * Helper to convert icu::UnicodeString to a UTF-32 km_core_usv buffer, + * nul-terminated + */ +km_core_usv *unicode_string_to_usv(icu::UnicodeString& src) { + UErrorCode icu_status = U_ZERO_ERROR; + + km_core_usv *dst = new km_core_usv[src.length() + 1]; + + src.toUTF32(reinterpret_cast(dst), src.length(), icu_status); + + assert(U_SUCCESS(icu_status)); + if(!U_SUCCESS(icu_status)) { + DebugLog("toUTF32 failed with %x", icu_status); + delete[] dst; + return nullptr; + } + + dst[src.length()] = 0; + return dst; +} + + + +/** + * Refresh app_context to match the cached_context. Does not do normalization, + * unlike `actions_normalize`. Used in conjunction with keyboard processors that + * do not support normalization. + * + * @param cached_context the cached context, in NFU, after transform has been + * applied to it by the keyboard processor + * @param app_context the app context, in NFU; transform has not been + * applied, and will effectively be applied by this + * function + * @return true on success, false on failure + */ +bool km::core::actions_update_app_context_nfu( + /* in */ km_core_context const *cached_context, + /* in, out */ km_core_context *app_context +) { + // We simply copy the cached_context to the app_context + km_core_status status = KM_CORE_STATUS_OK; + km_core_context_item *items = nullptr; + + if((status = km_core_context_get(cached_context, &items)) != KM_CORE_STATUS_OK) { + DebugLog("km_core_context_get failed with %d", status); + return false; + } + + if((status = km_core_context_set(app_context, items)) != KM_CORE_STATUS_OK) { + DebugLog("km_core_context_set failed with %d", status); + } + + delete [] items; + + return status == KM_CORE_STATUS_OK; +} \ No newline at end of file diff --git a/core/src/context.hpp b/core/src/context.hpp index dba7bec060..10ad878e9e 100644 --- a/core/src/context.hpp +++ b/core/src/context.hpp @@ -9,7 +9,7 @@ #pragma once #include #include -#include +#include "keyman_core.h" // Forward declarations class json; @@ -39,6 +39,11 @@ void context::push_marker(uint32_t marker) { emplace_back(km_core_context_item { KM_CORE_CT_MARKER, {0,}, {marker} }); } +// Context helper functions + +km_core_cp* get_context_as_string(km_core_context *context); +km_core_status set_context_from_string(km_core_context *context, km_core_cp const *new_context); + } // namespace core } // namespace km diff --git a/core/src/context_helpers.cpp b/core/src/context_helpers.cpp new file mode 100644 index 0000000000..936f68be52 --- /dev/null +++ b/core/src/context_helpers.cpp @@ -0,0 +1,69 @@ +/* + Copyright: © 2018-2024 SIL International. + Description: Helper functions for context data type conversions + Create Date: 18 Jan 2024 + Authors: Marc Durdin + History: 18 Jan 2024 - MCD - Refactor from km_core_state_context_set_if_needed.cpp +*/ +#include + +#include "keyman_core.h" +#include "context.hpp" + +using namespace km::core; + +/** + * Retrieves the context as a km_core_cp string, dropping markers + */ +km_core_cp* km::core::get_context_as_string(km_core_context *context) { + assert(context != nullptr); + if(context == nullptr) { + return nullptr; + } + + size_t buf_size = 0; + km_core_context_item* context_items = nullptr; + + if(km_core_context_get(context, &context_items) != KM_CORE_STATUS_OK) { + return nullptr; + } + + if(km_core_context_items_to_utf16(context_items, nullptr, &buf_size) != KM_CORE_STATUS_OK) { + km_core_context_items_dispose(context_items); + return nullptr; + } + + km_core_cp *app_context_string = new km_core_cp[buf_size]; + + km_core_status status = km_core_context_items_to_utf16(context_items, app_context_string, &buf_size); + km_core_context_items_dispose(context_items); + + if(status != KM_CORE_STATUS_OK) { + return nullptr; + } + + return app_context_string; +} + +/** + * Updates the context from the new_context km_core_cp string + */ +km_core_status km::core::set_context_from_string(km_core_context *context, km_core_cp const *new_context) { + assert(context != nullptr); + assert(new_context != nullptr); + if(context == nullptr || new_context == nullptr) { + return KM_CORE_STATUS_INVALID_ARGUMENT; + } + + km_core_context_item* new_context_items = nullptr; + + km_core_status status = km_core_context_items_from_utf16(new_context, &new_context_items); + if (status != KM_CORE_STATUS_OK) { + return status; + } + + km_core_context_set(context, new_context_items); + km_core_context_items_dispose(new_context_items); + + return KM_CORE_STATUS_OK; +} diff --git a/core/src/core_icu.h b/core/src/core_icu.h new file mode 100644 index 0000000000..0461bcecfa --- /dev/null +++ b/core/src/core_icu.h @@ -0,0 +1,13 @@ +/** + * ICU modules used by Keyman Core + */ +#pragma once + +#if !defined(HAVE_ICU4C) +#error icu4c is required for this code +#endif + +#define U_FALLTHROUGH +#include "unicode/utypes.h" +#include "unicode/unistr.h" +#include "unicode/normalizer2.h" diff --git a/core/src/debug.hpp b/core/src/debug.hpp index a68d9f1e57..33de3dd324 100644 --- a/core/src/debug.hpp +++ b/core/src/debug.hpp @@ -9,8 +9,7 @@ #include #include -#include -#include +#include "keyman_core.h" namespace km { namespace core diff --git a/core/src/keyboard.hpp b/core/src/keyboard.hpp index 6dd8517ce1..142bf40e86 100644 --- a/core/src/keyboard.hpp +++ b/core/src/keyboard.hpp @@ -11,7 +11,7 @@ #include #include -#include +#include "keyman_core.h" #include "option.hpp" #include "path.hpp" diff --git a/core/src/keyman_core.h b/core/src/keyman_core.h new file mode 100644 index 0000000000..8750b7b93e --- /dev/null +++ b/core/src/keyman_core.h @@ -0,0 +1,5 @@ +#pragma once + +#include +#include +#include \ No newline at end of file diff --git a/core/src/km_core_action_api.cpp b/core/src/km_core_action_api.cpp index 1177ec71fc..9a57760418 100644 --- a/core/src/km_core_action_api.cpp +++ b/core/src/km_core_action_api.cpp @@ -10,7 +10,8 @@ #include #include -#include +#include "keyman_core.h" + #include "jsonpp.hpp" #include "processor.hpp" @@ -32,7 +33,22 @@ km_core_actions const * km_core_state_get_actions( return nullptr; } - return action_item_list_to_actions_object(action_items); + km_core_actions * result = action_item_list_to_actions_object(action_items); + + if(state->processor().supports_normalization()) { + // Normalize to NFC for those keyboard processors that support it + if(!actions_normalize(km_core_state_context(state), km_core_state_app_context(state), result)) { + km_core_actions_dispose(result); + return nullptr; + } + } else { + // For all other keyboard processors, we just copy the cached_context to the app_context + if(!actions_update_app_context_nfu(km_core_state_context(state), km_core_state_app_context(state))) { + km_core_actions_dispose(result); + return nullptr; + } + } + return result; } km_core_status km_core_actions_dispose( diff --git a/core/src/km_core_context_api.cpp b/core/src/km_core_context_api.cpp index 46522710c0..ddf49e21e1 100644 --- a/core/src/km_core_context_api.cpp +++ b/core/src/km_core_context_api.cpp @@ -12,7 +12,7 @@ #include #include -#include +#include "keyman_core.h" #include "context.hpp" #include "jsonpp.hpp" diff --git a/core/src/km_core_debug_api.cpp b/core/src/km_core_debug_api.cpp index 2c6433a522..75e34b0b0c 100644 --- a/core/src/km_core_debug_api.cpp +++ b/core/src/km_core_debug_api.cpp @@ -8,7 +8,7 @@ #include #include -#include +#include "keyman_core.h" #include "processor.hpp" #include "state.hpp" diff --git a/core/src/km_core_keyboard_api.cpp b/core/src/km_core_keyboard_api.cpp index ae825ca402..964f73ca07 100644 --- a/core/src/km_core_keyboard_api.cpp +++ b/core/src/km_core_keyboard_api.cpp @@ -11,7 +11,8 @@ #include #include -#include +#include "keyman_core.h" + #include "keyboard.hpp" #include "processor.hpp" #include "kmx/kmx_processor.hpp" diff --git a/core/src/km_core_options_api.cpp b/core/src/km_core_options_api.cpp index 3cb0745fa9..4ea5c74062 100644 --- a/core/src/km_core_options_api.cpp +++ b/core/src/km_core_options_api.cpp @@ -11,7 +11,8 @@ #include #include -#include +#include "keyman_core.h" + #include "processor.hpp" #include "jsonpp.hpp" diff --git a/core/src/km_core_processevent_api.cpp b/core/src/km_core_processevent_api.cpp index 5295d39af7..7d5109a6b2 100644 --- a/core/src/km_core_processevent_api.cpp +++ b/core/src/km_core_processevent_api.cpp @@ -8,7 +8,8 @@ History: 17 Oct 2018 - TSE - Initial implementation. */ -#include +#include "keyman_core.h" + #include "processor.hpp" #include "state.hpp" diff --git a/core/src/km_core_state_api.cpp b/core/src/km_core_state_api.cpp index 2a25d6fcc7..03e041322d 100644 --- a/core/src/km_core_state_api.cpp +++ b/core/src/km_core_state_api.cpp @@ -12,13 +12,16 @@ #include #include #include +#include +#include -#include +#include "keyman_core.h" #include "jsonpp.hpp" #include "processor.hpp" #include "state.hpp" + using namespace km::core; // Forward declarations @@ -62,12 +65,20 @@ void km_core_state_dispose(km_core_state *state) } -km_core_context *km_core_state_context(km_core_state *state) +km_core_context *km_core_state_context(km_core_state const *state) { assert(state); if (!state) return nullptr; - return static_cast(&state->context()); + return static_cast(&(const_cast(state)->context())); +} + +km_core_context *km_core_state_app_context(km_core_state const *state) +{ + assert(state); + if (!state) return nullptr; + + return static_cast(&(const_cast(state)->app_context())); } km_core_status km_core_state_get_intermediate_context( @@ -261,95 +272,6 @@ void km_core_state_imx_deregister_callback(km_core_state *state) state->imx_deregister_callback(); } -bool is_context_valid(km_core_cp const * context, km_core_cp const * cached_context) { - if (context == nullptr || cached_context == nullptr || *cached_context == '\0') { - // If the cached_context is "empty" then it needs updating - return false; - } - km_core_cp const* context_p = context; - while(*context_p) { - context_p++; - } - - km_core_cp const* cached_context_p = cached_context; - while(*cached_context_p) { - cached_context_p++; - } - - // we need to compare from the end of the cached context - for(; context_p >= context && cached_context_p >= cached_context; context_p--, cached_context_p--) { - if(*context_p != *cached_context_p) { - // The cached context doesn't match the application context, so it is - // invalid - return false; - } - } - - if(cached_context_p > cached_context) { - // if the cached context is longer than the application context, then we also - // assume that it is invalid - return false; - } - - // It's acceptable for the application context to be longer than the cached - // context, so if we match the whole cached context, we can safely return true - return true; -} - -km_core_context_status km_core_state_context_set_if_needed( - km_core_state *state, - km_core_cp const *application_context -) { - assert(state != nullptr); - assert(application_context != nullptr); - if(state == nullptr || application_context == nullptr) { - return KM_CORE_CONTEXT_STATUS_INVALID_ARGUMENT; - } - - size_t buf_size; - km_core_context_item* context_items = nullptr; - - auto context = km_core_state_context(state); - if(km_core_context_get(context, &context_items) != KM_CORE_STATUS_OK) { - return KM_CORE_CONTEXT_STATUS_ERROR; - } - - if(km_core_context_items_to_utf16(context_items, nullptr, &buf_size) != KM_CORE_STATUS_OK) { - km_core_context_items_dispose(context_items); - return KM_CORE_CONTEXT_STATUS_ERROR; - } - - std::unique_ptr cached_context(new km_core_cp[buf_size]); - - km_core_status status = km_core_context_items_to_utf16(context_items, cached_context.get(), &buf_size); - km_core_context_items_dispose(context_items); - - if(status != KM_CORE_STATUS_OK) { - return KM_CORE_CONTEXT_STATUS_ERROR; - } - - bool is_valid = is_context_valid(application_context, cached_context.get()); - - if(is_valid) { - // We keep the context as is - return KM_CORE_CONTEXT_STATUS_UNCHANGED; - } - - km_core_context_item* new_context_items = nullptr; - - // We replace the cached context with the current application context - status = km_core_context_items_from_utf16(application_context, &new_context_items); - if (status != KM_CORE_STATUS_OK) { - km_core_context_clear(context); - return KM_CORE_CONTEXT_STATUS_CLEARED; - } - - km_core_context_set(context, new_context_items); - km_core_context_items_dispose(new_context_items); - return KM_CORE_CONTEXT_STATUS_UPDATED; -} - - km_core_status km_core_state_context_clear( km_core_state *state ) { @@ -358,5 +280,87 @@ km_core_status km_core_state_context_clear( return KM_CORE_STATUS_INVALID_ARGUMENT; } km_core_context_clear(km_core_state_context(state)); + km_core_context_clear(km_core_state_app_context(state)); return KM_CORE_STATUS_OK; } + +void km_core_cp_dispose( + km_core_cp *cp +) { + if(cp != nullptr) { + delete [] cp; + } +} + +km_core_cp * _new_error_string(std::u16string const str) { + km_core_cp* result = new km_core_cp[str.size()+1]; + str.copy(result, str.size()); + result[str.size()] = 0; + return result; +} + +km_core_cp * km_core_state_context_debug( + km_core_state *state, + km_core_debug_context_type context_type +) { + km_core_context_item * context_items = nullptr; + + if(context_type == KM_CORE_DEBUG_CONTEXT_INTERMEDIATE) { + if(km_core_state_get_intermediate_context(state, &context_items) != KM_CORE_STATUS_OK) { + return _new_error_string(u""); + } + } else if(context_type == KM_CORE_DEBUG_CONTEXT_CACHED) { + if(km_core_context_get(km_core_state_context(state), &context_items) != KM_CORE_STATUS_OK) { + return _new_error_string(u""); + } + } else if(context_type == KM_CORE_DEBUG_CONTEXT_APP) { + if(km_core_context_get(km_core_state_app_context(state), &context_items) != KM_CORE_STATUS_OK) { + return _new_error_string(u""); + } + } else { + return _new_error_string(u""); + } + + size_t buf_size; + if(km_core_context_items_to_utf8(context_items, nullptr, &buf_size) != KM_CORE_STATUS_OK) { + km_core_context_items_dispose(context_items); + return _new_error_string(u""); + } + + std::vector context_buffer(buf_size); + if(km_core_context_items_to_utf8(context_items, &context_buffer[0], &buf_size) != KM_CORE_STATUS_OK) { + km_core_context_items_dispose(context_items); + return _new_error_string(u""); + } + + // construct the log message + + std::stringstream buffer; + + int context_item_length = 0; + for(auto cp = context_items; cp->type != KM_CORE_CT_END; cp++, context_item_length++); + + buffer << "|" << std::string(&context_buffer[0]) << "| (len: " << context_item_length << ") ["; + for(auto cp = context_items; cp->type != KM_CORE_CT_END; cp++) { + auto flags = buffer.flags(); + if(cp->type == KM_CORE_CT_CHAR) { + // A single Unicode codepoint + buffer << " U+" << std::setfill('0') << std::setw(4) << std::hex << cp->character; + } else { + // A marker + buffer << " M(" << cp->marker << ")"; + } + buffer.flags(flags); + } + buffer << " ]"; + + km_core_context_items_dispose(context_items); + + std::u16string s = std::wstring_convert, char16_t>{}.from_bytes(buffer.str()); + + km_core_cp* result = new km_core_cp[s.size() + 1]; + s.copy(result, s.size()); + result[s.size()] = 0; + + return result; +} \ No newline at end of file diff --git a/core/src/km_core_state_context_set_if_needed.cpp b/core/src/km_core_state_context_set_if_needed.cpp new file mode 100644 index 0000000000..361ce27dae --- /dev/null +++ b/core/src/km_core_state_context_set_if_needed.cpp @@ -0,0 +1,165 @@ +/* + Copyright: © 2018-2024 SIL International. + Description: Implementation of the state API functions using internal + data structures and functions. + Create Date: 15 Jan 2024 + Authors: Marc Durdin + History: 15 Jan 2024 - MCD - Refactor our km_core_state_context_set_if_needed + and implement normalization +*/ +#include + +#include "keyman_core.h" + +#include "processor.hpp" +#include "state.hpp" +#include "debuglog.h" +#include "core_icu.h" + +using namespace km::core; + +// Forward declarations + +bool should_normalize(km_core_state *state); +bool is_context_valid(km_core_cp const * context, km_core_cp const * cached_context); +bool do_normalize_nfd(km_core_cp const * src, std::u16string &dst); +km_core_context_status do_fail(km_core_context *app_context, km_core_context *cached_context, const char* error); + +// --------------------------------------------------------------------------- + +km_core_context_status km_core_state_context_set_if_needed( + km_core_state *state, + km_core_cp const *new_app_context +) { + assert(state != nullptr); + assert(new_app_context != nullptr); + if(state == nullptr || new_app_context == nullptr) { + return KM_CORE_CONTEXT_STATUS_INVALID_ARGUMENT; + } + + auto app_context = km_core_state_app_context(state); + auto cached_context = km_core_state_context(state); + + // Retrieve the existing internally cached app context for comparison + + std::unique_ptr app_context_string(get_context_as_string(app_context)); + + // Compare the internal app context with the passed-in application context + + if(is_context_valid(new_app_context, app_context_string.get())) { + // We keep the context as is + return KM_CORE_CONTEXT_STATUS_UNCHANGED; + } + + // We replace the internal app context with the passed-in application context + + if(set_context_from_string(app_context, new_app_context) != KM_CORE_STATUS_OK) { + return do_fail(app_context, cached_context, "could not set new app context"); + } + + // Finally, we normalize and replace the cached context + + std::u16string normalized_buffer; + km_core_cp const *new_cached_context = nullptr; + + if(should_normalize(state)) { + if(!do_normalize_nfd(new_app_context, normalized_buffer)) { + return do_fail(app_context, cached_context, "could not normalize string"); + } + new_cached_context = normalized_buffer.c_str(); + } else { + new_cached_context = new_app_context; + } + + // TODO: #10100 will alter how we replace the cached context here -- maintaining + // markers as far as possible + + if(set_context_from_string(cached_context, new_cached_context) != KM_CORE_STATUS_OK) { + return do_fail(app_context, cached_context, "could not set new cached context"); + } + + return KM_CORE_CONTEXT_STATUS_UPDATED; +} + +/** + * Returns true if the current keyboard processor wants a normalized cached context + */ +bool should_normalize(km_core_state *state) { + return state->processor().supports_normalization(); +} + +/** + * Returns true if the internal app context does not need to be updated to the new + * app context + * + * TODO: #10100 will alter some of the assumptions here + */ +bool is_context_valid(km_core_cp const * new_app_context, km_core_cp const * app_context) { + if (new_app_context == nullptr || app_context == nullptr || *app_context == '\0') { + // If the app_context is "empty" then it needs updating + return false; + } + km_core_cp const* new_app_context_p = new_app_context; + while(*new_app_context_p) { + new_app_context_p++; + } + + km_core_cp const* app_context_p = app_context; + while(*app_context_p) { + app_context_p++; + } + + // we need to compare from the end of the cached context + for(; new_app_context_p >= new_app_context && app_context_p >= app_context; new_app_context_p--, app_context_p--) { + if(*new_app_context_p != *app_context_p) { + // The cached context doesn't match the application context, so it is + // invalid + return false; + } + } + + if(app_context_p > app_context) { + // if the cached context is longer than the application context, then we also + // assume that it is invalid + return false; + } + + // It's acceptable for the application context to be longer than the cached + // context, so if we match the whole cached context, we can safely return true + return true; +} + +/** + * Normalize the input string using ICU + */ +bool do_normalize_nfd(km_core_cp const * src, std::u16string &dst) { + UErrorCode icu_status = U_ZERO_ERROR; + const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(icu_status); + assert(U_SUCCESS(icu_status)); + if(!U_SUCCESS(icu_status)) { + // TODO: log the failure code + return false; + } + icu::UnicodeString udst; + icu::UnicodeString usrc = icu::UnicodeString(src); + nfd->normalize(usrc, udst, icu_status); + assert(U_SUCCESS(icu_status)); + if(!U_SUCCESS(icu_status)) { + // TODO: log the failure code + return false; + } + + dst.assign(udst.getBuffer(), udst.length()); + return true; +} + +/** + * Clear the context when we have a failure so we don't end up with inconsistent + * context buffers, and log the error to our diagnostic log. + */ +km_core_context_status do_fail(km_core_context *app_context, km_core_context *cached_context, const char* error) { + DebugLog("%s", error); + km_core_context_clear(app_context); + km_core_context_clear(cached_context); + return KM_CORE_CONTEXT_STATUS_CLEARED; +} \ No newline at end of file diff --git a/core/src/kmx/kmx_base.h b/core/src/kmx/kmx_base.h index fed03bc645..e798ff56b7 100644 --- a/core/src/kmx/kmx_base.h +++ b/core/src/kmx/kmx_base.h @@ -1,7 +1,6 @@ #pragma once -#include -#include +#include "keyman_core.h" #if defined(_WIN32) || defined(_WIN64) #define snprintf _snprintf diff --git a/core/src/kmx/kmx_options.h b/core/src/kmx/kmx_options.h index 14c280e411..cf038ac7dc 100644 --- a/core/src/kmx/kmx_options.h +++ b/core/src/kmx/kmx_options.h @@ -4,7 +4,7 @@ #include #include -#include +#include "keyman_core.h" #include "option.hpp" #include "kmx_base.h" diff --git a/core/src/kmx/kmx_processor.cpp b/core/src/kmx/kmx_processor.cpp index 1fa0734e07..95a9a09456 100644 --- a/core/src/kmx/kmx_processor.cpp +++ b/core/src/kmx/kmx_processor.cpp @@ -1,4 +1,4 @@ -#include +#include "keyman_core.h" #include "state.hpp" #include "kmx/kmx_processor.hpp" #include diff --git a/core/src/kmx/kmx_processor.hpp b/core/src/kmx/kmx_processor.hpp index b83e8bba9e..08b2c1d242 100644 --- a/core/src/kmx/kmx_processor.hpp +++ b/core/src/kmx/kmx_processor.hpp @@ -9,7 +9,7 @@ #pragma once #include -#include +#include "keyman_core.h" #include "kmx/kmx_processevent.h" #include "keyboard.hpp" #include "processor.hpp" @@ -80,6 +80,10 @@ namespace core km_core_keyboard_imx * get_imx_list() const override; + bool + supports_normalization() const override { + return false; + } }; } // namespace core diff --git a/core/src/ldml/ldml_markers.hpp b/core/src/ldml/ldml_markers.hpp index 68c7aa2f2d..4ec9b71ae2 100644 --- a/core/src/ldml/ldml_markers.hpp +++ b/core/src/ldml/ldml_markers.hpp @@ -16,18 +16,11 @@ #include #include "debuglog.h" -#if !defined(HAVE_ICU4C) -#error icu4c is required for this code -#endif - -#define U_FALLTHROUGH -#include "unicode/utypes.h" +#include "core_icu.h" #include "unicode/uniset.h" #include "unicode/usetiter.h" -#include "unicode/unistr.h" #include "unicode/regex.h" #include "unicode/utext.h" -#include "unicode/normalizer2.h" namespace km { namespace core { diff --git a/core/src/ldml/ldml_processor.cpp b/core/src/ldml/ldml_processor.cpp index 6e2bd4b652..769b6f11f2 100644 --- a/core/src/ldml/ldml_processor.cpp +++ b/core/src/ldml/ldml_processor.cpp @@ -151,24 +151,20 @@ bool ldml_processor::is_kmxplus_file(path const & kb_path, std::vector& km_core_status ldml_processor::process_queued_actions( - km_core_state *state + km_core_state *_kmn_unused(state) ) { - assert(state); - if (!state) - return KM_CORE_STATUS_INVALID_ARGUMENT; - // TODO Implement - return KM_CORE_STATUS_OK; + /* Not implemented. Only used by IMX. */ + assert(false); + return KM_CORE_STATUS_NOT_IMPLEMENTED; } bool ldml_processor::queue_action( - km_core_state * state, - km_core_action_item const* action_item + km_core_state * _kmn_unused(state), + km_core_action_item const* _kmn_unused(action_item) ) { - assert(state); - assert(action_item); - if ((!state) || (!action_item)) - return false; + /* Not implemented. Only used by IMX. */ + assert(false); return false; } @@ -178,59 +174,68 @@ ldml_processor::process_event( km_core_virtual_key vk, uint16_t modifier_state, uint8_t is_key_down, - uint16_t _kmn_unused(event_flags) + uint16_t event_flags ) { assert(state); if (!state) return KM_CORE_STATUS_INVALID_ARGUMENT; - try { - // At the start of every process_event always clear the action_items - state->actions().clear(); + // this will hold the parameters, and our response + ldml_event_state ldml_state(state, vk, modifier_state, is_key_down, event_flags); + ldml_state.clear(); + try { if (!is_key_down) { - process_key_up(state, vk, modifier_state); + process_key_up(ldml_state); } else { switch (vk) { // Currently, only one VK gets spoecial treatment. // Special handling for backspace VK case KM_CORE_VKEY_BKSP: - process_backspace(state); + process_backspace(ldml_state); break; default: // all other VKs - process_key_down(state, vk, modifier_state); + process_key_down(ldml_state); } // end of switch } // end of normal processing // all key-up and key-down events end up here. - state->actions().commit(); // always commit + // commit the ldml state into the core state + ldml_state.commit(); return KM_CORE_STATUS_OK; } catch (std::bad_alloc &) { // out of memory, clean up and get out - state->actions().clear(); + ldml_state.clear(); + // no actions will be set return KM_CORE_STATUS_NO_MEM; } } void -ldml_processor::process_key_up(km_core_state *state, km_core_virtual_key _kmn_unused(vk), uint16_t _kmn_unused(modifier_state)) +ldml_processor::process_key_up(ldml_event_state &ldml_state) const { // TODO-LDML: Implement caps lock handling - state->actions().clear(); // TODO-LDML: Why is clear here? + ldml_state.clear(); } void -ldml_processor::process_backspace(km_core_state *state) const { +ldml_processor::process_backspace(ldml_event_state &ldml_state) const { if (!!bksp_transforms) { - // process with an empty string - auto matchedContext = process_output(state, std::u32string(), bksp_transforms.get()); + // process with an empty string voa the bksp transforms + auto matchedContext = process_output(ldml_state, std::u32string(), bksp_transforms.get()); if (matchedContext > 0) { return; // The transform took care of the backspacing. } // else, fall through to default processing below. } + ldml_state.emit_backspace(); +} + +void ldml_event_state::emit_backspace() { + // this is called from user-initiated backspace, not internal backspacing. + // Find out what the last actual character was and remove it. // attempt to get the last char // TODO-LDML: emoji backspace @@ -243,6 +248,7 @@ ldml_processor::process_backspace(km_core_state *state) const { } else if ((*end).type == KM_CORE_BT_MARKER) { state->actions().push_backspace(KM_CORE_BT_MARKER, (*end).marker); state->context().pop_back(); + // TODO-LDML: fall through here? } } /* @@ -257,31 +263,31 @@ ldml_processor::process_backspace(km_core_state *state) const { } void -ldml_processor::process_key_down(km_core_state *state, km_core_virtual_key vk, uint16_t modifier_state) const { +ldml_processor::process_key_down(ldml_event_state &ldml_state) const { // Look up the key bool found = false; - const std::u16string key_str = keys.lookup(vk, modifier_state, found); + const std::u16string key_str = keys.lookup(ldml_state.get_vk(), ldml_state.get_modifier_state(), found); if (!found) { // no key was found, so pass the keystroke on to the Engine - emit_invalidate_passthrough_keystroke(state, vk, modifier_state); + ldml_state.emit_invalidate_passthrough_keystroke(); } else if (!key_str.empty()) { - process_key_string(state, key_str); + process_key_string(ldml_state, key_str); } // else no action: It's a gap or gap-like key. } void -ldml_processor::process_key_string(km_core_state *state, const std::u16string &key_str) const { +ldml_processor::process_key_string(ldml_event_state &ldml_state, const std::u16string &key_str) const { // We know that key_str is not empty per the caller. assert(!key_str.empty()); // we convert the keys str to UTF-32 here instead of using the emit_text() overload // so that we don't have to reconvert it inside the transform code. std::u32string key_str32 = kmx::u16string_to_u32string(key_str); - (void)process_output(state, key_str32, transforms.get()); + (void)process_output(ldml_state, key_str32, transforms.get()); } -size_t ldml_processor::process_output(km_core_state *state, const std::u32string &str, ldml::transforms *with_transforms) const { +size_t ldml_processor::process_output(ldml_event_state &ldml_state, const std::u32string &str, ldml::transforms *with_transforms) const { std::u32string nfd_str = str; // Note: // The normalize functions have assert and Debuglog at the bottom. @@ -291,12 +297,12 @@ size_t ldml_processor::process_output(km_core_state *state, const std::u32string // extract context string, in NFD std::u32string old_ctxtstr_nfd; - (void)context_to_string(state, old_ctxtstr_nfd, true); - (void)ldml::normalize_nfd_markers(old_ctxtstr_nfd); + (void)ldml_state.context_to_string(old_ctxtstr_nfd, true); + assert(ldml::normalize_nfd_markers(old_ctxtstr_nfd)); // TODO-LDML: else fail? // context string in NFD std::u32string ctxtstr; - (void)context_to_string(state, ctxtstr, true); // with markers + (void)ldml_state.context_to_string(ctxtstr, true); // TODO-LDML: remove this second call // add the newly added key output to ctxtstr ctxtstr.append(nfd_str); (void)ldml::normalize_nfd_markers(ctxtstr); @@ -367,20 +373,21 @@ size_t ldml_processor::process_output(km_core_state *state, const std::u32string // FIRST drop the old suffix. Note: this mutates old_ctxtstr_changed. // see remove_text() docs, this PUSHes actions, POPs context items, and TRIMS the string. - remove_text(state, old_ctxtstr_changed, old_ctxtstr_changed.length()); + ldml_state.remove_text(old_ctxtstr_changed, old_ctxtstr_changed.length()); assert(old_ctxtstr_changed.length() == 0); // old_ctxtstr_changed is now empty because it's been removed. // context is "aa" in the above example. // THEN add the new suffix, "CCC" in the above example - emit_text(state, new_ctxtstr_changed); + ldml_state.emit_text(new_ctxtstr_changed); // context is now "aaCCC" return matchedContext; } + void -ldml_processor::remove_text(km_core_state *state, std::u32string &str, size_t length) { +ldml_event_state::remove_text(std::u32string &str, size_t length) { // str is the string to remove, so it should be at least as long as length assert(length <= str.length()); /** track how many context items have been removed, via push_backspace() */ @@ -446,13 +453,13 @@ km_core_status ldml_processor::validate() const { } void -ldml_processor::emit_text(km_core_state *state, const std::u16string &str) { +ldml_event_state::emit_text(const std::u16string &str) { const std::u32string str32 = kmx::u16string_to_u32string(str); - emit_text(state, str32); + emit_text(str32); } void -ldml_processor::emit_text(km_core_state *state, const std::u32string &str) { +ldml_event_state::emit_text(const std::u32string &str) { for (auto it = str.begin(); it < str.end(); it++) { const auto ch = *it; // If we are at the start of a sequence: @@ -465,28 +472,28 @@ ldml_processor::emit_text(km_core_state *state, const std::u32string &str) { it++; // consume LDML_MARKER_CODE assert(it < str.end()); const auto marker_no = *it; - emit_marker(state, marker_no); + emit_marker(marker_no); } else { - emit_text(state, ch); + emit_text(ch); } } } void -ldml_processor::emit_text(km_core_state *state, km_core_usv ch) { +ldml_event_state::emit_text( km_core_usv ch) { assert(ch != LDML_UC_SENTINEL); state->context().push_character(ch); state->actions().push_character(ch); } void -ldml_processor::emit_marker(km_core_state *state, KMX_DWORD marker_no) { +ldml_event_state::emit_marker( KMX_DWORD marker_no) { assert(km::core::kmx::is_valid_marker(marker_no)); state->actions().push_marker(marker_no); state->context().push_marker(marker_no); } -void ldml_processor::emit_invalidate_passthrough_keystroke(km_core_state *state, km_core_virtual_key vk, uint16_t _kmn_unused(modifier_state)) { +void ldml_event_state::emit_invalidate_passthrough_keystroke() { if ((vk < 0x100) && km::core::kmx::vkey_to_contextreset[vk]) { state->actions().push_invalidate_context(); } else { @@ -496,10 +503,10 @@ void ldml_processor::emit_invalidate_passthrough_keystroke(km_core_state *state, } size_t -ldml_processor::context_to_string(km_core_state *state, std::u32string &str, bool include_markers) { +ldml_event_state::context_to_string(std::u32string &str, bool include_markers) { str.clear(); - auto &cp = state->context(); - size_t ctxlen = 0; // TODO-LDML: not used by callers? + auto &cp = state->context(); + size_t ctxlen = 0; // TODO-LDML: not used by callers? uint8_t last_type = KM_CORE_BT_UNKNOWN; for (auto c = cp.rbegin(); c != cp.rend(); c++, ctxlen++) { last_type = c->type; @@ -517,5 +524,26 @@ ldml_processor::context_to_string(km_core_state *state, std::u32string &str, boo return ctxlen; // consumed the entire context buffer. } +ldml_event_state::ldml_event_state( + km_core_state *s, + km_core_virtual_key v, + uint16_t m, + uint8_t i, + uint16_t e) { + this->state = s; + this->vk = v; + this->modifier_state = m; + this->is_key_down = i; + this->event_flags = e; +} + +void ldml_event_state::commit() { + state->actions().commit(); +} + +void ldml_event_state::clear() { + state->actions().clear(); +} + } // namespace core } // namespace km diff --git a/core/src/ldml/ldml_processor.hpp b/core/src/ldml/ldml_processor.hpp index 7ad997a944..5701de02a0 100644 --- a/core/src/ldml/ldml_processor.hpp +++ b/core/src/ldml/ldml_processor.hpp @@ -13,7 +13,7 @@ #include #include #include -#include +#include "keyman_core.h" #include "processor.hpp" #include "option.hpp" #include "ldml_vkeys.hpp" @@ -24,19 +24,16 @@ namespace core { #define KM_CORE_LMDL_PROCESSOR_VERSION u"1.0" - class ldml_processor : public abstract_processor { - private: - bool _valid; - std::unique_ptr transforms, bksp_transforms; - ldml::vkeys keys; +class ldml_event_state; + +/** our actual processor */ +class ldml_processor : public abstract_processor { public: ldml_processor( path const & kb_path, const std::vector & data ); -// ~ldml_processor() override; - static bool is_kmxplus_file( path const & kb_path, std::vector& data @@ -84,57 +81,110 @@ namespace core { km_core_keyboard_imx * get_imx_list() const override; + bool + supports_normalization() const override { + return true; + } + private: - /** emit text to context and actions */ - static void emit_text(km_core_state *state, const std::u16string &str); - /** emit text to context and actions */ - static void emit_text(km_core_state *state, const std::u32string &str); - /** emit char to context and actions */ - static void emit_text(km_core_state *state, km_core_usv ch); - /** emit a marker */ - static void emit_marker(km_core_state *state, KMX_DWORD marker); - /** emit a pass-through and invalidate */ - static void emit_invalidate_passthrough_keystroke(km_core_state *state, km_core_virtual_key vk, uint16_t modifier_state); + /** process a key-up */ + void process_key_up(ldml_event_state &ldml_state) const; - /** - * Delete text from the state, by: - * 1. calling actions().push_backspace() to push the appropriate backspaces - * 2. popping the same items from the context items - * 3. mutating 'str' by removing the same number of items. - * This function handles marker strings correctly. - * @param str string with text to remove, from the end - * @param length number of chars from the end of str to drop - */ - static void remove_text(km_core_state *state, std::u32string &str, size_t length); + /** process a key-down (if it wasn't handled exceptionally) */ + void process_key_down(ldml_event_state &ldml_state) const; - /** process a key-up */ - void process_key_up(km_core_state *state, km_core_virtual_key vk, uint16_t modifier_state) const; + /** process a typed key */ + void process_key_string(ldml_event_state &ldml_state, const std::u16string &key_str) const; - /** process a key-down (if it wasn't handled exceptionally) */ - void process_key_down(km_core_state *state, km_core_virtual_key vk, uint16_t modifier_state) const; + /** process a backspace */ + void process_backspace(ldml_event_state &ldml_state) const; - /** process a typed key */ - void process_key_string(km_core_state *state, const std::u16string &key_str) const; + /** + * common function for outputting a string with transforms/normalization applied. + * @param str string to output (such as from a key), or empty + * @param with_transforms transforms to use or nullptr + * @returns length of matched input context + */ + size_t process_output(ldml_event_state &ldml_state, const std::u32string &str, ldml::transforms *with_transforms) const; - /** process a backspace */ - void process_backspace(km_core_state *state) const; + private: + bool _valid; + std::unique_ptr transforms, bksp_transforms; + ldml::vkeys keys; +}; - /** - * common function for outputting a string with transforms/normalization applied. - * @param str string to output (such as from a key), or empty - * @param with_transforms transforms to use or nullptr - * @returns length of matched input context - */ - size_t process_output(km_core_state *state, const std::u32string &str, ldml::transforms *with_transforms) const; - /** - * add the string+marker portion of the context to the beginning of str. - * Stop when a non-string and non-marker is hit. - * Convert markers into the UC_SENTINEL format. - * @return the number of context items consumed - */ - static size_t context_to_string(km_core_state *state, std::u32string &str, bool include_markers = true); +/** class holding state as we process an event. mirrors process_event args. */ +class ldml_event_state { +public: + ldml_event_state( + km_core_state *state, + km_core_virtual_key vk, + uint16_t modifier_state, + uint8_t is_key_down, + uint16_t event_flags); + /** done with this, copy it into core state */ + void commit(); + /** clear this object out */ + void clear(); + + // getters + inline km_core_virtual_key get_vk() const; + inline uint16_t get_modifier_state() const; + + // actions + + /** emit text to context and actions */ + void emit_text(const std::u16string &str); + /** emit text to context and actions */ + void emit_text(const std::u32string &str); + /** emit char to context and actions */ + void emit_text(km_core_usv ch); + /** emit a marker */ + void emit_marker(KMX_DWORD marker); + /** emit a pass-through and invalidate */ + void emit_invalidate_passthrough_keystroke(); + /** emit a backspace (for a user-initiated deletion) */ + void emit_backspace(); + + /** + * Delete text from the state, by: + * 1. calling actions().push_backspace() to push the appropriate backspaces + * 2. popping the same items from the context items + * 3. mutating 'str' by removing the same number of items. + * This function handles marker strings correctly. + * @param str string with text to remove, from the end + * @param length number of chars from the end of str to drop + */ + void remove_text(std::u32string &str, size_t length); + + /** + * add the string+marker portion of the context to the beginning of str. + * Stop when a non-string and non-marker is hit. + * Convert markers into the UC_SENTINEL format. + * @return the number of context items consumed + */ + size_t context_to_string(std::u32string &str, bool include_markers = true); + + private: + km_core_virtual_key vk; + uint16_t modifier_state; + uint8_t is_key_down; + uint16_t event_flags; + km_core_state *state; +}; + + +// implementation +km_core_virtual_key +ldml_event_state::get_vk() const { + return vk; +} + +uint16_t +ldml_event_state::get_modifier_state() const { + return modifier_state; +} - }; } // namespace core } // namespace km diff --git a/core/src/ldml/ldml_transforms.hpp b/core/src/ldml/ldml_transforms.hpp index 72da5c68a7..e6ef4e1523 100644 --- a/core/src/ldml/ldml_transforms.hpp +++ b/core/src/ldml/ldml_transforms.hpp @@ -16,18 +16,11 @@ #include #include "debuglog.h" -#if !defined(HAVE_ICU4C) -#error icu4c is required for this code -#endif - -#define U_FALLTHROUGH -#include "unicode/utypes.h" +#include "core_icu.h" #include "unicode/uniset.h" #include "unicode/usetiter.h" -#include "unicode/unistr.h" #include "unicode/regex.h" #include "unicode/utext.h" -#include "unicode/normalizer2.h" namespace km { namespace core { diff --git a/core/src/ldml/ldml_vkeys.hpp b/core/src/ldml/ldml_vkeys.hpp index 22dea817ef..ed3368dc7e 100644 --- a/core/src/ldml/ldml_vkeys.hpp +++ b/core/src/ldml/ldml_vkeys.hpp @@ -13,7 +13,7 @@ #include #include -#include +#include "keyman_core.h" namespace km { namespace core { diff --git a/core/src/meson.build b/core/src/meson.build index e5314294ad..9943c2fd07 100644 --- a/core/src/meson.build +++ b/core/src/meson.build @@ -42,7 +42,9 @@ endif kmx_files = files( + 'actions_normalize.cpp', 'action.cpp', + 'context_helpers.cpp', 'option.cpp', 'keyboard.cpp', 'state.cpp', @@ -52,6 +54,7 @@ kmx_files = files( 'km_core_keyboard_api.cpp', 'km_core_options_api.cpp', 'km_core_state_api.cpp', + 'km_core_state_context_set_if_needed.cpp', 'km_core_debug_api.cpp', 'km_core_processevent_api.cpp', 'jsonpp.cpp', diff --git a/core/src/mock/mock_processor.hpp b/core/src/mock/mock_processor.hpp index b10f4ad8d2..87a99a49df 100644 --- a/core/src/mock/mock_processor.hpp +++ b/core/src/mock/mock_processor.hpp @@ -10,7 +10,7 @@ #include #include -#include +#include "keyman_core.h" #include "processor.hpp" #include "option.hpp" @@ -65,6 +65,10 @@ namespace core km_core_keyboard_imx * get_imx_list() const override; + bool + supports_normalization() const override { + return false; + } }; class null_processor : public mock_processor { diff --git a/core/src/option.hpp b/core/src/option.hpp index 43f3397049..60026798cf 100644 --- a/core/src/option.hpp +++ b/core/src/option.hpp @@ -12,7 +12,8 @@ #include -#include +#include "keyman_core.h" + // Forward declarations class json; diff --git a/core/src/path.hpp b/core/src/path.hpp index be6ae61422..aee70ad8ba 100644 --- a/core/src/path.hpp +++ b/core/src/path.hpp @@ -12,7 +12,8 @@ #include #include -#include +#include "keyman_core.h" + #include "jsonpp.hpp" #include "utfcodec.hpp" diff --git a/core/src/processor.hpp b/core/src/processor.hpp index 2dd52e82d4..8925ad65d1 100644 --- a/core/src/processor.hpp +++ b/core/src/processor.hpp @@ -11,7 +11,7 @@ #include #include -#include +#include "keyman_core.h" #include "keyboard.hpp" @@ -122,6 +122,9 @@ namespace core virtual km_core_keyboard_imx * get_imx_list() const = 0; + virtual bool + supports_normalization() const = 0; + friend json & operator << (json &j, abstract_processor const &opts); }; diff --git a/core/src/state.hpp b/core/src/state.hpp index b7f8baa486..c3afff5029 100644 --- a/core/src/state.hpp +++ b/core/src/state.hpp @@ -11,7 +11,7 @@ #include #include -#include +#include "keyman_core.h" #include "context.hpp" #include "option.hpp" @@ -123,6 +123,7 @@ class state { protected: core::context _ctxt; + core::context _app_ctxt; core::abstract_processor & _processor; core::actions _actions; core::debug_items _debug_items; @@ -138,6 +139,9 @@ public: core::context & context() noexcept { return _ctxt; } core::context const & context() const noexcept { return _ctxt; } + core::context & app_context() noexcept { return _app_ctxt; } + core::context const & app_context() const noexcept { return _app_ctxt; } + core::abstract_processor const & processor() const noexcept { return _processor; } core::abstract_processor & processor() noexcept { return _processor; } diff --git a/core/tests/unit/kmnkbd/action_api.cpp b/core/tests/unit/kmnkbd/action_api.cpp index 6a84eafa2d..a9e0058b56 100644 --- a/core/tests/unit/kmnkbd/action_api.cpp +++ b/core/tests/unit/kmnkbd/action_api.cpp @@ -6,7 +6,7 @@ History: 23 Oct 2023 - MCD - Initial implementation. */ #include -#include +#include "keyman_core.h" #include "path.hpp" #include "action.hpp" diff --git a/core/tests/unit/kmnkbd/action_items.hpp b/core/tests/unit/kmnkbd/action_items.hpp index 5b4723fd57..dc5ac666b6 100644 --- a/core/tests/unit/kmnkbd/action_items.hpp +++ b/core/tests/unit/kmnkbd/action_items.hpp @@ -1,4 +1,4 @@ -#include +#include "keyman_core.h" #include #include diff --git a/core/tests/unit/kmnkbd/action_set_api.cpp b/core/tests/unit/kmnkbd/action_set_api.cpp index 378f3bcab1..2f69f2baba 100644 --- a/core/tests/unit/kmnkbd/action_set_api.cpp +++ b/core/tests/unit/kmnkbd/action_set_api.cpp @@ -6,7 +6,8 @@ History: 23 Oct 2023 - MCD - Initial implementation. */ #include -#include + +#include "keyman_core.h" #include "path.hpp" #include "action.hpp" @@ -60,6 +61,7 @@ void setup(const char *keyboard, const km_core_cp* context) { try_status(km_core_state_create(test_kb, test_env_opts, &test_state)); try_status(km_core_context_items_from_utf16(context, &citems)); try_status(km_core_context_set(km_core_state_context(test_state), citems)); + try_status(km_core_context_set(km_core_state_app_context(test_state), citems)); } void run_test(km_core_action_item const * action_items, const km_core_actions &actions) { diff --git a/core/tests/unit/kmnkbd/context_api.cpp b/core/tests/unit/kmnkbd/context_api.cpp index d5225eb24f..149ebf9556 100644 --- a/core/tests/unit/kmnkbd/context_api.cpp +++ b/core/tests/unit/kmnkbd/context_api.cpp @@ -10,7 +10,8 @@ mutation functions. */ #include -#include + +#include "keyman_core.h" #include "context.hpp" #include "utfcodec.hpp" diff --git a/core/tests/unit/kmnkbd/debug_api.cpp b/core/tests/unit/kmnkbd/debug_api.cpp index de0cfce8e8..3313888477 100644 --- a/core/tests/unit/kmnkbd/debug_api.cpp +++ b/core/tests/unit/kmnkbd/debug_api.cpp @@ -10,7 +10,9 @@ #include #include #include -#include + +#include "keyman_core.h" + #include "path.hpp" #include "state.hpp" #include "kmx/kmx_base.h" @@ -66,6 +68,7 @@ void setup(const char *keyboard) { })); try_status(km_core_context_set(km_core_state_context(test_state), citems)); + try_status(km_core_context_set(km_core_state_app_context(test_state), citems)); } /** @@ -423,6 +426,7 @@ void test_backspace_markers() { {KM_CORE_CT_END} }; try_status(km_core_context_set(km_core_state_context(test_state), marker_context)); + try_status(km_core_context_set(km_core_state_app_context(test_state), marker_context)); // TODO: this needs to remove markers DEBUG_GROUP gp = {u"Main"}; diff --git a/core/tests/unit/kmnkbd/debug_items.hpp b/core/tests/unit/kmnkbd/debug_items.hpp index 400f499bd2..ae5f0e8cbd 100644 --- a/core/tests/unit/kmnkbd/debug_items.hpp +++ b/core/tests/unit/kmnkbd/debug_items.hpp @@ -1,6 +1,5 @@ #include -#include -#include +#include "keyman_core.h" #include "kmx/kmx_base.h" #include "kmx/kmx_xstring.h" diff --git a/core/tests/unit/kmnkbd/keyboard_api.cpp b/core/tests/unit/kmnkbd/keyboard_api.cpp index 95365e901c..bf232d69af 100644 --- a/core/tests/unit/kmnkbd/keyboard_api.cpp +++ b/core/tests/unit/kmnkbd/keyboard_api.cpp @@ -6,7 +6,7 @@ */ #include -#include +#include "keyman_core.h" #include "path.hpp" //#include "keyboard.hpp" diff --git a/core/tests/unit/kmnkbd/meson.build b/core/tests/unit/kmnkbd/meson.build index 703374bdf6..16f0c87e8d 100644 --- a/core/tests/unit/kmnkbd/meson.build +++ b/core/tests/unit/kmnkbd/meson.build @@ -26,6 +26,7 @@ tests = [ ['debug-api', 'debug_api.cpp'], ['kmx_xstring', 'test_kmx_xstring.cpp'], ['kmx_context', 'test_kmx_context.cpp'], + ['test_actions_normalize', 'test_actions_normalize.cpp'], ] test_path = join_paths(meson.current_build_dir(), '..', 'kmx') diff --git a/core/tests/unit/kmnkbd/options_api.cpp b/core/tests/unit/kmnkbd/options_api.cpp index a831180f41..7c04478fe7 100644 --- a/core/tests/unit/kmnkbd/options_api.cpp +++ b/core/tests/unit/kmnkbd/options_api.cpp @@ -7,7 +7,7 @@ #include #include -#include +#include "keyman_core.h" #include "option.hpp" #include "state.hpp" diff --git a/core/tests/unit/kmnkbd/state_api.cpp b/core/tests/unit/kmnkbd/state_api.cpp index e3d1717589..07acd8bdd4 100644 --- a/core/tests/unit/kmnkbd/state_api.cpp +++ b/core/tests/unit/kmnkbd/state_api.cpp @@ -9,7 +9,7 @@ #include #include -#include +#include "keyman_core.h" #include "path.hpp" #include "state.hpp" diff --git a/core/tests/unit/kmnkbd/state_context_api.cpp b/core/tests/unit/kmnkbd/state_context_api.cpp index 3a71d05045..83275539f9 100644 --- a/core/tests/unit/kmnkbd/state_context_api.cpp +++ b/core/tests/unit/kmnkbd/state_context_api.cpp @@ -44,6 +44,7 @@ setup(const char *keyboard, const km_core_cp *context) { try_status(km_core_state_create(test_kb, test_env_opts, &test_state)); try_status(km_core_context_items_from_utf16(context, &citems)); try_status(km_core_context_set(km_core_state_context(test_state), citems)); + try_status(km_core_context_set(km_core_state_app_context(test_state), citems)); } bool @@ -138,6 +139,7 @@ test_context_set_if_needed_cached_context_has_markers() { {KM_CORE_CT_MARKER, {0}, {3}}, {KM_CORE_CT_MARKER, {0}, {4}}, KM_CORE_CONTEXT_ITEM_END}; try_status(km_core_context_set(km_core_state_context(test_state), citems)); + try_status(km_core_context_set(km_core_state_app_context(test_state), citems)); assert(km_core_state_context_set_if_needed(test_state, application_context) == KM_CORE_CONTEXT_STATUS_UNCHANGED); km_core_context_item *citems_new; @@ -177,6 +179,48 @@ test_context_clear() { teardown(); } +//------------------------------------------------------------------------------------- + +void test_context_debug_empty() { + km_core_cp const *cached_context = u""; + setup("k_000___null_keyboard.kmx", cached_context); + auto str = km_core_state_context_debug(test_state, KM_CORE_DEBUG_CONTEXT_CACHED); + // std::cout << str << std::endl; + assert(std::u16string(str) == u"|| (len: 0) [ ]"); + km_core_cp_dispose(str); +} + +void test_context_debug_various() { + km_core_cp const *cached_context = u""; + setup("k_000___null_keyboard.kmx", cached_context); + + km_core_context_item const citems[] = { + { KM_CORE_CT_MARKER, {0}, { 5 } }, + { KM_CORE_CT_CHAR, {0}, { '1' } }, + { KM_CORE_CT_MARKER, {0}, { 1 } }, + { KM_CORE_CT_CHAR, {0}, { '2' } }, + { KM_CORE_CT_MARKER, {0}, { 2 } }, + { KM_CORE_CT_CHAR, {0}, { '3' } }, + { KM_CORE_CT_MARKER, {0}, { 3 } }, + { KM_CORE_CT_MARKER, {0}, { 4 } }, + { KM_CORE_CT_CHAR, {0}, { 0x1F923 /* 🤣 */ } }, + KM_CORE_CONTEXT_ITEM_END + }; + + try_status(km_core_context_set(km_core_state_context(test_state), citems)); + try_status(km_core_context_set(km_core_state_app_context(test_state), citems)); + + auto str = km_core_state_context_debug(test_state, KM_CORE_DEBUG_CONTEXT_CACHED); + // std::cout << str << std::endl; + assert(std::u16string(str) == u"|123🤣| (len: 9) [ M(5) U+0031 M(1) U+0032 M(2) U+0033 M(3) M(4) U+1f923 ]"); + km_core_cp_dispose(str); +} + +void test_context_debug() { + test_context_debug_empty(); + test_context_debug_various(); +} + //------------------------------------------------------------------------------------- // Launcher //------------------------------------------------------------------------------------- @@ -216,4 +260,5 @@ main(int argc, char *argv[]) { test_context_set_if_needed(); test_context_clear(); + test_context_debug(); } diff --git a/core/tests/unit/kmnkbd/test_actions_normalize.cpp b/core/tests/unit/kmnkbd/test_actions_normalize.cpp new file mode 100644 index 0000000000..dbe298e08c --- /dev/null +++ b/core/tests/unit/kmnkbd/test_actions_normalize.cpp @@ -0,0 +1,333 @@ +/* + Copyright: © 2018 SIL International. + Description: Tests for the context API family of functions. + Create Date: 23 Oct 2023 + Authors: Marc Durdin + History: 23 Oct 2023 - MCD - Initial implementation. +*/ +#include +#include "keyman_core.h" + +#include "path.hpp" +#include "action.hpp" +#include "context.hpp" + +#include +#include "../emscripten_filesystem.h" + +km_core_option_item test_env_opts[] = +{ + KM_CORE_OPTIONS_END +}; + +km_core_keyboard * test_kb = nullptr; +km_core_state * test_state = nullptr; +km_core_actions * test_actions = nullptr; +std::string arg_path; + +void teardown() { + if(test_state) { + km_core_state_dispose(test_state); + test_state = nullptr; + } + if(test_kb) { + km_core_keyboard_dispose(test_kb); + test_kb = nullptr; + } + if(test_actions) { + delete [] test_actions->output; + delete test_actions; + test_actions = nullptr; + } +} + + + +void setup(const km_core_cp *app_context, const km_core_cp *cached_context, int actions_code_points_to_delete, const std::u32string actions_output) { + teardown(); + + km::core::path path = km::core::path::join(arg_path, "..", "ldml", "keyboards", "k_001_tiny.kmx"); + try_status(km_core_keyboard_load(path.native().c_str(), &test_kb)); + try_status(km_core_state_create(test_kb, test_env_opts, &test_state)); + + try_status(set_context_from_string(km_core_state_context(test_state), cached_context)); + try_status(set_context_from_string(km_core_state_app_context(test_state), app_context)); + + test_actions = new km_core_actions; + test_actions->code_points_to_delete = actions_code_points_to_delete; + test_actions->output = new km_core_usv[actions_output.length() + 1]; + actions_output.copy(test_actions->output, actions_output.length()); + test_actions->output[actions_output.length()] = 0; +} + +//------------------------------------------------------------------------------------- + +/** + * Run a single test on actions_normalize. This is quite nuanced, because the + * input state is more than a little complex. We have inputs in NFU and NFD, and + * outputs counting NFU and inserting NFC. Be careful! + * + * @param name descriptive name for the test + * @param initial_app_context the app context stored in the state, + * _before_ transform is applied -- NFU + * @param final_cached_context cached context _after_ actions have been + * applied -- guaranteed NFD (essentially, + * this is initial_cached_context - + * actions_code_points_to_delete + + * actions_output) + * @param actions_code_points_to_delete number of NFD code points that the + * keyboard processor has asked to remove + * in its actions + * @param actions_output NFD string that the keyboard processor + * has asked to insert in its actions + * @param expected_delete expected: NFU code points to ask app to + * remove + * @param expected_output expected: adjusted NFC output to insert + * into the app + * @param expected_final_app_context expected: NFU adjusted final app + * context, which will be NFC from the + * boundary of the transform, but will not + * have been modified prior to that. Should + * match char-for-char what the app ends up + * with in its text buffer. + */ +void test( + const char *name, + const km_core_cp *initial_app_context, + const km_core_cp *final_cached_context, + int actions_code_points_to_delete, + const std::u32string actions_output, + + const unsigned int expected_delete, + const std::u32string expected_output, + const km_core_cp *expected_final_app_context +) { + std::cout << "test: " << name << std::endl; + + setup(initial_app_context, final_cached_context, actions_code_points_to_delete, actions_output); + + assert(km::core::actions_normalize(km_core_state_context(test_state), km_core_state_app_context(test_state), test_actions)); + + std::cout << " (" << name << "): delete: " << expected_delete << " output: |" << std::u32string(test_actions->output) << "|" << std::endl; + std::u32string o(test_actions->output); + for(auto i = o.begin(); i < o.end(); i++) { + std::cout << "U+" << std::hex << (int)(*i) << " "; + } + std::cout << std::endl; + + assert(expected_delete == test_actions->code_points_to_delete); + assert(expected_output == test_actions->output); + + auto actual_final_app_context = get_context_as_string(km_core_state_app_context(test_state)); + auto actual_final_app_context_string = std::u16string(actual_final_app_context); + auto expected_final_app_context_string = std::u16string(expected_final_app_context); + std::cout << " final app context: actual: |" << actual_final_app_context_string << "| expected: |" << expected_final_app_context_string << "|" << std::endl; + assert(actual_final_app_context_string == expected_final_app_context_string); + delete [] actual_final_app_context; + + teardown(); +} + +void run_tests() { + + // Null boundary tests + + test( + "noop", + /* app context pre transform: */ u"", + /* cached context post transform: */ u"", + /* action del, output: */ 0, U"", + // ---- results ---- + /* action del, output: */ 0, U"", + /* app_context: */ u"" + ); + + test( + "no_output", + /* app context pre transform: */ u"abc", + /* cached context post transform: */ u"abc", + /* action del, output: */ 0, U"", + // ---- results ---- + /* action del, output: */ 0, U"", + /* app_context: */ u"abc" + ); + + test( + "no_context", + /* app context pre transform: */ u"", + /* cached context post transform: */ u"def", + /* action del, output: */ 0, U"def", + // ---- results ---- + /* action del, output: */ 0, U"def", + /* app_context: */ u"def" + ); + + // Simple tests -- no deletions involved + + test( + "no_normalization", + /* app context pre transform: */ u"abc", + /* cached context post transform: */ u"abcdef", + /* action del, output: */ 0, U"def", + // ---- results ---- + /* action del, output: */ 0, U"def", + /* app_context: */ u"abcdef" + ); + + test( + "output_to_nfc_basic", + /* app context pre transform: */ u"abc", + /* cached context post transform: */ u"abcde\u0300f", + /* action del, output: */ 0, U"de\u0300f", + // ---- results ---- + /* action del, output: */ 0, U"dèf", + /* app_context: */ u"abcdèf" + ); + + test( + "output_to_nfc_hefty", + /* app context pre transform: */ u"abc", + /* cached context post transform: */ u"abcA\u0300" u"e\u0316\u0301" u"\u0073\u0323\u0307" u"\u0041\u030a" u"\U000114B9\U000114B0", + /* action del, output: */ 0, U"A\u0300" U"e\u0316\u0301" U"\u0073\u0323\u0307" U"\u0041\u030a" U"\U000114B9\U000114B0", + // ---- results ---- + /* action del, output: */ 0, U"À" U"é̖" U"\u1e69" U"\u00c5" U"\U000114BC", + /* app_context: */ u"abcÀé̖\u1e69\u00c5\U000114BC" + ); + + // Interaction with input context when not on normalization boundary + + test( + "Backtrack one character to combine as NFC", + /* app context pre transform: */ u"XYZA", + /* cached context post transform: */ u"XYZA\u0300abc", + /* action del, output: */ 0, U"\u0300abc", + // ---- results ---- + /* action del, output: */ 1, U"Àabc", + /* app_context: */ u"XYZÀabc" + ); + + test( + "Backtrack e+comb circ (2 chars) to combine as NFC", + /* app context pre transform: */ u"abce\u0302", + /* cached context post transform: */ u"abce\u0323\u0302", + /* action del, output: */ 1, U"\u0323\u0302", + // ---- results ---- + /* action del, output: */ 2, U"ệ", + /* app_context: */ u"abcệ" + ); + + test( + "One backspace for NFD converts into one char in NFC (ê) and recombine", + /* app context pre transform: */ u"abcê", + /* cached context post transform: */ u"abce\u0323\u0302", + /* action del, output: */ 1, U"\u0323\u0302", // NFD input; delete 1: \u0302 + // ---- results ---- + /* action del, output: */ 1, U"ệ", // NFC output; delete 1: ê + /* app_context: */ u"abcệ" + ); + + // a\u0300 should not be normalized because it is not otherwise impacted by + // the action. + test( + "Avoid editing too far back in context when finding normalization boundary", + /* app context pre transform: */ u"a\u0300bcê", + /* cached context post transform: */ u"a\u0300bce\u0323\u0302", + /* action del, output: */ 1, U"\u0323\u0302", // NFD input; delete 1: \u0302 + // ---- results ---- + /* action del, output: */ 1, U"ệ", // NFC output; delete 1: ê + /* app_context: */ u"a\u0300bcệ" + ); + + // If we don't reach a normalization boundary, we still should continue to work + test( + "Normalizable letters at start of context", + /* app context pre transform: */ u"\u0300", + /* cached context post transform: */ u"\u0323\u0300\u0302", + /* action del, output: */ 1, U"\u0323\u0300\u0302", // NFD input; + // ---- results ---- + /* action del, output: */ 1, U"\u0323\u0300\u0302", // NFC output is still decomposed because there is no base + /* app_context: */ u"\u0323\u0300\u0302" + ); + + // Modifies the base as well as diacritic + + test( + "Two backspaces for NFD converts into one char in NFC (ê) and recombine", + /* app context pre transform: */ u"abcê", + /* cached context post transform: */ u"abca\u0323\u0302", + /* action del, output: */ 2, U"a\u0323\u0302", // NFD input; delete 2: e\u0302 + // ---- results ---- + /* action del, output: */ 1, U"ậ", // NFC output; delete 1: ê + /* app_context: */ u"abcậ" + ); + + // surrogate pair tests + + test( + "Surrogate pair in context", + /* app context pre transform: */ u"abc\U0001F607ê", + /* cached context post transform: */ u"abc\U0001F607a\u0323\u0302", + /* action del, output: */ 2, U"a\u0323\u0302", + // ---- results ---- + /* action del, output: */ 1, U"ậ", + /* app_context: */ u"abc\U0001F607ậ" + ); + + test( + "Surrogate pair in output", + /* app context pre transform: */ u"abc", + /* cached context post transform: */ u"abc\U0001F607", + /* action del, output: */ 0, U"\U0001F607", + // ---- results ---- + /* action del, output: */ 0, U"\U0001F607", + /* app_context: */ u"abc\U0001F607" + ); + + test( + "Surrogate pairs in both context and output", + /* app context pre transform: */ u"a\U0001F607bcê", + /* cached context post transform: */ u"a\U0001F607bca\U0001F60E", + /* action del, output: */ 2, U"a\U0001F60E", + // ---- results ---- + /* action del, output: */ 1, U"a\U0001F60E", + /* app_context: */ u"a\U0001F607bca\U0001F60E" + ); +} + +//------------------------------------------------------------------------------------- +// Launcher +//------------------------------------------------------------------------------------- + +constexpr const auto help_str = "\ +test_actions_normalize [--color] \n\ +\n\ + --color Force color output\n\ + BUILD_PATH Path where test_actions_normalize.exe is found; kmx files are\n\ + located relative to this path.\n"; + +int error_args() { + std::cerr << "test_actions_normalize: Invalid arguments." << std::endl; + std::cout << help_str; + return 1; +} + +int main(int argc, char *argv []) { + + if(argc < 2) { + return error_args(); + } + + auto arg_color = std::string(argv[1]) == "--color"; + if(arg_color && argc < 3) { + return error_args(); + } + console_color::enabled = console_color::isaterminal() || arg_color; + +#ifdef __EMSCRIPTEN__ + arg_path = get_wasm_file_path(argv[arg_color ? 2 : 1]); +#else + arg_path = argv[arg_color ? 2 : 1]; +#endif + + run_tests(); +} diff --git a/core/tests/unit/kmx/kmx.cpp b/core/tests/unit/kmx/kmx.cpp index 794c744a7c..1e94f4cf86 100644 --- a/core/tests/unit/kmx/kmx.cpp +++ b/core/tests/unit/kmx/kmx.cpp @@ -17,6 +17,8 @@ #include #include +#include "keyman_core.h" + #include #include diff --git a/core/tests/unit/ldml/ldml.cpp b/core/tests/unit/ldml/ldml.cpp index 453ef8ac8e..db95141eba 100644 --- a/core/tests/unit/ldml/ldml.cpp +++ b/core/tests/unit/ldml/ldml.cpp @@ -24,6 +24,8 @@ #include #include +#include "keyman_core.h" + #include // for surrogate pair macros #include "ldml_test_source.hpp" diff --git a/core/tests/unit/ldml/ldml_test_source.cpp b/core/tests/unit/ldml/ldml_test_source.cpp index 4f59468dcd..0b481cd68a 100644 --- a/core/tests/unit/ldml/ldml_test_source.cpp +++ b/core/tests/unit/ldml/ldml_test_source.cpp @@ -33,15 +33,9 @@ #include "ldml_test_source.hpp" #include "ldml_test_utils.hpp" -#if defined(HAVE_ICU4C) -// TODO-LDML: Needed this for some compiler warnings -#define U_FALLTHROUGH -#include "unicode/utypes.h" +#include "core_icu.h" #include "unicode/uniset.h" #include "unicode/usetiter.h" -#else -#error icu4c is required for this test -#endif #define assert_or_return(expr) if(!(expr)) { \ std::wcerr << __FILE__ << ":" << __LINE__ << ": " << \ diff --git a/core/tests/unit/ldml/ldml_test_utils.hpp b/core/tests/unit/ldml/ldml_test_utils.hpp index 1d876365bb..ec180200ed 100644 --- a/core/tests/unit/ldml/ldml_test_utils.hpp +++ b/core/tests/unit/ldml/ldml_test_utils.hpp @@ -13,7 +13,8 @@ #include // for char to vk mapping tables #include // for surrogate pair macros #include -#include + +#include "keyman_core.h" namespace km { namespace tests { diff --git a/core/tests/unit/ldml/meson.build b/core/tests/unit/ldml/meson.build index f85014bebe..9e65ff343d 100644 --- a/core/tests/unit/ldml/meson.build +++ b/core/tests/unit/ldml/meson.build @@ -83,6 +83,23 @@ t = executable('test_transforms', 'test_transforms.cpp', objects: lib.extract_all_objects(recursive: false)) test('test_transforms', t, suite: 'ldml') +# run test_context_normalization ldml unit test + +normalization_tests_flags = tests_flags + +if cpp_compiler.get_id() == 'emscripten' + normalization_tests_flags += ['-lnodefs.js', '-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\']'] +endif + +t = executable('test_context_normalization', + ['test_context_normalization.cpp', '../emscripten_filesystem.cpp'], + cpp_args: defns + warns, + include_directories: [inc, libsrc, '../../../../developer/src/ext/json'], + link_args: links + normalization_tests_flags, + dependencies: [icu_uc, icu_i18n], + objects: lib.extract_all_objects(recursive: false)) +test('test_context_normalization', t, suite: 'ldml') + # Run tests on all keyboards (`tests` defined in keyboards/meson.build) foreach kbd : tests diff --git a/core/tests/unit/ldml/test_context_normalization.cpp b/core/tests/unit/ldml/test_context_normalization.cpp new file mode 100644 index 0000000000..11a3b8ca0e --- /dev/null +++ b/core/tests/unit/ldml/test_context_normalization.cpp @@ -0,0 +1,162 @@ +/* + Copyright: © 2018 SIL International. + Description: Tests for normalization in the context API. + Create Date: 15 Jan 2024 + Authors: Marc Durdin + History: 15 Jan 2024 - MCD - Initial implementation. +*/ +#include +#include "keyman_core.h" + +#include "path.hpp" +#include "action.hpp" + +#include +#include "../emscripten_filesystem.h" + +//------------------------------------------------------------------------------------- +// Context normalization tests +//------------------------------------------------------------------------------------- + +km_core_option_item test_env_opts[] = +{ + KM_CORE_OPTIONS_END +}; + +km_core_keyboard * test_kb = nullptr; +km_core_state * test_state = nullptr; +std::string arg_path; + +void teardown() { + if(test_state) { + km_core_state_dispose(test_state); + test_state = nullptr; + } + if(test_kb) { + km_core_keyboard_dispose(test_kb); + test_kb = nullptr; + } +} + +void setup(const char *keyboard) { + teardown(); + + km::core::path path = km::core::path::join(arg_path, "keyboards", keyboard); + try_status(km_core_keyboard_load(path.native().c_str(), &test_kb)); + try_status(km_core_state_create(test_kb, test_env_opts, &test_state)); +} + +void debug_context(km_core_debug_context_type context_type) { + auto context = km_core_state_context_debug(test_state, context_type); + if(context_type == KM_CORE_DEBUG_CONTEXT_APP) { + std::cout << "app context: " << context << std::endl; + } else { + std::cout << "cached context: " << context << std::endl; + } + km_core_cp_dispose(context); +} + +bool is_identical_context(km_core_cp const *cached_context, km_core_debug_context_type context_type) { + size_t buf_size; + km_core_context_item * citems = nullptr; + + debug_context(context_type); + + if(context_type == KM_CORE_DEBUG_CONTEXT_APP) { + try_status(km_core_context_get(km_core_state_app_context(test_state), &citems)); + } else { + try_status(km_core_context_get(km_core_state_context(test_state), &citems)); + } + try_status(km_core_context_items_to_utf16(citems, nullptr, &buf_size)); + km_core_cp* new_cached_context = new km_core_cp[buf_size]; + try_status(km_core_context_items_to_utf16(citems, new_cached_context, &buf_size)); + + km_core_context_items_dispose(citems); + + bool result = std::u16string(cached_context) == new_cached_context; + delete[] new_cached_context; + return result; +} + +void test_context_normalization_already_nfd() { + km_core_cp const *app_context_nfd = u"A\u0300"; + setup("k_001_tiny.kmx"); + assert(km_core_state_context_set_if_needed(test_state, app_context_nfd) == KM_CORE_CONTEXT_STATUS_UPDATED); + assert(is_identical_context(app_context_nfd, KM_CORE_DEBUG_CONTEXT_APP)); + assert(is_identical_context(app_context_nfd, KM_CORE_DEBUG_CONTEXT_CACHED)); + teardown(); +} + +void test_context_normalization_basic() { + km_core_cp const *application_context = u"This is a test À"; + km_core_cp const *cached_context = u"This is a test A\u0300"; + setup("k_001_tiny.kmx"); + assert(km_core_state_context_set_if_needed(test_state, application_context) == KM_CORE_CONTEXT_STATUS_UPDATED); + assert(is_identical_context(application_context, KM_CORE_DEBUG_CONTEXT_APP)); + assert(is_identical_context(cached_context, KM_CORE_DEBUG_CONTEXT_CACHED)); + teardown(); +} + +void test_context_normalization_hefty() { + // Latin Latin "ṩ" "Å" Tirhuta U+114bc -> U+114B9 U+114B0 + km_core_cp const *application_context = u"À" u"é̖" u"\u1e69" u"\u212b" u"\U000114BC"; + km_core_cp const *cached_context = u"A\u0300" u"e\u0316\u0301" u"\u0073\u0323\u0307" u"\u0041\u030a" u"\U000114B9\U000114B0"; + setup("k_001_tiny.kmx"); + assert(km_core_state_context_set_if_needed(test_state, application_context) == KM_CORE_CONTEXT_STATUS_UPDATED); + assert(is_identical_context(application_context, KM_CORE_DEBUG_CONTEXT_APP)); + assert(is_identical_context(cached_context, KM_CORE_DEBUG_CONTEXT_CACHED)); + teardown(); +} + +void test_context_normalization_invalid_unicode() { + // unpaired surrogate illegal + km_core_cp const application_context[] = { 0xDC01, 0x0020, 0x0020, 0xFFFF, 0x0000 }; + km_core_cp const cached_context[] = { 0xDC01, 0x0020, 0x0020, 0xFFFF, 0x0000 }; + setup("k_001_tiny.kmx"); + assert(km_core_state_context_set_if_needed(test_state, application_context) == KM_CORE_CONTEXT_STATUS_UPDATED); + assert(is_identical_context(application_context, KM_CORE_DEBUG_CONTEXT_APP)); + assert(is_identical_context(cached_context, KM_CORE_DEBUG_CONTEXT_CACHED)); + teardown(); +} + +void test_context_normalization() { + test_context_normalization_already_nfd(); + test_context_normalization_basic(); + test_context_normalization_hefty(); + // TODO: we need to strip illegal chars: test_context_normalization_invalid_unicode(); // -- unpaired surrogate, illegals +} + +//------------------------------------------------------------------------------------- +// Launcher +//------------------------------------------------------------------------------------- + +constexpr const auto help_str = "\ +context_normalization [--color]\n\ +\n\ + --color Force color output\n"; + +int error_args() { + std::cerr << "context_normalization: Invalid arguments." << std::endl; + std::cout << help_str; + return 1; +} + +int main(int argc, char *argv []) { + auto arg_color = argc > 1 && std::string(argv[1]) == "--color"; + console_color::enabled = console_color::isaterminal() || arg_color; + + // Get the path of the current executable + arg_path = argv[0]; + auto last = arg_path.find_last_of("/\\"); + if(last == std::string::npos) { + std::cerr << "could not parse argv[0]: " << argv[0] << std::endl; + return 1; + } + arg_path.resize(last+1); + +#ifdef __EMSCRIPTEN__ + arg_path = get_wasm_file_path(arg_path); +#endif + + test_context_normalization(); +} diff --git a/linux/debian/libkeymancore.symbols b/linux/debian/libkeymancore.symbols index 1be1069251..7e4522850d 100644 --- a/linux/debian/libkeymancore.symbols +++ b/linux/debian/libkeymancore.symbols @@ -16,6 +16,7 @@ libkeymancore.so.1 libkeymancore #MINVER# km_core_context_length@Base 17.0.195 km_core_context_set@Base 17.0.195 km_core_context_shrink@Base 17.0.195 + km_core_cp_dispose@Base 17.0.244 km_core_event@Base 17.0.195 km_core_get_engine_attrs@Base 17.0.195 km_core_keyboard_dispose@Base 17.0.195 @@ -29,9 +30,11 @@ libkeymancore.so.1 libkeymancore #MINVER# km_core_process_event@Base 17.0.195 km_core_process_queued_actions@Base 17.0.195 km_core_state_action_items@Base 17.0.195 + km_core_state_app_context@Base 17.0.245 km_core_state_clone@Base 17.0.195 km_core_state_context@Base 17.0.195 km_core_state_context_clear@Base 17.0.197 + km_core_state_context_debug@Base 17.0.244 km_core_state_context_set_if_needed@Base 17.0.197 km_core_state_create@Base 17.0.195 km_core_state_debug_get@Base 17.0.195 diff --git a/linux/ibus-keyman/src/engine.c b/linux/ibus-keyman/src/engine.c index 3666e9c119..e12ed49813 100644 --- a/linux/ibus-keyman/src/engine.c +++ b/linux/ibus-keyman/src/engine.c @@ -205,27 +205,17 @@ ibus_keyman_engine_class_init (IBusKeymanEngineClass *klass) } static gchar * -get_context_debug(IBusEngine *engine, km_core_context *context) { +get_context_debug(IBusEngine *engine) { IBusKeymanEngine *keyman = (IBusKeymanEngine *)engine; - km_core_context *state_context = context ? context : km_core_state_context(keyman->state); - size_t buf_size8 = 512, buf_size16 = 512; - km_core_context_item *context_items; - gchar *current_context_utf8 = g_new0(gchar, buf_size8); - km_core_cp *current_context_utf16 = g_new0(km_core_cp, buf_size16); - if (km_core_context_get(state_context, &context_items) == KM_CORE_STATUS_OK) { - km_core_context_items_to_utf8(context_items, current_context_utf8, &buf_size8); - km_core_context_items_to_utf16(context_items, current_context_utf16, &buf_size16); + km_core_cp *buf = km_core_state_context_debug(keyman->state, KM_CORE_DEBUG_CONTEXT_CACHED); + gchar *result = g_utf16_to_utf8((gunichar2 *)buf, -1, NULL, NULL, NULL); + km_core_cp_dispose(buf); + if(result) { + return result; } - GString *output = g_string_new(""); - g_string_append_printf(output, "|%s| (len:%zu) [", current_context_utf8, km_core_context_length(state_context)); - for (int i = 0; i < buf_size16 - 1; i++) { - g_string_append_printf(output, "U+%04x ", current_context_utf16[i]); - } - g_string_append(output, "]"); - km_core_context_items_dispose(context_items); - g_free(current_context_utf16); - g_free(current_context_utf8); + + GString *output = g_string_new("Error converting debug context"); #if GLIB_CHECK_VERSION(2, 76, 0) return g_string_free_and_steal(output); #else @@ -264,7 +254,7 @@ set_context_if_needed(IBusEngine *engine) { guint cursor_pos, anchor_pos, context_start, context_end; g_autofree gchar *debug_context = NULL; - g_message("%s: current core context : %s", __FUNCTION__, debug_context = get_context_debug(engine, NULL)); + g_message("%s: current core context : %s", __FUNCTION__, debug_context = get_context_debug(engine)); ibus_engine_get_surrounding_text(engine, &text, &cursor_pos, &anchor_pos); @@ -934,10 +924,10 @@ ibus_keyman_engine_process_key_event( km_mod_state |= KM_CORE_MODIFIER_CAPS; } g_message("DAR: %s - km_mod_state=0x%x", __FUNCTION__, km_mod_state); - g_autofree gchar *debug_context = NULL; - g_message("%s: before process key event: %s", __FUNCTION__, debug_context = get_context_debug(engine, NULL)); + g_autofree gchar *debug_context0 = NULL, *debug_context1 = NULL, *debug_context2 = NULL; + g_message("%s: before process key event: %s", __FUNCTION__, debug_context0 = get_context_debug(engine)); km_core_process_event(keyman->state, keycode_to_vk[keycode], km_mod_state, isKeyDown, KM_CORE_EVENT_FLAG_DEFAULT); - g_message("%s: after process key event : %s", __FUNCTION__, debug_context = get_context_debug(engine, NULL)); + g_message("%s: after process key event : %s", __FUNCTION__, debug_context1 = get_context_debug(engine)); // km_core_state_action_items to get action items size_t num_action_items; @@ -953,7 +943,7 @@ ibus_keyman_engine_process_key_event( return FALSE; } - g_message("%s: after processing all actions: %s", __FUNCTION__, debug_context = get_context_debug(engine, NULL)); + g_message("%s: after processing all actions: %s", __FUNCTION__, debug_context2 = get_context_debug(engine)); return TRUE; } diff --git a/mac/Keyman4MacIM/KeymanTests/TestAppDelegate.m b/mac/Keyman4MacIM/KeymanTests/TestAppDelegate.m index a17e1f78da..e646e33905 100644 --- a/mac/Keyman4MacIM/KeymanTests/TestAppDelegate.m +++ b/mac/Keyman4MacIM/KeymanTests/TestAppDelegate.m @@ -18,7 +18,7 @@ if (_kme == nil) { _kme = [[KMEngine alloc] initWithKMX:nil context:self.contextBuffer verboseLogging:self.debugMode]; } - + return _kme; } @@ -31,7 +31,7 @@ if (_contextBuffer == nil) { _contextBuffer = [[NSMutableString alloc] initWithString:@""]; } - + return _contextBuffer; } @@ -39,7 +39,7 @@ _contextBuffer = [contextBuffer mutableCopy]; if (_contextBuffer.length) [_contextBuffer replaceOccurrencesOfString:@"\0" withString:[NSString nullChar] options:0 range:NSMakeRange(0, 1)]; - [self.kme setCoreContext:self.contextBuffer]; + [self.kme setCoreContextIfNeeded:self.contextBuffer]; } - (BOOL)debugMode { diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.h b/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.h index a3111d425e..1109b1130f 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.h +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.h @@ -1,9 +1,9 @@ /** * Keyman is copyright (C) SIL International. MIT License. - * + * * CoreWrapper.h * Keyman - * + * * Created by Shawn Schantz on 2022-12-12. */ @@ -28,8 +28,7 @@ NS_ASSUME_NONNULL_BEGIN withModifiers:(NSEventModifierFlags)modifierState withKeyDown:(BOOL)isKeyDown; -(void)setContextIfNeeded:(NSString*)context; --(void)setContext:(NSString*)context; --(NSString*)context; +-(NSString*)contextDebug; -(void)clearCoreContext; -(void)dealloc; diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m index 59c791aabf..ac011ce35e 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m @@ -225,124 +225,6 @@ const int CORE_ENVIRONMENT_ARRAY_LENGTH = 6; return capsLock; } --(NSArray*)loadActionsUsingCore { - size_t actionCount = 0; - km_core_action_item const * actionList = - km_core_state_action_items(self.coreState, &actionCount); - - NSMutableArray *eventArray = [NSMutableArray arrayWithCapacity:actionCount]; - - for (int i = 0; i < actionCount; i++) { - km_core_action_item action = actionList[i]; - CoreAction *coreAction = [self createCoreActionForActionStruct:&action]; - [eventArray insertObject:coreAction atIndex:i]; - } - - return eventArray; -} - --(CoreAction*)createCoreActionForActionStruct:(km_core_action_item*)actionStruct { - CoreAction* action = nil; - switch (actionStruct->type) - { - case KM_CORE_IT_END: { - action = [[CoreAction alloc] initWithType: EndAction actionContent:@"" backspaceCount:0 key:@"" value:@"" scope:0]; - break; - } - case KM_CORE_IT_CHAR: { - NSString *characterString = [self.coreHelper utf32ValueToString:actionStruct->character]; - action = [[CoreAction alloc] initWithType: CharacterAction actionContent:characterString backspaceCount:0 key:@"" value:@"" scope:0]; - [self.coreHelper logDebugMessage:@"createCoreActionForActionStruct actionStruct->character decimal: %u, hex: %X", actionStruct->character, actionStruct->character]; - [self.coreHelper logDebugMessage:@"createCoreActionForActionStruct converted unicode string: '%@' length=%lu", characterString, characterString.length]; - break; - } - case KM_CORE_IT_MARKER: { - action = [[CoreAction alloc] initWithType: MarkerAction actionContent:@"" backspaceCount:0 key:@"" value:@"" scope:0]; - break; - } - case KM_CORE_IT_ALERT: { - action = [[CoreAction alloc] initWithType: AlertAction actionContent:@"" backspaceCount:0 key:@"" value:@"" scope:0]; - break; - } - case KM_CORE_IT_BACK: { - km_core_backspace_item backspace = actionStruct->backspace; - - if (backspace.expected_type == KM_CORE_BT_CHAR) { - NSString *charString = [self.coreHelper utf32ValueToString:backspace.expected_value]; - [self.coreHelper logDebugMessage:@"createCoreActionForActionStruct charString = %@", charString]; - action = [[CoreAction alloc] initCharacterBackspaceAction:charString]; - [self.coreHelper logDebugMessage:@"createCoreActionForActionStruct converted character backspace, expected value =%lu, expected type =%u", backspace.expected_value, backspace.expected_type]; - } else if(backspace.expected_type == KM_CORE_BT_MARKER) { - action = [[CoreAction alloc] initMarkerBackspaceAction:actionStruct->backspace.expected_value]; - [self.coreHelper logDebugMessage:@"createCoreActionForActionStruct converted marker backspace, expected value =%lu, expected type =%u", backspace.expected_value, backspace.expected_type]; - } else { - [self.coreHelper logDebugMessage:@"createCoreActionForActionStruct did not convert unknown backspace, expected value =%lu, expected type =%u", backspace.expected_value, backspace.expected_type]; - } - break; - } - case KM_CORE_IT_PERSIST_OPT: { - [self.coreHelper logDebugMessage:@"***createCoreActionForActionStruct Persist Options encountered."]; - km_core_option_item const * option = actionStruct->option; - NSString *keyString = [self.coreHelper createNSStringFromUnicharString:option->key]; - NSString *valueString = [self.coreHelper createNSStringFromUnicharString:option->value]; - - [self.coreHelper logDebugMessage:@"***createCoreActionForActionStruct converted Persist Options, key = %@, value = %@, scope = %d", keyString, valueString, option->scope]; - - action = [[CoreAction alloc] initPersistOptionAction:keyString value:valueString scope:option->scope]; - break; - } - case KM_CORE_IT_EMIT_KEYSTROKE: { - action = [[CoreAction alloc] initWithType: EmitKeystrokeAction actionContent:@"" backspaceCount:0 key:@"" value:@"" scope:0]; - break; - } - case KM_CORE_IT_INVALIDATE_CONTEXT: { - action = [[CoreAction alloc] initWithType: InvalidateContextAction actionContent:@"" backspaceCount:0 key:@"" value:@"" scope:0]; - break; - } - case KM_CORE_IT_CAPSLOCK: { - action = [[CoreAction alloc] initWithType: CapsLockAction actionContent:@"" backspaceCount:0 key:@"" value:@"" scope:0]; - break; - } - default: { - NSLog(@"createCoreActionForActionStruct unrecognized type of km_core_action_item = %u\n", actionStruct->type); - } - } - return action; -} - --(NSString *)getContextAsStringUsingCore { - km_core_context * context = km_core_state_context(self.coreState); - - km_core_context_item * contextItemsArray = nil; - size_t contextLength = km_core_context_length(context); - - NSMutableString *contextString = [[NSMutableString alloc]init]; - - if (contextLength==0) { - [self.coreHelper logDebugMessage:@"CoreWrapper getContextAsStringUsingCore, context is empty."]; - } else { - km_core_status result = km_core_context_get(context, &contextItemsArray); - if (result==KM_CORE_STATUS_OK) { - for (int i = 0; i < contextLength; i++) { - km_core_context_item contextItem = contextItemsArray[i]; - if (contextItem.type == KM_CORE_CT_CHAR) { - NSString *unicodeString = [self.coreHelper utf32ValueToString:contextItem.character]; - [contextString appendString:unicodeString]; - } - } - } - } - NSString *immutableString = [NSString stringWithString:contextString]; - - // dispose of context items array - if (contextItemsArray) { - km_core_context_items_dispose(contextItemsArray); - } - - [self.coreHelper logDebugMessage:@"CoreWrapper getContextAsStringUsingCore = %@", immutableString]; - return immutableString; -} - -(void)clearContextUsingCore { km_core_state_context_clear(self.coreState); [self.coreHelper logDebugMessage:@"km_core_state_context_clear called"]; @@ -354,28 +236,13 @@ const int CORE_ENVIRONMENT_ARRAY_LENGTH = 6; [self.coreHelper logDebugMessage:@"CoreWrapper setContextIfNeeded, context=%@, km_core_state_context_set_if_needed result=%i", context, result]; } --(void)setContext:(NSString*)context { - if (context.length == 0) { - [self clearContextUsingCore]; - } else { - char const *coreString = [context cStringUsingEncoding:NSUTF8StringEncoding]; - km_core_context_item *contextItemArray; +-(NSString*)contextDebug { + km_core_cp * context = km_core_state_context_debug(self.coreState, KM_CORE_DEBUG_CONTEXT_CACHED); + NSString *debugString = [self.coreHelper createNSStringFromUnicharString:context]; + km_core_cp_dispose(context); - // create array of context items - km_core_status result = km_core_context_items_from_utf8(coreString, &contextItemArray); - [self.coreHelper logDebugMessage:@"km_core_context_items_from_utf8, result=%i", result]; - - // set the context in core using the array - km_core_context * coreContext = km_core_state_context(self.coreState); - km_core_context_set(coreContext, contextItemArray); - // dispose - km_core_context_items_dispose(contextItemArray); - } -} - - --(NSString*)context { - return [self getContextAsStringUsingCore]; + [self.coreHelper logDebugMessage:@"CoreWrapper contextDebug = %@", debugString]; + return debugString; } //TODO: create and save as static diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.h b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.h index 4ec0c595cb..c79b02ba32 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.h +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.h @@ -21,10 +21,9 @@ @property (assign, nonatomic) BOOL debugMode; - (id)initWithKMX:(KMXFile *)kmx context:(NSString *)ctxBuf verboseLogging:(BOOL)enableDebugLogging; -- (NSString *)getCoreContext; +- (NSString *)getCoreContextDebug; - (void)clearCoreContext; - (void)setCoreContextIfNeeded:(NSString *)context; -- (void)setCoreContext:(NSString *)context; - (void)setCoreOptions:(NSString *)key withValue:(NSString *)value; - (CoreKeyOutput *)processEvent:(NSEvent *)event; diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m index 16c1b0c704..ea59fb950d 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m @@ -36,10 +36,10 @@ const NSString* kEasterEggKmxName = @"EnglishSpanish.kmx"; self.debugMode = enableDebugLogging; _kmx = kmx; _coreHelper = [[CoreHelper alloc] initWithDebugMode:enableDebugLogging]; - + if (kmx) { [self loadCoreWrapperFromKmxFile:self.kmx.filePath]; - [self.coreWrapper setContext:contextString]; + [self.coreWrapper setContextIfNeeded:contextString]; } } @@ -71,7 +71,7 @@ const NSString* kEasterEggKmxName = @"EnglishSpanish.kmx"; if (self.coreHelper) { self.coreHelper.debugMode = useVerboseLogging; } - + if (useVerboseLogging) { NSLog(@"KMEngine - Turning verbose logging on"); // In Keyman Engine if "debugMode" is turned on (explicitly) with "English plus Spanish" as the current keyboard and you type "Sentrycrash#KME", @@ -89,8 +89,8 @@ const NSString* kEasterEggKmxName = @"EnglishSpanish.kmx"; NSLog(@"KMEngine - Turning verbose logging off"); } -- (NSString *)getCoreContext { - return self.coreWrapper.context; +- (NSString *)getCoreContextDebug { + return self.coreWrapper.contextDebug; } - (void)clearCoreContext { @@ -101,10 +101,6 @@ const NSString* kEasterEggKmxName = @"EnglishSpanish.kmx"; [self.coreWrapper setContextIfNeeded:context]; } -- (void)setCoreContext:(NSString *)context { - [self.coreWrapper setContext:context]; -} - - (void)setCoreOptions:(NSString *)key withValue:(NSString *)value { BOOL success = [self.coreWrapper setOptionsForCore:key value:value]; [self.coreHelper logDebugMessage:@"setCoreOptions for key: %@, value: %@ succeeded = %@", key, value, success ? @"YES" : @"NO"]; diff --git a/mac/KeymanEngine4Mac/KeymanEngine4MacTests/CoreWrapperTests.m b/mac/KeymanEngine4Mac/KeymanEngine4MacTests/CoreWrapperTests.m index d3e35bf464..2748695a81 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4MacTests/CoreWrapperTests.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4MacTests/CoreWrapperTests.m @@ -1,11 +1,11 @@ /** * Keyman is copyright (C) SIL International. MIT License. - * + * * CoreWrapperTest.m * CoreWrapperTests - * + * * Created by Shawn Schantz on 2023-02-17. - * + * * Description... */ @@ -29,7 +29,7 @@ CoreWrapper *mockWrapper; NSString *khmerKeyboardPath = [[[NSBundle mainBundle] resourcePath] stringByAppendingPathComponent:@"khmer_angkor.kmx"]; NSLog(@"mockKmxFilePath = %@\n", mockKmxFilePath); - + mockWrapper = [[CoreWrapper alloc] initWithHelper: [CoreTestStaticHelperMethods helper] kmxFilePath:mockKmxFilePath]; } @@ -57,7 +57,7 @@ CoreWrapper *mockWrapper; - (void)testprocessEvent_lowercaseA_returnsExpectedCharacterForKmx { NSString *kmxPath = [CoreTestStaticHelperMethods getKmxFilePathTestMacEngine]; CoreWrapper *core = [[CoreWrapper alloc] initWithHelper: [CoreTestStaticHelperMethods helper] kmxFilePath:kmxPath]; - + // expecting character with 'Ç' CoreKeyOutput *coreOutput = [core processMacVirtualKey:MVK_A withModifiers:0 withKeyDown:YES]; XCTAssert([coreOutput.textToInsert isEqualToString:@"\u00C7"], @"Expected capital C cedille (U+00C7)"); @@ -66,9 +66,11 @@ CoreWrapper *mockWrapper; - (void)testgetContextAsString_ContextContainsEmojis_ReturnsSameContext { NSString *kmxPath = [CoreTestStaticHelperMethods getKmxFilePathTestMacEngine]; CoreWrapper *core = [[CoreWrapper alloc] initWithHelper: [CoreTestStaticHelperMethods helper] kmxFilePath:kmxPath]; - [core setContext:@"🤔?👍🏻✅"]; - NSString *finalContext = core.context; - XCTAssert([finalContext isEqualToString:@"🤔?👍🏻✅"], @"Expected '🤔?👍🏻✅' in context buffer"); + [core setContextIfNeeded:@"🤔?👍🏻✅"]; + NSString *finalContext = core.contextDebug; + // Note: relying on km_core_state_context_debug output format is just barely + // acceptable for a unit test + XCTAssert([finalContext isEqualToString:@"|🤔?👍🏻✅| (len: 5) [ U+1f914 U+003f U+1f44d U+1f3fb U+2705 ]"], @"Expected '🤔?👍🏻✅' in context buffer"); } @end diff --git a/mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMEngineTests.m b/mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMEngineTests.m index 9ea38e0a10..7bb1ed97fa 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMEngineTests.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMEngineTests.m @@ -41,21 +41,27 @@ NSString * names[nCombinations]; KMXFile *kmxFile = [KeymanEngineTestsStaticHelperMethods getKmxFileTestMacEngine]; KMEngine *engine = [[KMEngine alloc] initWithKMX:kmxFile context:@"" verboseLogging:YES]; XCTAssert(engine != nil, @"Expected non-nil engine"); - XCTAssert(engine.getCoreContext.length == 0, @"Expected empty context buffer"); + // Note: relying on km_core_state_context_debug output format is just barely + // acceptable for a unit test + XCTAssert([engine.getCoreContextDebug isEqualToString:@"|| (len: 0) [ ]"], @"Expected empty context buffer"); } - (void)testinitWithKMX_ValidKmxNonEmptyContext_InitializedWithContext { KMXFile *kmxFile = [KeymanEngineTestsStaticHelperMethods getKmxFileTestMacEngine]; KMEngine *engine = [[KMEngine alloc] initWithKMX:kmxFile context:@"abc" verboseLogging:YES]; XCTAssert(engine != nil, @"Expected non-nil engine"); - XCTAssert([engine.getCoreContext isEqualToString:@"abc"], @"Expected 'abc' in context buffer"); + // Note: relying on km_core_state_context_debug output format is just barely + // acceptable for a unit test + XCTAssert([engine.getCoreContextDebug isEqualToString:@"|abc| (len: 3) [ U+0061 U+0062 U+0063 ]"], @"Expected 'abc' in context buffer"); } - (void)testsetCoreContextIfNeeded_NonEmptyContext_InitialContextUpdated { KMXFile *kmxFile = [KeymanEngineTestsStaticHelperMethods getKmxFileTestMacEngine]; KMEngine *engine = [[KMEngine alloc] initWithKMX:kmxFile context:@"a" verboseLogging:YES]; [engine setCoreContextIfNeeded:@"xyz"]; - XCTAssert([engine.getCoreContext isEqualToString:@"xyz"], @"Expected 'xyz' in context buffer"); + // Note: relying on km_core_state_context_debug output format is just barely + // acceptable for a unit test + XCTAssert([engine.getCoreContextDebug isEqualToString:@"|xyz| (len: 3) [ U+0078 U+0079 U+007a ]"], @"Expected 'xyz' in context buffer"); } // TODO: re-enable this one after the core API km_core_state_context_set_if_needed is fixed @@ -64,7 +70,9 @@ NSString * names[nCombinations]; KMXFile *kmxFile = [KeymanEngineTestsStaticHelperMethods getKmxFileTestMacEngine]; KMEngine *engine = [[KMEngine alloc] initWithKMX:kmxFile context:@"" verboseLogging:YES]; [engine setCoreContextIfNeeded:@"xyz"]; - XCTAssert([engine.getCoreContext isEqualToString:@"xyz"], @"Expected 'xyz' in context buffer"); + // Note: relying on km_core_state_context_debug output format is just barely + // acceptable for a unit test + XCTAssert([engine.getCoreContextDebug isEqualToString:@"|xyz| (len: 3) [ U+0078 U+0079 U+007a ]"], @"Expected 'xyz' in context buffer"); } */ @@ -278,7 +286,7 @@ NSString * names[nCombinations]; NSString * characters = charactersIgnoringModifiers; if (modifiers[i] & (LEFT_ALT_FLAG | RIGHT_ALT_FLAG)) characters = [characters stringByAppendingString:@"\u030A"]; - + NSLog(@"Test case: %lu", (NSUInteger)modifiers[i]); // NOTE: 'a' happens to be keyCode 0 (see initVirtualKeyMapping in CoreHelper) NSEvent *event = [NSEvent keyEventWithType:NSEventTypeKeyDown location:NSMakePoint(0, 0) modifierFlags:modifiers[i] timestamp:0 windowNumber:0 context:nil characters:characters charactersIgnoringModifiers:charactersIgnoringModifiers isARepeat:NO keyCode:0]; @@ -335,10 +343,10 @@ NSString * names[nCombinations]; - (void)testprocessEvent_eventForSWithModifiers_ReturnsCharacterActionWithExpectedCharacterBasedOnKmx { int i = 0; [KMEngineTests fillInNamesAndModifiersForAllChiralCombinations]; - + KMXFile *kmxFile = [KeymanEngineTestsStaticHelperMethods getKmxFileTestMacEngine]; KMEngine *engine = [[KMEngine alloc] initWithKMX:kmxFile context:@"" verboseLogging:YES]; - + for (i = 0; i < nCombinations; i++) { [engine clearCoreContext]; NSString *charactersIgnoringModifiers = (modifiers[i] & (LEFT_SHIFT_FLAG | RIGHT_SHIFT_FLAG)) ? @"S" : @"s"; @@ -349,7 +357,7 @@ NSString * names[nCombinations]; else characters = @"ß"; } - + NSLog(@"Test case: %lu", (NSUInteger)modifiers[i]); // NOTE: 's' happens to be keyCode 1 (see initVirtualKeyMapping in CoreHelper) NSEvent *event = [NSEvent keyEventWithType:NSEventTypeKeyDown location:NSMakePoint(0, 0) modifierFlags:modifiers[i] timestamp:0 windowNumber:0 context:nil characters:characters charactersIgnoringModifiers:charactersIgnoringModifiers isARepeat:NO keyCode:1]; @@ -554,8 +562,10 @@ NSString * names[nCombinations]; NSLog(@"output: %@", output); XCTAssert(output.codePointsToDeleteBeforeInsert == 1, @"Expected output to delete one code point"); XCTAssert(!output.hasTextToInsert, @"expected to insert nothing"); - NSString *context = engine.getCoreContext; - XCTAssert([context isEqualToString:@""], @"Context should be empty."); + NSString *context = engine.getCoreContextDebug; + // Note: relying on km_core_state_context_debug output format is just barely + // acceptable for a unit test + XCTAssert([context isEqualToString:@"|| (len: 0) [ ]"], @"Context should be empty."); } - (void)testCoreProcessEvent_eventReturnWithElNuerKmx_EmitWithContextEmpty { @@ -564,8 +574,10 @@ NSString * names[nCombinations]; NSEvent *event = [NSEvent keyEventWithType:NSEventTypeKeyDown location:NSMakePoint(0, 0) modifierFlags:0 timestamp:0 windowNumber:0 context:nil characters:@"\n" charactersIgnoringModifiers:@"\n" isARepeat:NO keyCode:kVK_Return]; CoreKeyOutput *output = [engine processEvent:event]; XCTAssert(output.emitKeystroke, @"Expected emitKeystroke==YES"); - NSString *context = engine.getCoreContext; - XCTAssert([context isEqualToString:@""], @"Context should be cleared."); + NSString *context = engine.getCoreContextDebug; + // Note: relying on km_core_state_context_debug output format is just barely + // acceptable for a unit test + XCTAssert([context isEqualToString:@"|| (len: 0) [ ]"], @"Context should be cleared."); } - (void)testCoreProcessEvent_eventTabWithElNuerKmx_EmitWithContextEmpty { @@ -574,8 +586,10 @@ NSString * names[nCombinations]; NSEvent *event = [NSEvent keyEventWithType:NSEventTypeKeyDown location:NSMakePoint(0, 0) modifierFlags:0 timestamp:0 windowNumber:0 context:nil characters:@"\t" charactersIgnoringModifiers:@"\t" isARepeat:NO keyCode:kVK_Tab]; CoreKeyOutput *output = [engine processEvent:event]; XCTAssert(output.emitKeystroke, @"Expected emitKeystroke==YES"); - NSString *context = engine.getCoreContext; - XCTAssert([context isEqualToString:@""], @"Context should be cleared."); + NSString *context = engine.getCoreContextDebug; + // Note: relying on km_core_state_context_debug output format is just barely + // acceptable for a unit test + XCTAssert([context isEqualToString:@"|| (len: 0) [ ]"], @"Context should be cleared."); } - (void)testCoreProcessEvent_eventSingleQuoteWithElNuerKmx_ReturnsDiacritic { @@ -585,8 +599,10 @@ NSString * names[nCombinations]; NSEvent *event = [NSEvent keyEventWithType:NSEventTypeKeyDown location:NSMakePoint(0, 0) modifierFlags:0 timestamp:0 windowNumber:0 context:nil characters:@"'" charactersIgnoringModifiers:@"'" isARepeat:NO keyCode:kVK_ANSI_Quote]; CoreKeyOutput *output = [engine processEvent:event]; XCTAssert(output.hasTextToInsert, @"returns text to insert"); - context = engine.getCoreContext; - XCTAssert([context isEqualToString:@"\u025B\u0308"], @"Context updated with diacritic."); + context = engine.getCoreContextDebug; + // Note: relying on km_core_state_context_debug output format is just barely + // acceptable for a unit test + XCTAssert([context isEqualToString:@"|\u025B\u0308| (len: 2) [ U+025b U+0308 ]"], @"Context updated with diacritic."); } @end diff --git a/windows/src/engine/keyman32/calldll.cpp b/windows/src/engine/keyman32/calldll.cpp index d71b73145d..8bb3e494df 100644 --- a/windows/src/engine/keyman32/calldll.cpp +++ b/windows/src/engine/keyman32/calldll.cpp @@ -154,38 +154,17 @@ LogContext(km_core_state *lpCoreKeyboardState, uint8_t context_type) { if (!lpCoreKeyboardState) { return FALSE; } - km_core_context_item *citems = nullptr; - km_core_status error_status; - char *log_str_title = nullptr; - char* const int_context = "Intermediate Context"; - char* const core_context = "Core Context"; - switch(context_type){ - case CONTEXT_CORE: - error_status = (km_core_status_codes)km_core_context_get(km_core_state_context(lpCoreKeyboardState), &citems); - log_str_title = int_context; - break; - case CONTEXT_INT: - error_status = (km_core_status_codes)km_core_state_get_intermediate_context(lpCoreKeyboardState, &citems); - log_str_title = core_context; - break; - default: - error_status = KM_CORE_STATUS_INVALID_ARGUMENT; - } - if (error_status != KM_CORE_STATUS_OK){ - km_core_context_items_dispose(citems); - return FALSE; - } - DWORD context_length = (DWORD)km_core_context_item_list_size(citems); - WCHAR *buf = new WCHAR[(context_length * 3) +1 ]; // *3 if every context item was a deadkey - if (!ContextItemToAppContext(citems, buf, context_length)) { - km_core_context_items_dispose(citems); - delete[] buf; - return FALSE; - } - km_core_context_items_dispose(citems); - SendDebugMessageFormat(0, sdmKeyboard, 0, "%s: [%s]", log_str_title, Debug_UnicodeString(buf)); - delete[] buf; + km_core_cp* buffer = km_core_state_context_debug( + lpCoreKeyboardState, + context_type == CONTEXT_CORE ? KM_CORE_DEBUG_CONTEXT_CACHED : KM_CORE_DEBUG_CONTEXT_INTERMEDIATE + ); + char* const log_str_title = context_type == CONTEXT_CORE ? "Core Context" : "Intermediate Context"; + + SendDebugMessageFormat(0, sdmKeyboard, 0, "%s: %ls", log_str_title, buffer); + + km_core_cp_dispose(buffer); + return TRUE; } diff --git a/windows/src/engine/keyman32/keymanengine.h b/windows/src/engine/keyman32/keymanengine.h index c0db193fdf..9befcb302a 100644 --- a/windows/src/engine/keyman32/keymanengine.h +++ b/windows/src/engine/keyman32/keymanengine.h @@ -40,6 +40,7 @@ #include #include "../../../../common/windows/cpp/include/legacy_kmx_file.h" #include +#include // for intermediate context #include /***************************************************************************/ diff --git a/windows/src/engine/keyman32/kmprocess.cpp b/windows/src/engine/keyman32/kmprocess.cpp index b83079b03d..c95c7ce5fd 100644 --- a/windows/src/engine/keyman32/kmprocess.cpp +++ b/windows/src/engine/keyman32/kmprocess.cpp @@ -70,31 +70,6 @@ BOOL fOutputKeystroke; -char *getcontext_debug() { - - PKEYMAN64THREADDATA _td = ThreadGlobals(); - if (!_td || !_td->lpActiveKeyboard || !_td->lpActiveKeyboard->lpCoreKeyboardState){ - return ""; - } - - WCHAR buf[(MAXCONTEXT * 3) + 1]; // *3 if every context item was a deadkey - km_core_context_item *citems = nullptr; - - if (KM_CORE_STATUS_OK != km_core_context_get( - km_core_state_context(_td->lpActiveKeyboard->lpCoreKeyboardState), &citems)) { - return ""; - } - - DWORD context_length = (DWORD)km_core_context_item_list_size(citems); - if (!ContextItemToAppContext(citems, buf, context_length)) { - km_core_context_items_dispose(citems); - return ""; - } - km_core_context_items_dispose(citems); - return Debug_UnicodeString(buf); - -} - /** * Process the key stroke using the core processor * @@ -148,8 +123,18 @@ BOOL ProcessHook() if(_td->state.msg.message == wm_keymankeydown) { // I4827 if (ShouldDebug(sdmKeyboard)) { - SendDebugMessageFormat(_td->state.msg.hwnd, sdmKeyboard, 0, "Key pressed: %s Context '%s'", - Debug_VirtualKey(_td->state.vkey), getcontext_debug()); + if(!_td->lpActiveKeyboard || !_td->lpActiveKeyboard->lpCoreKeyboardState) { + SendDebugMessageFormat(_td->state.msg.hwnd, sdmKeyboard, 0, "Key pressed: %s Context ", + Debug_VirtualKey(_td->state.vkey)); + } else { + km_core_cp* debug_context = km_core_state_context_debug( + _td->lpActiveKeyboard->lpCoreKeyboardState, + KM_CORE_DEBUG_CONTEXT_CACHED + ); + SendDebugMessageFormat(_td->state.msg.hwnd, sdmKeyboard, 0, "Key pressed: %s Context '%ls'", + Debug_VirtualKey(_td->state.vkey), debug_context); + km_core_cp_dispose(debug_context); + } } }