erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-10-06 14:07:32 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 11

Merge branch 'epic/core/9999-normalization' into chore/merge-master-into-core/9999-normalization

Browse source
This commit is contained in:
Marc Durdin 2024-01-19 17:42:40 +11:00 • committed by GitHub
commit 249ed77835
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
67 changed files with 2144 additions and 886 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
common/include/test_color.h
View file

@ -23,7 +23,7 @@ enum ansi_code {
BRIGHT_RED = 196
};
bool enabled = false;
bool enabled = false; // TODO: move to test_color.c because test_color.h cannot be #included in more than 1 file in a project otherwise.
class fg {
ansi_code code;

3
core/README.md Normal file
View file

@ -0,0 +1,3 @@
# Keyman Core
Version 1.0.0 of libkeymancore released with Keyman 17.0

421
core/include/keyman/keyman_core_api.h
View file

@ -115,7 +115,6 @@ typedef uint32_t km_core_status; // Status return code.
// Opaque object types.
//
typedef struct km_core_context km_core_context;
typedef struct km_core_keyboard km_core_keyboard;
typedef struct km_core_state km_core_state;
typedef struct km_core_options km_core_options;
@ -146,6 +145,7 @@ enum km_core_status_codes {
KM_CORE_STATUS_INSUFFICENT_BUFFER = 5,
KM_CORE_STATUS_INVALID_UTF = 6,
KM_CORE_STATUS_INVALID_KEYBOARD = 7,
KM_CORE_STATUS_NOT_IMPLEMENTED = 8,
KM_CORE_STATUS_OS_ERROR = 0x80000000
};
@ -155,357 +155,7 @@ The final status code KM_CORE_STATUS_OS_ERROR is intended to allow encapsulating
a platform error code; the remaining 31 low bits are the error code returned by
the OS for cases where the failure mode is platform specific. For HRESULT codes
this only permits failure codes to be passed.
### Context
The context is the text prior to the insertion point (caret, cursor).
The context is constructed by the Platform layer, typically by interrogating the
Client Application. The context will be updated by the engine for keystroke
events. If the Platform layer code caches the context, the context should be
reset when a context state change is detected. Context state changes can occur
when the user uses the mouse to move the insertion point, uses cursor keys,
switches applications or input fields, or presses hotkeys such as Ctrl+N to
start a new document. The full set of context state change triggers is up to the
Platform layer.
Context can also contain positional Markers (also known as 'deadkeys' in kmn
keyboards), which are transitory state flags that are erased whenever a context
state change is detected. Markers are always controlled by the Engine.
Contexts are always owned by their state. They may be set to a list of
context_items or interrogated for their current list of context items.
```c
*/
enum km_core_context_type {
KM_CORE_CT_END,
KM_CORE_CT_CHAR,
KM_CORE_CT_MARKER
};
typedef struct {
uint8_t type;
uint8_t _reserved[3];
union {
km_core_usv character;
uint32_t marker;
};
} km_core_context_item;
#define KM_CORE_CONTEXT_ITEM_END {KM_CORE_CT_END, {0,}, {0,}}
/*
```
### `km_core_context_items_from_utf16`
##### Description:
Convert a UTF16 encoded Unicode string into an array of `km_core_context_item`
structures. Allocates memory as needed.
##### Return status:
- `KM_CORE_STATUS_OK`: On success.
- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null.
- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated for the
output buffer.
- `KM_CORE_STATUS_INVALID_UTF`: In the event the UTF16 string cannot be decoded
because it contains unpaired surrogate codeunits.
##### Parameters:
- __text__: a pointer to a null terminated array of utf16 encoded data.
- __out_ptr__: a pointer to the result variable:
A pointer to the start of the `km_core_context_item` array containing the
representation of the input string.
Terminated with a type of `KM_CORE_CT_END`. Must be disposed of with
`km_core_context_items_dispose`.
```c
*/
KMN_API
km_core_status
km_core_context_items_from_utf16(km_core_cp const *text,
km_core_context_item **out_ptr);
/*
```
### `km_core_context_items_from_utf8`
##### Description:
Convert an UTF8 encoded Unicode string into an array of `km_core_context_item`
structures. Allocates memory as needed.
##### Status:
- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null.
- `KM_CORE_STATUS_NO_MEM`: In the event it cannot allocate enough memory for the
output buffer.
- `KM_CORE_STATUS_INVALID_UTF`: In the event the UTF8 string cannot be
decoded.
##### Parameters:
- __text__: a pointer to a null terminated array of utf8 encoded data.
- __out_ptr__: a pointer to the result variable:
A pointer to the start of the `km_core_context_item` array containing the
representation of the input string.
Terminated with a type of `KM_CORE_CT_END`.
```c
*/
KMN_API
km_core_status
km_core_context_items_from_utf8(char const *text,
km_core_context_item **out_ptr);
/*
```
### `km_core_context_items_to_utf16`
##### Description:
Convert a context item array into a UTF-16 encoded string placing it into
the supplied buffer of specified size, and return the number of code units
actually used in the conversion. If null is passed as the buffer the
number of codeunits required is returned. Any markers in the context will
not be included in the output buffer.
##### Return status:
- `KM_CORE_STATUS_OK`: On success.
- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null.
- `KM_CORE_STATUS_INSUFFICENT_BUFFER`: If the buffer is not large enough.
`buf_size` will contain the space required. The contents of the buffer are
undefined.
##### Parameters:
- __context_items__: A pointer to the start of an array `km_core_context_item`.
Must be terminated with a type of `KM_CORE_CT_END`.
- __buf__: A pointer to the buffer to place the UTF-16 string into.
May be null to request size calculation.
- __buf_size__: a pointer to the result variable:
The size of the supplied buffer in codeunits if `buf` is given.
On return will be the size required if `buf` is null.
```c
*/
KMN_API
km_core_status
km_core_context_items_to_utf16(km_core_context_item const *item,
km_core_cp *buf,
size_t *buf_size);
/*
```
### `km_core_context_items_to_utf8`
##### Description:
Convert a context item array into a UTF-8 encoded string placing it into
the supplied buffer of specified size, and return the number of code units
actually used in the conversion. If null is passed as the buffer the
number of codeunits required is returned. Any markers in the context will
not be included in the output buffer.
##### Return status:
- `KM_CORE_STATUS_OK`: On success.
- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null.
- `KM_CORE_STATUS_INSUFFICENT_BUFFER`: If the buffer is not large enough.
`buf_size` will contain the space required. The contents of the buffer are
undefined.
##### Parameters:
- __context_items__: A pointer to the start of an array `km_core_context_item`.
Must be terminated with a type of `KM_CORE_CT_END`.
- __buf__: A pointer to the buffer to place the UTF-8 string into.
May be null to request size calculation.
- __buf_size__: a pointer to the result variable:
The size of the supplied buffer in codeunits if `buf` is given.
On return will be the size required if `buf` is null.
```c
*/
KMN_API
km_core_status
km_core_context_items_to_utf8(km_core_context_item const *item,
char *buf,
size_t *buf_size);
/*
```
### `km_core_context_items_to_utf32`
##### Description:
Convert a context item array into a UTF-32 encoded string placing it into
the supplied buffer of specified size, and return the number of codepoints
actually used in the conversion. If null is passed as the buffer the
number of codepoints required is returned. Any markers in the context will
not be included in the output buffer.
##### Return status:
- `KM_CORE_STATUS_OK`: On success.
- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null.
- `KM_CORE_STATUS_INSUFFICENT_BUFFER`: If the buffer is not large enough.
`buf_size` will contain the space required. The contents of the buffer are
undefined.
##### Parameters:
- __context_items__: A pointer to the start of an array `km_core_context_item`.
Must be terminated with a type of `KM_CORE_CT_END`.
- __buf__: A pointer to the buffer to place the UTF-32 string into.
May be null to request size calculation.
- __buf_size__: a pointer to the result variable:
The size of the supplied buffer in codepoints if `buf` is given.
On return will be the size required if `buf` is null.
```c
*/
KMN_API
km_core_status
km_core_context_items_to_utf32(km_core_context_item const *item,
km_core_usv *buf,
size_t *buf_size);
/*
```
### `km_core_context_items_dispose`
##### Description:
Free the allocated memory belonging to a `km_core_context_item` array previously
returned by `km_core_context_items_from_utf16` or `km_core_context_get`
##### Parameters:
- __context_items__: A pointer to the start of the `km_core_context_item` array
to be disposed of.
```c
*/
KMN_API
void
km_core_context_items_dispose(km_core_context_item *context_items);
/*
```
### `km_core_context_set`
##### Description:
Replace the contents of the current context with a new sequence of
`km_core_context_item` entries.
##### Return status:
- `KM_CORE_STATUS_OK`: On success.
- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null.
- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated to
grow the context buffer internally.
##### Parameters:
- __context__: A pointer to an opaque context object
- __context_items__: A pointer to the start of the `km_core_context_item`
array containing the new context. It must be terminated with an item
of type `KM_CORE_CT_END`.
```c
*/
KMN_API
km_core_status
km_core_context_set(km_core_context *context,
km_core_context_item const *context_items);
/*
```
### `km_core_context_get`
##### Description:
Copies all items in the context into a new array and returns the new array.
This must be disposed of by caller using `km_core_context_items_dispose`.
##### Return status:
- `KM_CORE_STATUS_OK`: On success.
- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null.
- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated for the
output buffer.
##### Parameters:
- __context_items__: A pointer to the start of an array `km_core_context_item`.
- __out__: a pointer to the result variable:
A pointer to the start of the `km_core_context_item` array containing a
copy of the context. Terminated with a type of `KM_CORE_CT_END`. Must be
disposed of with `km_core_context_items_dispose`.
```c
*/
KMN_API
km_core_status
km_core_context_get(km_core_context const *context_items,
km_core_context_item **out);
/*
```
### `km_core_context_clear`
##### Description:
Removes all context_items from the internal array. If `context` is
null, has no effect.
##### Parameters:
- __context__: A pointer to an opaque context object
```c
*/
KMN_API
void
km_core_context_clear(km_core_context *);
/*
```
### `km_core_context_length`
##### Description:
Return the number of items in the context.
##### Return:
The number of items in the context, and will return 0 if passed a null `context`
pointer.
##### Parameters:
- __context__: A pointer to an opaque context object
```c
*/
KMN_API
size_t
km_core_context_length(km_core_context *);
/*
```
### `km_core_context_append`
##### Description:
Add more items to the end (insertion point) of the context. If these exceed the
maximum context length the same number of items will be dropped from the
beginning of the context.
##### Return status:
- `KM_CORE_STATUS_OK`: On success.
- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null.
- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated to
grow the context buffer internally.
##### Parameters:
- __context__: A pointer to an opaque context object.
- __context_items__: A pointer to the start of the `KM_CORE_CT_END` terminated
array of `km_core_context_item` to append.
```c
*/
KMN_API
km_core_status
km_core_context_append(km_core_context *context,
km_core_context_item const *context_items);
/*
```
### `km_core_context_shrink`
##### Description:
Remove a specified number of items from the end of the context, optionally
add up to the same number of the supplied items to the front of the context.
##### Return status:
- `KM_CORE_STATUS_OK`: On success.
- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null.
- `KM_CORE_STATUS_NO_MEM`: in the event it cannot allocated enough memory to grow
the context internally.
##### Parameters:
- __context__: A pointer to an opaque context object.
- __num__: The number of items to remove from the end of context.
- __context_items__: Pointer to the start of the `KM_CORE_CT_END` terminated
array of `km_core_context_item` to add to the front. Up to `num` items will
be prepended. This may be null if not required.
```c
*/
KMN_API
km_core_status
km_core_context_shrink(km_core_context *context,
size_t num,
km_core_context_item const *prefix);
/*
```
### `km_core_context_item_list_size`
##### Description:
Return the length of a terminated `km_core_context_item` array.
##### Return:
The number of items in the list, not including terminating item,
or 0 if `context_items` is null.
##### Parameters:
- __context_items__: A pointer to a `KM_CORE_CT_END` terminated array of
`km_core_context_item` values.
```c
*/
KMN_API
size_t
km_core_context_item_list_size(km_core_context_item const *context_items);
/*
```
@ -1106,39 +756,70 @@ km_core_state_dispose(km_core_state *state);
/*
```
### `km_core_state_context`
##### Description:
Get access to the state object's context.
##### Return:
A pointer to an opaque state object. This pointer is valid for the lifetime
of the state object. If null is passed in, then null is returned.
##### Parameters:
- __state__: A pointer to the opaque state object to be queried.
### Context Debug Reporting
As of version 17, the cached context is an internal property of the
state, not exposed to the consumer of the API -- apart from the
Keyman Developer Keyboard Debugger. However, for other debug
purposes, it is helpful to be able to examine the cached context, so
a debug-formatted version of the context is made available with
`km_core_state_context_debug`. This is not intended to be parsed for
reading the context for other purposes, and the format may change.
The three context types are:
* cached: the internal context used by Core, which may be normalized
and may contain markers. This is set via
km_core_state_context_set_if_needed, and will be modified
during keystroke event processing.
* intermediate: internal context used by IMX, only valid during
keystroke event processing.
* app: an exact copy of the current context passed in to
km_core_state_context_set_if_needed, which is used to verify
the precise text manipulations required when emitted changes.
This input context is in "NFU" -- normalization form unknown,
and may be mixed normalization so may require fixups when
it is manipulated by keyboard processors that support
normalization, such as the LDML keyboard processor.
```c
*/
KMN_API
km_core_context *
km_core_state_context(km_core_state *state);
typedef enum {
KM_CORE_DEBUG_CONTEXT_CACHED = 0,
KM_CORE_DEBUG_CONTEXT_INTERMEDIATE = 1,
KM_CORE_DEBUG_CONTEXT_APP = 2
} km_core_debug_context_type;
/*
```
### `km_core_state_get_intermediate_context`
### `km_core_state_context_debug`
##### Description:
Get access to the state object's keyboard processor's intermediate context. This context
is used during an IMX callback, part way through processing a keystroke.
Returns a debug formatted string of the context from the state.
##### Return:
A pointer to an context item array. Must be disposed of by a call
to `km_core_context_items_dispose`.
A pointer to a km_core_cp UTF-16 string. Must be disposed of by a call
to `km_core_cp_dispose`.
##### Parameters:
- __state__: A pointer to the opaque state object to be queried.
- __context_type__: the type of context to retrieve from the state
```c
*/
KMN_API
km_core_status
km_core_state_get_intermediate_context(km_core_state *state, km_core_context_item ** context_items);
km_core_cp *
km_core_state_context_debug(km_core_state *state, km_core_debug_context_type context_type);
/*
```
### `km_core_cp_dispose`
##### Description:
Free the allocated memory belonging to a `km_core_cp` array previously
returned by `km_core_state_context_debug`. May be `nullptr`.
##### Parameters:
- __cp__: A pointer to the start of the `km_core_cp` array
to be disposed of.
```c
*/
KMN_API
void
km_core_cp_dispose(km_core_cp *cp);
/*
```

486
core/include/keyman/keyman_core_api_context.h Normal file
View file

@ -0,0 +1,486 @@
/*
* Keyman is copyright (C) SIL International. MIT License.
*
* Keyman Keyboard Processor API - Debugger Interfaces
*
* The debugger interfaces are still very dependent on .kmx
* objects.
*
* Note: this file is subject to change; the debugger
* interfaces are not stable across versions.
*
*/
#pragma once
#include <stdint.h>
#include <stdlib.h>
#include <keyman/keyman_core_api_bits.h>
#include <keyman/keyman_core_api_vkeys.h>
// Currently, the Core unit tests use private context APIs defined in
// keyman_core_api_context.h, which are unused by other consumers. We are
// hoping to remove these entirely in the future, so we restrict access
// by default with this macro. Keyman Core internally uses these functions
// #define _KM_CORE_ACCESS_PRIVATE_CONTEXT_API
#if defined(__cplusplus)
extern "C"
{
#endif
typedef struct km_core_context km_core_context;
// ----------------------------------------------------------------------------------
// Context APIs are now available only to the keyboard debugger, IMX, and Core unit
// tests (17.0)
// ----------------------------------------------------------------------------------
/*
### Context
The context is the text prior to the insertion point (caret, cursor). The
context is constructed by the Platform layer, typically by interrogating the
Client Application. The context will be updated by the engine for keystroke
events. If the Platform layer code caches the context, the context should be
reset when a context state change is detected. Context state changes can occur
when the user uses the mouse to move the insertion point, uses cursor keys,
switches applications or input fields, or presses hotkeys such as Ctrl+N to
start a new document. The full set of context state change triggers is up to the
Platform layer.
Context can also contain positional Markers (also known as 'deadkeys' in kmn
keyboards), which are transitory state flags that are erased whenever a context
state change is detected. Markers are always controlled by the Engine.
Contexts are always owned by their state. They may be set to a list of
context_items or interrogated for their current list of context items.
Core maintains and caches the context. Engine can update the context with
`km_core_state_context_set_if_needed` and `km_core_state_context_clear`. These
two functions are available in keyman_core_api.h.
The Keyboard Debugger in Keyman Developer, and IMX in Keyman for Windows, make
use of the context functionality in this header, but these functions should not
be used in other places.
```c
*/
enum km_core_context_type {
KM_CORE_CT_END,
KM_CORE_CT_CHAR,
KM_CORE_CT_MARKER
};
typedef struct {
uint8_t type;
uint8_t _reserved[3];
union {
km_core_usv character;
uint32_t marker;
};
} km_core_context_item;
#define KM_CORE_CONTEXT_ITEM_END {KM_CORE_CT_END, {0,}, {0,}}
/*
```
### `km_core_state_get_intermediate_context`
##### Description:
Get access to the state object's keyboard processor's intermediate context. This context
is used during an IMX callback, part way through processing a keystroke.
##### Return:
A pointer to an context item array. Must be disposed of by a call
to `km_core_context_items_dispose`.
##### Parameters:
- __state__: A pointer to the opaque state object to be queried.
```c
*/
KMN_API
km_core_status
km_core_state_get_intermediate_context(km_core_state *state, km_core_context_item ** context_items);
/*
```
### `km_core_context_items_dispose`
##### Description:
Free the allocated memory belonging to a `km_core_context_item` array previously
returned by `km_core_context_items_from_utf16` or `km_core_context_get`
##### Parameters:
- __context_items__: A pointer to the start of the `km_core_context_item` array
to be disposed of.
```c
*/
KMN_API
void
km_core_context_items_dispose(km_core_context_item *context_items);
/**
* Get access to the state object's cached context.
* @param state A pointer to the opaque state object to be queried.
* @returns A pointer to an opaque context object. This pointer is valid for the
* lifetime of the state object. If null is passed in, then null is
* returned.
*/
KMN_API
km_core_context *
km_core_state_context(km_core_state const *state);
/**
* Get access to the state object's application context.
* @param state A pointer to the opaque state object to be queried.
* @returns A pointer to an opaque context object. This pointer is valid for the
* lifetime of the state object. If null is passed in, then null is
* returned.
*/
KMN_API
km_core_context *
km_core_state_app_context(km_core_state const *state);
/*
```
### `km_core_context_items_from_utf16`
##### Description:
Convert a UTF16 encoded Unicode string into an array of `km_core_context_item`
structures. Allocates memory as needed.
##### Return status:
- `KM_CORE_STATUS_OK`: On success.
- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null.
- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated for the
output buffer.
- `KM_CORE_STATUS_INVALID_UTF`: In the event the UTF16 string cannot be decoded
because it contains unpaired surrogate codeunits.
##### Parameters:
- __text__: a pointer to a null terminated array of utf16 encoded data.
- __out_ptr__: a pointer to the result variable:
A pointer to the start of the `km_core_context_item` array containing the
representation of the input string.
Terminated with a type of `KM_CORE_CT_END`. Must be disposed of with
`km_core_context_items_dispose`.
```c
*/
#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API
KMN_API
km_core_status
km_core_context_items_from_utf16(km_core_cp const *text,
km_core_context_item **out_ptr);
#endif
/*
```
### `km_core_context_items_from_utf8`
##### Description:
Convert an UTF8 encoded Unicode string into an array of `km_core_context_item`
structures. Allocates memory as needed.
##### Status:
- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null.
- `KM_CORE_STATUS_NO_MEM`: In the event it cannot allocate enough memory for the
output buffer.
- `KM_CORE_STATUS_INVALID_UTF`: In the event the UTF8 string cannot be
decoded.
##### Parameters:
- __text__: a pointer to a null terminated array of utf8 encoded data.
- __out_ptr__: a pointer to the result variable:
A pointer to the start of the `km_core_context_item` array containing the
representation of the input string.
Terminated with a type of `KM_CORE_CT_END`.
```c
*/
#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API
KMN_API
km_core_status
km_core_context_items_from_utf8(char const *text,
km_core_context_item **out_ptr);
#endif
/*
```
### `km_core_context_items_to_utf16`
##### Description:
Convert a context item array into a UTF-16 encoded string placing it into
the supplied buffer of specified size, and return the number of code units
actually used in the conversion. If null is passed as the buffer the
number of codeunits required is returned. Any markers in the context will
not be included in the output buffer.
##### Return status:
- `KM_CORE_STATUS_OK`: On success.
- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null.
- `KM_CORE_STATUS_INSUFFICENT_BUFFER`: If the buffer is not large enough.
`buf_size` will contain the space required. The contents of the buffer are
undefined.
##### Parameters:
- __context_items__: A pointer to the start of an array `km_core_context_item`.
Must be terminated with a type of `KM_CORE_CT_END`.
- __buf__: A pointer to the buffer to place the UTF-16 string into.
May be null to request size calculation.
- __buf_size__: a pointer to the result variable:
The size of the supplied buffer in codeunits if `buf` is given.
On return will be the size required if `buf` is null.
```c
*/
#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API
KMN_API
km_core_status
km_core_context_items_to_utf16(km_core_context_item const *item,
km_core_cp *buf,
size_t *buf_size);
#endif
/*
```
### `km_core_context_items_to_utf8`
##### Description:
Convert a context item array into a UTF-8 encoded string placing it into
the supplied buffer of specified size, and return the number of code units
actually used in the conversion. If null is passed as the buffer the
number of codeunits required is returned. Any markers in the context will
not be included in the output buffer.
##### Return status:
- `KM_CORE_STATUS_OK`: On success.
- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null.
- `KM_CORE_STATUS_INSUFFICENT_BUFFER`: If the buffer is not large enough.
`buf_size` will contain the space required. The contents of the buffer are
undefined.
##### Parameters:
- __context_items__: A pointer to the start of an array `km_core_context_item`.
Must be terminated with a type of `KM_CORE_CT_END`.
- __buf__: A pointer to the buffer to place the UTF-8 string into.
May be null to request size calculation.
- __buf_size__: a pointer to the result variable:
The size of the supplied buffer in codeunits if `buf` is given.
On return will be the size required if `buf` is null.
```c
*/
#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API
KMN_API
km_core_status
km_core_context_items_to_utf8(km_core_context_item const *item,
char *buf,
size_t *buf_size);
#endif
/*
```
### `km_core_context_items_to_utf32`
##### Description:
Convert a context item array into a UTF-32 encoded string placing it into
the supplied buffer of specified size, and return the number of codepoints
actually used in the conversion. If null is passed as the buffer the
number of codepoints required is returned. Any markers in the context will
not be included in the output buffer.
##### Return status:
- `KM_CORE_STATUS_OK`: On success.
- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null.
- `KM_CORE_STATUS_INSUFFICENT_BUFFER`: If the buffer is not large enough.
`buf_size` will contain the space required. The contents of the buffer are
undefined.
##### Parameters:
- __context_items__: A pointer to the start of an array `km_core_context_item`.
Must be terminated with a type of `KM_CORE_CT_END`.
- __buf__: A pointer to the buffer to place the UTF-32 string into.
May be null to request size calculation.
- __buf_size__: a pointer to the result variable:
The size of the supplied buffer in codepoints if `buf` is given.
On return will be the size required if `buf` is null.
```c
*/
#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API
KMN_API
km_core_status
km_core_context_items_to_utf32(km_core_context_item const *item,
km_core_usv *buf,
size_t *buf_size);
#endif
/*
```
### `km_core_context_set`
##### Description:
Replace the contents of the current context with a new sequence of
`km_core_context_item` entries.
##### Return status:
- `KM_CORE_STATUS_OK`: On success.
- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null.
- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated to
grow the context buffer internally.
##### Parameters:
- __context__: A pointer to an opaque context object
- __context_items__: A pointer to the start of the `km_core_context_item`
array containing the new context. It must be terminated with an item
of type `KM_CORE_CT_END`.
```c
*/
KMN_API
km_core_status
km_core_context_set(km_core_context *context,
km_core_context_item const *context_items);
/*
```
### `km_core_context_get`
##### Description:
Copies all items in the context into a new array and returns the new array.
This must be disposed of by caller using `km_core_context_items_dispose`.
##### Return status:
- `KM_CORE_STATUS_OK`: On success.
- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null.
- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated for the
output buffer.
##### Parameters:
- __context_items__: A pointer to the start of an array `km_core_context_item`.
- __out__: a pointer to the result variable:
A pointer to the start of the `km_core_context_item` array containing a
copy of the context. Terminated with a type of `KM_CORE_CT_END`. Must be
disposed of with `km_core_context_items_dispose`.
```c
*/
#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API
KMN_API
km_core_status
km_core_context_get(km_core_context const *context_items,
km_core_context_item **out);
#endif
/*
```
### `km_core_context_clear`
##### Description:
Removes all context_items from the internal array. If `context` is
null, has no effect.
##### Parameters:
- __context__: A pointer to an opaque context object
```c
*/
KMN_API
void
km_core_context_clear(km_core_context *);
/*
```
### `km_core_context_length`
##### Description:
Return the number of items in the context.
##### Return:
The number of items in the context, and will return 0 if passed a null `context`
pointer.
##### Parameters:
- __context__: A pointer to an opaque context object
```c
*/
#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API
KMN_API
size_t
km_core_context_length(km_core_context *);
#endif
/*
```
### `km_core_context_append`
##### Description:
Add more items to the end (insertion point) of the context. If these exceed the
maximum context length the same number of items will be dropped from the
beginning of the context.
##### Return status:
- `KM_CORE_STATUS_OK`: On success.
- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null.
- `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be allocated to
grow the context buffer internally.
##### Parameters:
- __context__: A pointer to an opaque context object.
- __context_items__: A pointer to the start of the `KM_CORE_CT_END` terminated
array of `km_core_context_item` to append.
```c
*/
#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API
KMN_API
km_core_status
km_core_context_append(km_core_context *context,
km_core_context_item const *context_items);
#endif
/*
```
### `km_core_context_shrink`
##### Description:
Remove a specified number of items from the end of the context, optionally
add up to the same number of the supplied items to the front of the context.
##### Return status:
- `KM_CORE_STATUS_OK`: On success.
- `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are null.
- `KM_CORE_STATUS_NO_MEM`: in the event it cannot allocated enough memory to grow
the context internally.
##### Parameters:
- __context__: A pointer to an opaque context object.
- __num__: The number of items to remove from the end of context.
- __context_items__: Pointer to the start of the `KM_CORE_CT_END` terminated
array of `km_core_context_item` to add to the front. Up to `num` items will
be prepended. This may be null if not required.
```c
*/
#ifdef _KM_CORE_ACCESS_PRIVATE_CONTEXT_API
KMN_API
km_core_status
km_core_context_shrink(km_core_context *context,
size_t num,
km_core_context_item const *prefix);
#endif
/*
```
### `km_core_context_item_list_size`
##### Description:
Return the length of a terminated `km_core_context_item` array.
##### Return:
The number of items in the list, not including terminating item,
or 0 if `context_items` is null.
##### Parameters:
- __context_items__: A pointer to a `KM_CORE_CT_END` terminated array of
`km_core_context_item` values.
```c
*/
KMN_API
size_t
km_core_context_item_list_size(km_core_context_item const *context_items);
#if defined(__cplusplus)
} // extern "C"
#endif

3
core/meson.build
View file

@ -29,6 +29,9 @@ python = py.find_installation()
# For now, we use KM_CORE_LIBRARY to inject the km::core::kmx namespace
defns += ['-DKM_CORE_LIBRARY']
# See keyman_core_api_context.h
defns += ['-D_KM_CORE_ACCESS_PRIVATE_CONTEXT_API']
# #define DEBUG when we are on a debug build
if get_option('buildtype') == 'debug'
add_global_arguments('-DDEBUG', language : 'cpp')

3
core/src/action.cpp
View file

@ -11,13 +11,12 @@
#include <sstream>
#include <memory>
#include <keyman/keyman_core_api.h>
#include "action.hpp"
#include "state.hpp"
#include "option.hpp"
km_core_actions const * km::core::action_item_list_to_actions_object(
km_core_actions * km::core::action_item_list_to_actions_object(
km_core_action_item const *action_items
) {
assert(action_items != nullptr);

16
core/src/action.hpp
View file

@ -8,14 +8,26 @@
#pragma once
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
#include <state.hpp>
namespace km {
namespace core
{
km_core_actions const *action_item_list_to_actions_object(
km_core_actions *action_item_list_to_actions_object(
km_core_action_item const *action_items
);
bool actions_normalize(
/* in */ km_core_context const *cached_context,
/* in, out */ km_core_context *app_context,
/* in, out */ km_core_actions *actions
);
bool actions_update_app_context_nfu(
/* in */ km_core_context const *cached_context,
/* in, out */ km_core_context *app_context
);
} // namespace core
} // namespace km

312
core/src/actions_normalize.cpp Normal file
View file

@ -0,0 +1,312 @@
/*
Copyright: © 2024 SIL International.
Description: Implementation of the action output normalization.
Create Date: 16 Jan 2024
Authors: Marc Durdin (MCD)
History: 16 Jan 2024 - MCD - Initial implementation from #9999
*/
#include <algorithm>
#include <sstream>
#include <memory>
#include <cassert>
#include "context.hpp"
#include "action.hpp"
#include "state.hpp"
#include "option.hpp"
#include "debuglog.h"
#include "core_icu.h"
// forward declarations
icu::UnicodeString context_items_to_unicode_string(km_core_context const *context);
km_core_usv *unicode_string_to_usv(icu::UnicodeString& src);
/**
* Normalize the output from an action to NFC, across the context | output
* boundary, fixing up the app_context and the output actions to take into
* account the NFU input app_context
*
* @param cached_context the cached context, in NFD, after transform has been
* applied to it by the keyboard processor
* @param app_context the app context, in NFU; transform has not been
* applied, and will be applied by this function
* @param actions transform to apply, in NFD, which will be converted
* to NFC by this function
* @return true on success, false on failure
*/
bool km::core::actions_normalize(
/* in */ km_core_context const *cached_context,
/* in, out */ km_core_context *app_context,
/* in, out */ km_core_actions *actions
) {
assert(actions != nullptr);
assert(cached_context != nullptr);
assert(app_context != nullptr);
if(actions == nullptr || cached_context == nullptr || app_context == nullptr) {
return false;
}
/*
The code_points_to_delete value at this point is in NFD. The cached_context
is in NFD and has already been updated by the keyboard processor to the
expected result of the action, so we need to remove the output from a string
copy of the cached_context to start, in order to get it to the same position
as the app_context.
The app_context is in NFU. We need to figure out how many characters to
remove from the end of app_context in order to correctly normalize across
the boundary, without normalizing more of the string than necessary.
We do not need to mutate the cached_context itself, because it is already
correct. This is good, because it means we will not lose track of markers
within it. The app_context will be mutated, as it will need the new output
appended, in order to match the expected result. Remember that the
app_context does not contain markers; these are maintained only in the
cached_context.
*/
/*
Initialization
*/
UErrorCode icu_status = U_ZERO_ERROR;
const icu::Normalizer2 *nfc = icu::Normalizer2::getNFCInstance(icu_status);
assert(U_SUCCESS(icu_status));
if(!U_SUCCESS(icu_status)) {
DebugLog("getNFCInstance failed with %x", icu_status);
return false;
}
const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(icu_status);
assert(U_SUCCESS(icu_status));
if(!U_SUCCESS(icu_status)) {
DebugLog("getNFDInstance failed with %x", icu_status);
return false;
}
icu::UnicodeString output = icu::UnicodeString::fromUTF32(reinterpret_cast<const UChar32*>(actions->output), -1);
icu::UnicodeString cached_context_string = context_items_to_unicode_string(cached_context);
icu::UnicodeString app_context_string = context_items_to_unicode_string(app_context);
assert(!output.isBogus());
assert(!cached_context_string.isBogus());
assert(!app_context_string.isBogus());
if(output.isBogus() || cached_context_string.isBogus() || app_context_string.isBogus()) {
return false;
}
int nfu_to_delete = 0;
/*
Further debug assertion of inputs
*/
assert(nfd->isNormalized(output, icu_status) && U_SUCCESS(icu_status));
assert(nfd->isNormalized(cached_context_string, icu_status) && U_SUCCESS(icu_status));
/*
The keyboard processor will have updated the cached_context already,
applying the transform to it, so we need to rewind this. Remove the output
from cached_context_string to start
*/
assert(cached_context_string.length() >= output.length());
int n = cached_context_string.length() - output.length();
assert(cached_context_string.compare(n, output.length(), output) == 0);
cached_context_string.remove(n);
/*
While cached_context is guaranteed to be normalized, actions->output may not
start at a normalization boundary. In order to achieve the correct NFC
normalization in our output, we now need to look for a normalization
boundary prior to the intersection of the cached_context and the output.
*/
while(n > 0 && output[0] && !nfd->hasBoundaryBefore(output[0])) {
// The output may interact with the context further in normalization. We
// need to copy characters back further until we reach a normalization
// boundary.
// Remove last code point from the context ...
n = cached_context_string.moveIndex32(n, -1);
UChar32 chr = cached_context_string.char32At(n);
cached_context_string.remove(n);
// And prepend it to the output ...
output.insert(0, chr);
// And finally remember that we now need to delete an additional NFD codepoint
actions->code_points_to_delete++;
}
/*
At this point, our output and cached_context are coherent and normalization
will be complete at the edit boundary.
Now, we need to adjust the delete_back to match the number of characters
that must actually be deleted from the applications's NFU context
To adjust, we remove one character at a time from the app_context until
its normalized form matches the cached_context normalized form.
*/
while(app_context_string.length()) {
icu::UnicodeString app_context_nfd;
nfd->normalize(app_context_string, app_context_nfd, icu_status);
assert(U_SUCCESS(icu_status));
if(!U_SUCCESS(icu_status)) {
DebugLog("nfd->normalize failed with %x", icu_status);
return false;
}
if(app_context_nfd.compare(cached_context_string) == 0) {
break;
}
app_context_string.remove(app_context_string.length()-1);
nfu_to_delete++;
}
/*
Normalize our output string
*/
icu::UnicodeString output_nfc;
nfc->normalize(output, output_nfc, icu_status);
assert(U_SUCCESS(icu_status));
if(!U_SUCCESS(icu_status)) {
DebugLog("nfc->normalize failed with %x", icu_status);
return false;
}
auto new_output = unicode_string_to_usv(output_nfc);
if(!new_output) {
// error logging handled in unicode_string_to_usv
return false;
}
/*
Final steps -- set our outputs
*/
// Append the new NFC output to our reduced app_context
app_context_string.append(output_nfc);
km_core_context_item *app_context_items = nullptr;
km_core_status status = KM_CORE_STATUS_OK;
if((status = km_core_context_items_from_utf16(app_context_string.getTerminatedBuffer(), &app_context_items)) != KM_CORE_STATUS_OK) {
DebugLog("km_core_context_items_from_utf16 failed with %x", status);
delete [] new_output;
return false;
}
if((status = km_core_context_set(app_context, app_context_items)) != KM_CORE_STATUS_OK) {
DebugLog("km_core_context_set failed with %x", status);
km_core_context_items_dispose(app_context_items);
delete [] new_output;
return false;
}
km_core_context_items_dispose(app_context_items);
// Update actions with new NFC output + count of NFU code points to delete
delete [] actions->output;
actions->output = new_output;
actions->code_points_to_delete = nfu_to_delete;
return true;
}
/**
* Helper to convert km_core_context list into a icu::UnicodeString
*/
icu::UnicodeString context_items_to_unicode_string(km_core_context const *context) {
icu::UnicodeString nullString;
nullString.setToBogus();
km_core_context_item *items = nullptr;
km_core_status status;
if((status = km_core_context_get(context, &items)) != KM_CORE_STATUS_OK) {
DebugLog("Failed to retrieve context with %s", status);
return nullString;
}
size_t buf_size = 0;
if((status = km_core_context_items_to_utf32(items, nullptr, &buf_size)) != KM_CORE_STATUS_OK) {
DebugLog("Failed to retrieve context size with %s", status);
km_core_context_items_dispose(items);
return nullString;
}
km_core_usv *buf = new km_core_usv[buf_size];
if((status = km_core_context_items_to_utf32(items, buf, &buf_size)) != KM_CORE_STATUS_OK) {
DebugLog("Failed to retrieve context with %s", status);
km_core_context_items_dispose(items);
delete [] buf;
return nullString;
}
auto result = icu::UnicodeString::fromUTF32(reinterpret_cast<const UChar32*>(buf), -1);
km_core_context_items_dispose(items);
delete [] buf;
return result;
}
/**
* Helper to convert icu::UnicodeString to a UTF-32 km_core_usv buffer,
* nul-terminated
*/
km_core_usv *unicode_string_to_usv(icu::UnicodeString& src) {
UErrorCode icu_status = U_ZERO_ERROR;
km_core_usv *dst = new km_core_usv[src.length() + 1];
src.toUTF32(reinterpret_cast<UChar32*>(dst), src.length(), icu_status);
assert(U_SUCCESS(icu_status));
if(!U_SUCCESS(icu_status)) {
DebugLog("toUTF32 failed with %x", icu_status);
delete[] dst;
return nullptr;
}
dst[src.length()] = 0;
return dst;
}
/**
* Refresh app_context to match the cached_context. Does not do normalization,
* unlike `actions_normalize`. Used in conjunction with keyboard processors that
* do not support normalization.
*
* @param cached_context the cached context, in NFU, after transform has been
* applied to it by the keyboard processor
* @param app_context the app context, in NFU; transform has not been
* applied, and will effectively be applied by this
* function
* @return true on success, false on failure
*/
bool km::core::actions_update_app_context_nfu(
/* in */ km_core_context const *cached_context,
/* in, out */ km_core_context *app_context
) {
// We simply copy the cached_context to the app_context
km_core_status status = KM_CORE_STATUS_OK;
km_core_context_item *items = nullptr;
if((status = km_core_context_get(cached_context, &items)) != KM_CORE_STATUS_OK) {
DebugLog("km_core_context_get failed with %d", status);
return false;
}
if((status = km_core_context_set(app_context, items)) != KM_CORE_STATUS_OK) {
DebugLog("km_core_context_set failed with %d", status);
}
delete [] items;
return status == KM_CORE_STATUS_OK;
}

7
core/src/context.hpp
View file

@ -9,7 +9,7 @@
#pragma once
#include <list>
#include <vector>
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
// Forward declarations
class json;
@ -39,6 +39,11 @@ void context::push_marker(uint32_t marker) {
emplace_back(km_core_context_item { KM_CORE_CT_MARKER, {0,}, {marker} });
}
// Context helper functions
km_core_cp* get_context_as_string(km_core_context *context);
km_core_status set_context_from_string(km_core_context *context, km_core_cp const *new_context);
} // namespace core
} // namespace km

69
core/src/context_helpers.cpp Normal file
View file

@ -0,0 +1,69 @@
/*
Copyright: © 2018-2024 SIL International.
Description: Helper functions for context data type conversions
Create Date: 18 Jan 2024
Authors: Marc Durdin
History: 18 Jan 2024 - MCD - Refactor from km_core_state_context_set_if_needed.cpp
*/
#include <cassert>
#include "keyman_core.h"
#include "context.hpp"
using namespace km::core;
/**
* Retrieves the context as a km_core_cp string, dropping markers
*/
km_core_cp* km::core::get_context_as_string(km_core_context *context) {
assert(context != nullptr);
if(context == nullptr) {
return nullptr;
}
size_t buf_size = 0;
km_core_context_item* context_items = nullptr;
if(km_core_context_get(context, &context_items) != KM_CORE_STATUS_OK) {
return nullptr;
}
if(km_core_context_items_to_utf16(context_items, nullptr, &buf_size) != KM_CORE_STATUS_OK) {
km_core_context_items_dispose(context_items);
return nullptr;
}
km_core_cp *app_context_string = new km_core_cp[buf_size];
km_core_status status = km_core_context_items_to_utf16(context_items, app_context_string, &buf_size);
km_core_context_items_dispose(context_items);
if(status != KM_CORE_STATUS_OK) {
return nullptr;
}
return app_context_string;
}
/**
* Updates the context from the new_context km_core_cp string
*/
km_core_status km::core::set_context_from_string(km_core_context *context, km_core_cp const *new_context) {
assert(context != nullptr);
assert(new_context != nullptr);
if(context == nullptr || new_context == nullptr) {
return KM_CORE_STATUS_INVALID_ARGUMENT;
}
km_core_context_item* new_context_items = nullptr;
km_core_status status = km_core_context_items_from_utf16(new_context, &new_context_items);
if (status != KM_CORE_STATUS_OK) {
return status;
}
km_core_context_set(context, new_context_items);
km_core_context_items_dispose(new_context_items);
return KM_CORE_STATUS_OK;
}

13
core/src/core_icu.h Normal file
View file

@ -0,0 +1,13 @@
/**
* ICU modules used by Keyman Core
*/
#pragma once
#if !defined(HAVE_ICU4C)
#error icu4c is required for this code
#endif
#define U_FALLTHROUGH
#include "unicode/utypes.h"
#include "unicode/unistr.h"
#include "unicode/normalizer2.h"

3
core/src/debug.hpp
View file

@ -9,8 +9,7 @@
#include <cassert>
#include <vector>
#include <keyman/keyman_core_api.h>
#include <keyman/keyman_core_api_debug.h>
#include "keyman_core.h"
namespace km {
namespace core

2
core/src/keyboard.hpp
View file

@ -11,7 +11,7 @@
#include <string>
#include <vector>
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
#include "option.hpp"
#include "path.hpp"

5
core/src/keyman_core.h Normal file
View file

@ -0,0 +1,5 @@
#pragma once
#include <keyman/keyman_core_api.h>
#include <keyman/keyman_core_api_context.h>
#include <keyman/keyman_core_api_debug.h>

20
core/src/km_core_action_api.cpp
View file

@ -10,7 +10,8 @@
#include <algorithm>
#include <sstream>
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
#include "jsonpp.hpp"
#include "processor.hpp"
@ -32,7 +33,22 @@ km_core_actions const * km_core_state_get_actions(
return nullptr;
}
return action_item_list_to_actions_object(action_items);
km_core_actions * result = action_item_list_to_actions_object(action_items);
if(state->processor().supports_normalization()) {
// Normalize to NFC for those keyboard processors that support it
if(!actions_normalize(km_core_state_context(state), km_core_state_app_context(state), result)) {
km_core_actions_dispose(result);
return nullptr;
}
} else {
// For all other keyboard processors, we just copy the cached_context to the app_context
if(!actions_update_app_context_nfu(km_core_state_context(state), km_core_state_app_context(state))) {
km_core_actions_dispose(result);
return nullptr;
}
}
return result;
}
km_core_status km_core_actions_dispose(

2
core/src/km_core_context_api.cpp
View file

@ -12,7 +12,7 @@
#include <algorithm>
#include <vector>
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
#include "context.hpp"
#include "jsonpp.hpp"

2
core/src/km_core_debug_api.cpp
View file

@ -8,7 +8,7 @@
#include <algorithm>
#include <sstream>
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
#include "processor.hpp"
#include "state.hpp"

3
core/src/km_core_keyboard_api.cpp
View file

@ -11,7 +11,8 @@
#include <cassert>
#include <vector>
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
#include "keyboard.hpp"
#include "processor.hpp"
#include "kmx/kmx_processor.hpp"

3
core/src/km_core_options_api.cpp
View file

@ -11,7 +11,8 @@
#include <cassert>
#include <sstream>
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
#include "processor.hpp"
#include "jsonpp.hpp"

3
core/src/km_core_processevent_api.cpp
View file

@ -8,7 +8,8 @@
History: 17 Oct 2018 - TSE - Initial implementation.
*/
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
#include "processor.hpp"
#include "state.hpp"

188
core/src/km_core_state_api.cpp
View file

@ -12,13 +12,16 @@
#include <algorithm>
#include <sstream>
#include <memory>
#include <iomanip>
#include <codecvt>
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
#include "jsonpp.hpp"
#include "processor.hpp"
#include "state.hpp"
using namespace km::core;
// Forward declarations
@ -62,12 +65,20 @@ void km_core_state_dispose(km_core_state *state)
}
km_core_context *km_core_state_context(km_core_state *state)
km_core_context *km_core_state_context(km_core_state const *state)
{
assert(state);
if (!state) return nullptr;
return static_cast<km_core_context *>(&state->context());
return static_cast<km_core_context *>(&(const_cast<km_core_state *>(state)->context()));
}
km_core_context *km_core_state_app_context(km_core_state const *state)
{
assert(state);
if (!state) return nullptr;
return static_cast<km_core_context *>(&(const_cast<km_core_state *>(state)->app_context()));
}
km_core_status km_core_state_get_intermediate_context(
@ -261,95 +272,6 @@ void km_core_state_imx_deregister_callback(km_core_state *state)
state->imx_deregister_callback();
}
bool is_context_valid(km_core_cp const * context, km_core_cp const * cached_context) {
if (context == nullptr || cached_context == nullptr || *cached_context == '\0') {
// If the cached_context is "empty" then it needs updating
return false;
}
km_core_cp const* context_p = context;
while(*context_p) {
context_p++;
}
km_core_cp const* cached_context_p = cached_context;
while(*cached_context_p) {
cached_context_p++;
}
// we need to compare from the end of the cached context
for(; context_p >= context && cached_context_p >= cached_context; context_p--, cached_context_p--) {
if(*context_p != *cached_context_p) {
// The cached context doesn't match the application context, so it is
// invalid
return false;
}
}
if(cached_context_p > cached_context) {
// if the cached context is longer than the application context, then we also
// assume that it is invalid
return false;
}
// It's acceptable for the application context to be longer than the cached
// context, so if we match the whole cached context, we can safely return true
return true;
}
km_core_context_status km_core_state_context_set_if_needed(
km_core_state *state,
km_core_cp const *application_context
) {
assert(state != nullptr);
assert(application_context != nullptr);
if(state == nullptr || application_context == nullptr) {
return KM_CORE_CONTEXT_STATUS_INVALID_ARGUMENT;
}
size_t buf_size;
km_core_context_item* context_items = nullptr;
auto context = km_core_state_context(state);
if(km_core_context_get(context, &context_items) != KM_CORE_STATUS_OK) {
return KM_CORE_CONTEXT_STATUS_ERROR;
}
if(km_core_context_items_to_utf16(context_items, nullptr, &buf_size) != KM_CORE_STATUS_OK) {
km_core_context_items_dispose(context_items);
return KM_CORE_CONTEXT_STATUS_ERROR;
}
std::unique_ptr<km_core_cp[]> cached_context(new km_core_cp[buf_size]);
km_core_status status = km_core_context_items_to_utf16(context_items, cached_context.get(), &buf_size);
km_core_context_items_dispose(context_items);
if(status != KM_CORE_STATUS_OK) {
return KM_CORE_CONTEXT_STATUS_ERROR;
}
bool is_valid = is_context_valid(application_context, cached_context.get());
if(is_valid) {
// We keep the context as is
return KM_CORE_CONTEXT_STATUS_UNCHANGED;
}
km_core_context_item* new_context_items = nullptr;
// We replace the cached context with the current application context
status = km_core_context_items_from_utf16(application_context, &new_context_items);
if (status != KM_CORE_STATUS_OK) {
km_core_context_clear(context);
return KM_CORE_CONTEXT_STATUS_CLEARED;
}
km_core_context_set(context, new_context_items);
km_core_context_items_dispose(new_context_items);
return KM_CORE_CONTEXT_STATUS_UPDATED;
}
km_core_status km_core_state_context_clear(
km_core_state *state
) {
@ -358,5 +280,87 @@ km_core_status km_core_state_context_clear(
return KM_CORE_STATUS_INVALID_ARGUMENT;
}
km_core_context_clear(km_core_state_context(state));
km_core_context_clear(km_core_state_app_context(state));
return KM_CORE_STATUS_OK;
}
void km_core_cp_dispose(
km_core_cp *cp
) {
if(cp != nullptr) {
delete [] cp;
}
}
km_core_cp * _new_error_string(std::u16string const str) {
km_core_cp* result = new km_core_cp[str.size()+1];
str.copy(result, str.size());
result[str.size()] = 0;
return result;
}
km_core_cp * km_core_state_context_debug(
km_core_state *state,
km_core_debug_context_type context_type
) {
km_core_context_item * context_items = nullptr;
if(context_type == KM_CORE_DEBUG_CONTEXT_INTERMEDIATE) {
if(km_core_state_get_intermediate_context(state, &context_items) != KM_CORE_STATUS_OK) {
return _new_error_string(u"<error retrieving intermediate context>");
}
} else if(context_type == KM_CORE_DEBUG_CONTEXT_CACHED) {
if(km_core_context_get(km_core_state_context(state), &context_items) != KM_CORE_STATUS_OK) {
return _new_error_string(u"<error retrieving cached context>");
}
} else if(context_type == KM_CORE_DEBUG_CONTEXT_APP) {
if(km_core_context_get(km_core_state_app_context(state), &context_items) != KM_CORE_STATUS_OK) {
return _new_error_string(u"<error retrieving app context>");
}
} else {
return _new_error_string(u"<invalid context type>");
}
size_t buf_size;
if(km_core_context_items_to_utf8(context_items, nullptr, &buf_size) != KM_CORE_STATUS_OK) {
km_core_context_items_dispose(context_items);
return _new_error_string(u"<could not retrieve context buffer size>");
}
std::vector<char> context_buffer(buf_size);
if(km_core_context_items_to_utf8(context_items, &context_buffer[0], &buf_size) != KM_CORE_STATUS_OK) {
km_core_context_items_dispose(context_items);
return _new_error_string(u"<could not retrieve context buffer>");
}
// construct the log message
std::stringstream buffer;
int context_item_length = 0;
for(auto cp = context_items; cp->type != KM_CORE_CT_END; cp++, context_item_length++);
buffer << "|" << std::string(&context_buffer[0]) << "| (len: " << context_item_length << ") [";
for(auto cp = context_items; cp->type != KM_CORE_CT_END; cp++) {
auto flags = buffer.flags();
if(cp->type == KM_CORE_CT_CHAR) {
// A single Unicode codepoint
buffer << " U+" << std::setfill('0') << std::setw(4) << std::hex << cp->character;
} else {
// A marker
buffer << " M(" << cp->marker << ")";
}
buffer.flags(flags);
}
buffer << " ]";
km_core_context_items_dispose(context_items);
std::u16string s = std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t>{}.from_bytes(buffer.str());
km_core_cp* result = new km_core_cp[s.size() + 1];
s.copy(result, s.size());
result[s.size()] = 0;
return result;
}

165
core/src/km_core_state_context_set_if_needed.cpp Normal file
View file

@ -0,0 +1,165 @@
/*
Copyright: © 2018-2024 SIL International.
Description: Implementation of the state API functions using internal
data structures and functions.
Create Date: 15 Jan 2024
Authors: Marc Durdin
History: 15 Jan 2024 - MCD - Refactor our km_core_state_context_set_if_needed
and implement normalization
*/
#include <cassert>
#include "keyman_core.h"
#include "processor.hpp"
#include "state.hpp"
#include "debuglog.h"
#include "core_icu.h"
using namespace km::core;
// Forward declarations
bool should_normalize(km_core_state *state);
bool is_context_valid(km_core_cp const * context, km_core_cp const * cached_context);
bool do_normalize_nfd(km_core_cp const * src, std::u16string &dst);
km_core_context_status do_fail(km_core_context *app_context, km_core_context *cached_context, const char* error);
// ---------------------------------------------------------------------------
km_core_context_status km_core_state_context_set_if_needed(
km_core_state *state,
km_core_cp const *new_app_context
) {
assert(state != nullptr);
assert(new_app_context != nullptr);
if(state == nullptr || new_app_context == nullptr) {
return KM_CORE_CONTEXT_STATUS_INVALID_ARGUMENT;
}
auto app_context = km_core_state_app_context(state);
auto cached_context = km_core_state_context(state);
// Retrieve the existing internally cached app context for comparison
std::unique_ptr<km_core_cp[]> app_context_string(get_context_as_string(app_context));
// Compare the internal app context with the passed-in application context
if(is_context_valid(new_app_context, app_context_string.get())) {
// We keep the context as is
return KM_CORE_CONTEXT_STATUS_UNCHANGED;
}
// We replace the internal app context with the passed-in application context
if(set_context_from_string(app_context, new_app_context) != KM_CORE_STATUS_OK) {
return do_fail(app_context, cached_context, "could not set new app context");
}
// Finally, we normalize and replace the cached context
std::u16string normalized_buffer;
km_core_cp const *new_cached_context = nullptr;
if(should_normalize(state)) {
if(!do_normalize_nfd(new_app_context, normalized_buffer)) {
return do_fail(app_context, cached_context, "could not normalize string");
}
new_cached_context = normalized_buffer.c_str();
} else {
new_cached_context = new_app_context;
}
// TODO: #10100 will alter how we replace the cached context here -- maintaining
// markers as far as possible
if(set_context_from_string(cached_context, new_cached_context) != KM_CORE_STATUS_OK) {
return do_fail(app_context, cached_context, "could not set new cached context");
}
return KM_CORE_CONTEXT_STATUS_UPDATED;
}
/**
* Returns true if the current keyboard processor wants a normalized cached context
*/
bool should_normalize(km_core_state *state) {
return state->processor().supports_normalization();
}
/**
* Returns true if the internal app context does not need to be updated to the new
* app context
*
* TODO: #10100 will alter some of the assumptions here
*/
bool is_context_valid(km_core_cp const * new_app_context, km_core_cp const * app_context) {
if (new_app_context == nullptr || app_context == nullptr || *app_context == '\0') {
// If the app_context is "empty" then it needs updating
return false;
}
km_core_cp const* new_app_context_p = new_app_context;
while(*new_app_context_p) {
new_app_context_p++;
}
km_core_cp const* app_context_p = app_context;
while(*app_context_p) {
app_context_p++;
}
// we need to compare from the end of the cached context
for(; new_app_context_p >= new_app_context && app_context_p >= app_context; new_app_context_p--, app_context_p--) {
if(*new_app_context_p != *app_context_p) {
// The cached context doesn't match the application context, so it is
// invalid
return false;
}
}
if(app_context_p > app_context) {
// if the cached context is longer than the application context, then we also
// assume that it is invalid
return false;
}
// It's acceptable for the application context to be longer than the cached
// context, so if we match the whole cached context, we can safely return true
return true;
}
/**
* Normalize the input string using ICU
*/
bool do_normalize_nfd(km_core_cp const * src, std::u16string &dst) {
UErrorCode icu_status = U_ZERO_ERROR;
const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(icu_status);
assert(U_SUCCESS(icu_status));
if(!U_SUCCESS(icu_status)) {
// TODO: log the failure code
return false;
}
icu::UnicodeString udst;
icu::UnicodeString usrc = icu::UnicodeString(src);
nfd->normalize(usrc, udst, icu_status);
assert(U_SUCCESS(icu_status));
if(!U_SUCCESS(icu_status)) {
// TODO: log the failure code
return false;
}
dst.assign(udst.getBuffer(), udst.length());
return true;
}
/**
* Clear the context when we have a failure so we don't end up with inconsistent
* context buffers, and log the error to our diagnostic log.
*/
km_core_context_status do_fail(km_core_context *app_context, km_core_context *cached_context, const char* error) {
DebugLog("%s", error);
km_core_context_clear(app_context);
km_core_context_clear(cached_context);
return KM_CORE_CONTEXT_STATUS_CLEARED;
}

3
core/src/kmx/kmx_base.h
View file

@ -1,7 +1,6 @@
#pragma once
#include <keyman/keyman_core_api_bits.h>
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
#if defined(_WIN32) || defined(_WIN64)
#define snprintf _snprintf

2
core/src/kmx/kmx_options.h
View file

@ -4,7 +4,7 @@
#include <string>
#include <vector>
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
#include "option.hpp"
#include "kmx_base.h"

2
core/src/kmx/kmx_processor.cpp
View file

@ -1,4 +1,4 @@
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
#include "state.hpp"
#include "kmx/kmx_processor.hpp"
#include <map>

6
core/src/kmx/kmx_processor.hpp
View file

@ -9,7 +9,7 @@
#pragma once
#include <string>
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
#include "kmx/kmx_processevent.h"
#include "keyboard.hpp"
#include "processor.hpp"
@ -80,6 +80,10 @@ namespace core
km_core_keyboard_imx * get_imx_list() const override;
bool
supports_normalization() const override {
return false;
}
};
} // namespace core

9
core/src/ldml/ldml_markers.hpp
View file

@ -16,18 +16,11 @@
#include <utility>
#include "debuglog.h"
#if !defined(HAVE_ICU4C)
#error icu4c is required for this code
#endif
#define U_FALLTHROUGH
#include "unicode/utypes.h"
#include "core_icu.h"
#include "unicode/uniset.h"
#include "unicode/usetiter.h"
#include "unicode/unistr.h"
#include "unicode/regex.h"
#include "unicode/utext.h"
#include "unicode/normalizer2.h"
namespace km {
namespace core {

128
core/src/ldml/ldml_processor.cpp
View file

@ -151,24 +151,20 @@ bool ldml_processor::is_kmxplus_file(path const & kb_path, std::vector<uint8_t>&
km_core_status
ldml_processor::process_queued_actions(
km_core_state *state
km_core_state *_kmn_unused(state)
) {
assert(state);
if (!state)
return KM_CORE_STATUS_INVALID_ARGUMENT;
// TODO Implement
return KM_CORE_STATUS_OK;
/* Not implemented. Only used by IMX. */
assert(false);
return KM_CORE_STATUS_NOT_IMPLEMENTED;
}
bool ldml_processor::queue_action(
km_core_state * state,
km_core_action_item const* action_item
km_core_state * _kmn_unused(state),
km_core_action_item const* _kmn_unused(action_item)
)
{
assert(state);
assert(action_item);
if ((!state) || (!action_item))
return false;
/* Not implemented. Only used by IMX. */
assert(false);
return false;
}
@ -178,59 +174,68 @@ ldml_processor::process_event(
km_core_virtual_key vk,
uint16_t modifier_state,
uint8_t is_key_down,
uint16_t _kmn_unused(event_flags)
uint16_t event_flags
) {
assert(state);
if (!state)
return KM_CORE_STATUS_INVALID_ARGUMENT;
try {
// At the start of every process_event always clear the action_items
state->actions().clear();
// this will hold the parameters, and our response
ldml_event_state ldml_state(state, vk, modifier_state, is_key_down, event_flags);
ldml_state.clear();
try {
if (!is_key_down) {
process_key_up(state, vk, modifier_state);
process_key_up(ldml_state);
} else {
switch (vk) {
// Currently, only one VK gets spoecial treatment.
// Special handling for backspace VK
case KM_CORE_VKEY_BKSP:
process_backspace(state);
process_backspace(ldml_state);
break;
default:
// all other VKs
process_key_down(state, vk, modifier_state);
process_key_down(ldml_state);
} // end of switch
} // end of normal processing
// all key-up and key-down events end up here.
state->actions().commit(); // always commit
// commit the ldml state into the core state
ldml_state.commit();
return KM_CORE_STATUS_OK;
} catch (std::bad_alloc &) {
// out of memory, clean up and get out
state->actions().clear();
ldml_state.clear();
// no actions will be set
return KM_CORE_STATUS_NO_MEM;
}
}
void
ldml_processor::process_key_up(km_core_state *state, km_core_virtual_key _kmn_unused(vk), uint16_t _kmn_unused(modifier_state))
ldml_processor::process_key_up(ldml_event_state &ldml_state)
const {
// TODO-LDML: Implement caps lock handling
state->actions().clear(); // TODO-LDML: Why is clear here?
ldml_state.clear();
}
void
ldml_processor::process_backspace(km_core_state *state) const {
ldml_processor::process_backspace(ldml_event_state &ldml_state) const {
if (!!bksp_transforms) {
// process with an empty string
auto matchedContext = process_output(state, std::u32string(), bksp_transforms.get());
// process with an empty string voa the bksp transforms
auto matchedContext = process_output(ldml_state, std::u32string(), bksp_transforms.get());
if (matchedContext > 0) {
return; // The transform took care of the backspacing.
} // else, fall through to default processing below.
}
ldml_state.emit_backspace();
}
void ldml_event_state::emit_backspace() {
// this is called from user-initiated backspace, not internal backspacing.
// Find out what the last actual character was and remove it.
// attempt to get the last char
// TODO-LDML: emoji backspace
@ -243,6 +248,7 @@ ldml_processor::process_backspace(km_core_state *state) const {
} else if ((*end).type == KM_CORE_BT_MARKER) {
state->actions().push_backspace(KM_CORE_BT_MARKER, (*end).marker);
state->context().pop_back();
// TODO-LDML: fall through here?
}
}
/*
@ -257,31 +263,31 @@ ldml_processor::process_backspace(km_core_state *state) const {
}
void
ldml_processor::process_key_down(km_core_state *state, km_core_virtual_key vk, uint16_t modifier_state) const {
ldml_processor::process_key_down(ldml_event_state &ldml_state) const {
// Look up the key
bool found = false;
const std::u16string key_str = keys.lookup(vk, modifier_state, found);
const std::u16string key_str = keys.lookup(ldml_state.get_vk(), ldml_state.get_modifier_state(), found);
if (!found) {
// no key was found, so pass the keystroke on to the Engine
emit_invalidate_passthrough_keystroke(state, vk, modifier_state);
ldml_state.emit_invalidate_passthrough_keystroke();
} else if (!key_str.empty()) {
process_key_string(state, key_str);
process_key_string(ldml_state, key_str);
} // else no action: It's a gap or gap-like key.
}
void
ldml_processor::process_key_string(km_core_state *state, const std::u16string &key_str) const {
ldml_processor::process_key_string(ldml_event_state &ldml_state, const std::u16string &key_str) const {
// We know that key_str is not empty per the caller.
assert(!key_str.empty());
// we convert the keys str to UTF-32 here instead of using the emit_text() overload
// so that we don't have to reconvert it inside the transform code.
std::u32string key_str32 = kmx::u16string_to_u32string(key_str);
(void)process_output(state, key_str32, transforms.get());
(void)process_output(ldml_state, key_str32, transforms.get());
}
size_t ldml_processor::process_output(km_core_state *state, const std::u32string &str, ldml::transforms *with_transforms) const {
size_t ldml_processor::process_output(ldml_event_state &ldml_state, const std::u32string &str, ldml::transforms *with_transforms) const {
std::u32string nfd_str = str;
// Note:
// The normalize functions have assert and Debuglog at the bottom.
@ -291,12 +297,12 @@ size_t ldml_processor::process_output(km_core_state *state, const std::u32string
// extract context string, in NFD
std::u32string old_ctxtstr_nfd;
(void)context_to_string(state, old_ctxtstr_nfd, true);
(void)ldml::normalize_nfd_markers(old_ctxtstr_nfd);
(void)ldml_state.context_to_string(old_ctxtstr_nfd, true);
assert(ldml::normalize_nfd_markers(old_ctxtstr_nfd)); // TODO-LDML: else fail?
// context string in NFD
std::u32string ctxtstr;
(void)context_to_string(state, ctxtstr, true); // with markers
(void)ldml_state.context_to_string(ctxtstr, true); // TODO-LDML: remove this second call
// add the newly added key output to ctxtstr
ctxtstr.append(nfd_str);
(void)ldml::normalize_nfd_markers(ctxtstr);
@ -367,20 +373,21 @@ size_t ldml_processor::process_output(km_core_state *state, const std::u32string
// FIRST drop the old suffix. Note: this mutates old_ctxtstr_changed.
// see remove_text() docs, this PUSHes actions, POPs context items, and TRIMS the string.
remove_text(state, old_ctxtstr_changed, old_ctxtstr_changed.length());
ldml_state.remove_text(old_ctxtstr_changed, old_ctxtstr_changed.length());
assert(old_ctxtstr_changed.length() == 0);
// old_ctxtstr_changed is now empty because it's been removed.
// context is "aa" in the above example.
// THEN add the new suffix, "CCC" in the above example
emit_text(state, new_ctxtstr_changed);
ldml_state.emit_text(new_ctxtstr_changed);
// context is now "aaCCC"
return matchedContext;
}
void
ldml_processor::remove_text(km_core_state *state, std::u32string &str, size_t length) {
ldml_event_state::remove_text(std::u32string &str, size_t length) {
// str is the string to remove, so it should be at least as long as length
assert(length <= str.length());
/** track how many context items have been removed, via push_backspace() */
@ -446,13 +453,13 @@ km_core_status ldml_processor::validate() const {
}
void
ldml_processor::emit_text(km_core_state *state, const std::u16string &str) {
ldml_event_state::emit_text(const std::u16string &str) {
const std::u32string str32 = kmx::u16string_to_u32string(str);
emit_text(state, str32);
emit_text(str32);
}
void
ldml_processor::emit_text(km_core_state *state, const std::u32string &str) {
ldml_event_state::emit_text(const std::u32string &str) {
for (auto it = str.begin(); it < str.end(); it++) {
const auto ch = *it;
// If we are at the start of a sequence:
@ -465,28 +472,28 @@ ldml_processor::emit_text(km_core_state *state, const std::u32string &str) {
it++; // consume LDML_MARKER_CODE
assert(it < str.end());
const auto marker_no = *it;
emit_marker(state, marker_no);
emit_marker(marker_no);
} else {
emit_text(state, ch);
emit_text(ch);
}
}
}
void
ldml_processor::emit_text(km_core_state *state, km_core_usv ch) {
ldml_event_state::emit_text( km_core_usv ch) {
assert(ch != LDML_UC_SENTINEL);
state->context().push_character(ch);
state->actions().push_character(ch);
}
void
ldml_processor::emit_marker(km_core_state *state, KMX_DWORD marker_no) {
ldml_event_state::emit_marker( KMX_DWORD marker_no) {
assert(km::core::kmx::is_valid_marker(marker_no));
state->actions().push_marker(marker_no);
state->context().push_marker(marker_no);
}
void ldml_processor::emit_invalidate_passthrough_keystroke(km_core_state *state, km_core_virtual_key vk, uint16_t _kmn_unused(modifier_state)) {
void ldml_event_state::emit_invalidate_passthrough_keystroke() {
if ((vk < 0x100) && km::core::kmx::vkey_to_contextreset[vk]) {
state->actions().push_invalidate_context();
} else {
@ -496,10 +503,10 @@ void ldml_processor::emit_invalidate_passthrough_keystroke(km_core_state *state,
}
size_t
ldml_processor::context_to_string(km_core_state *state, std::u32string &str, bool include_markers) {
ldml_event_state::context_to_string(std::u32string &str, bool include_markers) {
str.clear();
auto &cp = state->context();
size_t ctxlen = 0; // TODO-LDML: not used by callers?
auto &cp = state->context();
size_t ctxlen = 0; // TODO-LDML: not used by callers?
uint8_t last_type = KM_CORE_BT_UNKNOWN;
for (auto c = cp.rbegin(); c != cp.rend(); c++, ctxlen++) {
last_type = c->type;
@ -517,5 +524,26 @@ ldml_processor::context_to_string(km_core_state *state, std::u32string &str, boo
return ctxlen; // consumed the entire context buffer.
}
ldml_event_state::ldml_event_state(
km_core_state *s,
km_core_virtual_key v,
uint16_t m,
uint8_t i,
uint16_t e) {
this->state = s;
this->vk = v;
this->modifier_state = m;
this->is_key_down = i;
this->event_flags = e;
}
void ldml_event_state::commit() {
state->actions().commit();
}
void ldml_event_state::clear() {
state->actions().clear();
}
} // namespace core
} // namespace km

152
core/src/ldml/ldml_processor.hpp
View file

@ -13,7 +13,7 @@
#include <utility>
#include <vector>
#include <memory>
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
#include "processor.hpp"
#include "option.hpp"
#include "ldml_vkeys.hpp"
@ -24,19 +24,16 @@ namespace core {
#define KM_CORE_LMDL_PROCESSOR_VERSION u"1.0"
class ldml_processor : public abstract_processor {
private:
bool _valid;
std::unique_ptr<ldml::transforms> transforms, bksp_transforms;
ldml::vkeys keys;
class ldml_event_state;
/** our actual processor */
class ldml_processor : public abstract_processor {
public:
ldml_processor(
path const & kb_path,
const std::vector<uint8_t> & data
);
// ~ldml_processor() override;
static bool is_kmxplus_file(
path const & kb_path,
std::vector<uint8_t>& data
@ -84,57 +81,110 @@ namespace core {
km_core_keyboard_imx * get_imx_list() const override;
bool
supports_normalization() const override {
return true;
}
private:
/** emit text to context and actions */
static void emit_text(km_core_state *state, const std::u16string &str);
/** emit text to context and actions */
static void emit_text(km_core_state *state, const std::u32string &str);
/** emit char to context and actions */
static void emit_text(km_core_state *state, km_core_usv ch);
/** emit a marker */
static void emit_marker(km_core_state *state, KMX_DWORD marker);
/** emit a pass-through and invalidate */
static void emit_invalidate_passthrough_keystroke(km_core_state *state, km_core_virtual_key vk, uint16_t modifier_state);
/** process a key-up */
void process_key_up(ldml_event_state &ldml_state) const;
/**
* Delete text from the state, by:
* 1. calling actions().push_backspace() to push the appropriate backspaces
* 2. popping the same items from the context items
* 3. mutating 'str' by removing the same number of items.
* This function handles marker strings correctly.
* @param str string with text to remove, from the end
* @param length number of chars from the end of str to drop
*/
static void remove_text(km_core_state *state, std::u32string &str, size_t length);
/** process a key-down (if it wasn't handled exceptionally) */
void process_key_down(ldml_event_state &ldml_state) const;
/** process a key-up */
void process_key_up(km_core_state *state, km_core_virtual_key vk, uint16_t modifier_state) const;
/** process a typed key */
void process_key_string(ldml_event_state &ldml_state, const std::u16string &key_str) const;
/** process a key-down (if it wasn't handled exceptionally) */
void process_key_down(km_core_state *state, km_core_virtual_key vk, uint16_t modifier_state) const;
/** process a backspace */
void process_backspace(ldml_event_state &ldml_state) const;
/** process a typed key */
void process_key_string(km_core_state *state, const std::u16string &key_str) const;
/**
* common function for outputting a string with transforms/normalization applied.
* @param str string to output (such as from a key), or empty
* @param with_transforms transforms to use or nullptr
* @returns length of matched input context
*/
size_t process_output(ldml_event_state &ldml_state, const std::u32string &str, ldml::transforms *with_transforms) const;
/** process a backspace */
void process_backspace(km_core_state *state) const;
private:
bool _valid;
std::unique_ptr<ldml::transforms> transforms, bksp_transforms;
ldml::vkeys keys;
};
/**
* common function for outputting a string with transforms/normalization applied.
* @param str string to output (such as from a key), or empty
* @param with_transforms transforms to use or nullptr
* @returns length of matched input context
*/
size_t process_output(km_core_state *state, const std::u32string &str, ldml::transforms *with_transforms) const;
/**
* add the string+marker portion of the context to the beginning of str.
* Stop when a non-string and non-marker is hit.
* Convert markers into the UC_SENTINEL format.
* @return the number of context items consumed
*/
static size_t context_to_string(km_core_state *state, std::u32string &str, bool include_markers = true);
/** class holding state as we process an event. mirrors process_event args. */
class ldml_event_state {
public:
ldml_event_state(
km_core_state *state,
km_core_virtual_key vk,
uint16_t modifier_state,
uint8_t is_key_down,
uint16_t event_flags);
/** done with this, copy it into core state */
void commit();
/** clear this object out */
void clear();
// getters
inline km_core_virtual_key get_vk() const;
inline uint16_t get_modifier_state() const;
// actions
/** emit text to context and actions */
void emit_text(const std::u16string &str);
/** emit text to context and actions */
void emit_text(const std::u32string &str);
/** emit char to context and actions */
void emit_text(km_core_usv ch);
/** emit a marker */
void emit_marker(KMX_DWORD marker);
/** emit a pass-through and invalidate */
void emit_invalidate_passthrough_keystroke();
/** emit a backspace (for a user-initiated deletion) */
void emit_backspace();
/**
* Delete text from the state, by:
* 1. calling actions().push_backspace() to push the appropriate backspaces
* 2. popping the same items from the context items
* 3. mutating 'str' by removing the same number of items.
* This function handles marker strings correctly.
* @param str string with text to remove, from the end
* @param length number of chars from the end of str to drop
*/
void remove_text(std::u32string &str, size_t length);
/**
* add the string+marker portion of the context to the beginning of str.
* Stop when a non-string and non-marker is hit.
* Convert markers into the UC_SENTINEL format.
* @return the number of context items consumed
*/
size_t context_to_string(std::u32string &str, bool include_markers = true);
private:
km_core_virtual_key vk;
uint16_t modifier_state;
uint8_t is_key_down;
uint16_t event_flags;
km_core_state *state;
};
// implementation
km_core_virtual_key
ldml_event_state::get_vk() const {
return vk;
}
uint16_t
ldml_event_state::get_modifier_state() const {
return modifier_state;
}
};
} // namespace core
} // namespace km

9
core/src/ldml/ldml_transforms.hpp
View file

@ -16,18 +16,11 @@
#include <utility>
#include "debuglog.h"
#if !defined(HAVE_ICU4C)
#error icu4c is required for this code
#endif
#define U_FALLTHROUGH
#include "unicode/utypes.h"
#include "core_icu.h"
#include "unicode/uniset.h"
#include "unicode/usetiter.h"
#include "unicode/unistr.h"
#include "unicode/regex.h"
#include "unicode/utext.h"
#include "unicode/normalizer2.h"
namespace km {
namespace core {

2
core/src/ldml/ldml_vkeys.hpp
View file

@ -13,7 +13,7 @@
#include <utility>
#include <vector>
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
namespace km {
namespace core {

3
core/src/meson.build
View file

@ -42,7 +42,9 @@ endif
kmx_files = files(
'actions_normalize.cpp',
'action.cpp',
'context_helpers.cpp',
'option.cpp',
'keyboard.cpp',
'state.cpp',
@ -52,6 +54,7 @@ kmx_files = files(
'km_core_keyboard_api.cpp',
'km_core_options_api.cpp',
'km_core_state_api.cpp',
'km_core_state_context_set_if_needed.cpp',
'km_core_debug_api.cpp',
'km_core_processevent_api.cpp',
'jsonpp.cpp',

6
core/src/mock/mock_processor.hpp
View file

@ -10,7 +10,7 @@
#include <string>
#include <unordered_map>
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
#include "processor.hpp"
#include "option.hpp"
@ -65,6 +65,10 @@ namespace core
km_core_keyboard_imx * get_imx_list() const override;
bool
supports_normalization() const override {
return false;
}
};
class null_processor : public mock_processor {

3
core/src/option.hpp
View file

@ -12,7 +12,8 @@
#include <string>
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
// Forward declarations
class json;

3
core/src/path.hpp
View file

@ -12,7 +12,8 @@
#include <string>
#include <type_traits>
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
#include "jsonpp.hpp"
#include "utfcodec.hpp"

5
core/src/processor.hpp
View file

@ -11,7 +11,7 @@
#include <string>
#include <unordered_map>
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
#include "keyboard.hpp"
@ -122,6 +122,9 @@ namespace core
virtual km_core_keyboard_imx *
get_imx_list() const = 0;
virtual bool
supports_normalization() const = 0;
friend json & operator << (json &j, abstract_processor const &opts);
};

6
core/src/state.hpp
View file

@ -11,7 +11,7 @@
#include <cassert>
#include <vector>
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
#include "context.hpp"
#include "option.hpp"
@ -123,6 +123,7 @@ class state
{
protected:
core::context _ctxt;
core::context _app_ctxt;
core::abstract_processor & _processor;
core::actions _actions;
core::debug_items _debug_items;
@ -138,6 +139,9 @@ public:
core::context & context() noexcept { return _ctxt; }
core::context const & context() const noexcept { return _ctxt; }
core::context & app_context() noexcept { return _app_ctxt; }
core::context const & app_context() const noexcept { return _app_ctxt; }
core::abstract_processor const & processor() const noexcept { return _processor; }
core::abstract_processor & processor() noexcept { return _processor; }

2
core/tests/unit/kmnkbd/action_api.cpp
View file

@ -6,7 +6,7 @@
History: 23 Oct 2023 - MCD - Initial implementation.
*/
#include <string>
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
#include "path.hpp"
#include "action.hpp"

2
core/tests/unit/kmnkbd/action_items.hpp
View file

@ -1,4 +1,4 @@
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
#include <string>
#include <iostream>

4
core/tests/unit/kmnkbd/action_set_api.cpp
View file

@ -6,7 +6,8 @@
History: 23 Oct 2023 - MCD - Initial implementation.
*/
#include <string>
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
#include "path.hpp"
#include "action.hpp"
@ -60,6 +61,7 @@ void setup(const char *keyboard, const km_core_cp* context) {
try_status(km_core_state_create(test_kb, test_env_opts, &test_state));
try_status(km_core_context_items_from_utf16(context, &citems));
try_status(km_core_context_set(km_core_state_context(test_state), citems));
try_status(km_core_context_set(km_core_state_app_context(test_state), citems));
}
void run_test(km_core_action_item const * action_items, const km_core_actions &actions) {

3
core/tests/unit/kmnkbd/context_api.cpp
View file

@ -10,7 +10,8 @@
mutation functions.
*/
#include <string>
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
#include "context.hpp"
#include "utfcodec.hpp"

6
core/tests/unit/kmnkbd/debug_api.cpp
View file

@ -10,7 +10,9 @@
#include <algorithm>
#include <iterator>
#include <string>
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
#include "path.hpp"
#include "state.hpp"
#include "kmx/kmx_base.h"
@ -66,6 +68,7 @@ void setup(const char *keyboard) {
}));
try_status(km_core_context_set(km_core_state_context(test_state), citems));
try_status(km_core_context_set(km_core_state_app_context(test_state), citems));
}
/**
@ -423,6 +426,7 @@ void test_backspace_markers() {
{KM_CORE_CT_END}
};
try_status(km_core_context_set(km_core_state_context(test_state), marker_context));
try_status(km_core_context_set(km_core_state_app_context(test_state), marker_context)); // TODO: this needs to remove markers
DEBUG_GROUP gp = {u"Main"};

3
core/tests/unit/kmnkbd/debug_items.hpp
View file

@ -1,6 +1,5 @@
#include <iostream>
#include <keyman/keyman_core_api.h>
#include <keyman/keyman_core_api_debug.h>
#include "keyman_core.h"
#include "kmx/kmx_base.h"
#include "kmx/kmx_xstring.h"

2
core/tests/unit/kmnkbd/keyboard_api.cpp
View file

@ -6,7 +6,7 @@
*/
#include <string>
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
#include "path.hpp"
//#include "keyboard.hpp"

1
core/tests/unit/kmnkbd/meson.build
View file

@ -26,6 +26,7 @@ tests = [
['debug-api', 'debug_api.cpp'],
['kmx_xstring', 'test_kmx_xstring.cpp'],
['kmx_context', 'test_kmx_context.cpp'],
['test_actions_normalize', 'test_actions_normalize.cpp'],
]
test_path = join_paths(meson.current_build_dir(), '..', 'kmx')

2
core/tests/unit/kmnkbd/options_api.cpp
View file

@ -7,7 +7,7 @@
#include <cstdlib>
#include <string>
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
#include "option.hpp"
#include "state.hpp"

2
core/tests/unit/kmnkbd/state_api.cpp
View file

@ -9,7 +9,7 @@
#include <iostream>
#include <string>
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
#include "path.hpp"
#include "state.hpp"

45
core/tests/unit/kmnkbd/state_context_api.cpp
View file

@ -44,6 +44,7 @@ setup(const char *keyboard, const km_core_cp *context) {
try_status(km_core_state_create(test_kb, test_env_opts, &test_state));
try_status(km_core_context_items_from_utf16(context, &citems));
try_status(km_core_context_set(km_core_state_context(test_state), citems));
try_status(km_core_context_set(km_core_state_app_context(test_state), citems));
}
bool
@ -138,6 +139,7 @@ test_context_set_if_needed_cached_context_has_markers() {
{KM_CORE_CT_MARKER, {0}, {3}}, {KM_CORE_CT_MARKER, {0}, {4}}, KM_CORE_CONTEXT_ITEM_END};
try_status(km_core_context_set(km_core_state_context(test_state), citems));
try_status(km_core_context_set(km_core_state_app_context(test_state), citems));
assert(km_core_state_context_set_if_needed(test_state, application_context) == KM_CORE_CONTEXT_STATUS_UNCHANGED);
km_core_context_item *citems_new;
@ -177,6 +179,48 @@ test_context_clear() {
teardown();
}
//-------------------------------------------------------------------------------------
void test_context_debug_empty() {
km_core_cp const *cached_context = u"";
setup("k_000___null_keyboard.kmx", cached_context);
auto str = km_core_state_context_debug(test_state, KM_CORE_DEBUG_CONTEXT_CACHED);
// std::cout << str << std::endl;
assert(std::u16string(str) == u"|| (len: 0) [ ]");
km_core_cp_dispose(str);
}
void test_context_debug_various() {
km_core_cp const *cached_context = u"";
setup("k_000___null_keyboard.kmx", cached_context);
km_core_context_item const citems[] = {
{ KM_CORE_CT_MARKER, {0}, { 5 } },
{ KM_CORE_CT_CHAR, {0}, { '1' } },
{ KM_CORE_CT_MARKER, {0}, { 1 } },
{ KM_CORE_CT_CHAR, {0}, { '2' } },
{ KM_CORE_CT_MARKER, {0}, { 2 } },
{ KM_CORE_CT_CHAR, {0}, { '3' } },
{ KM_CORE_CT_MARKER, {0}, { 3 } },
{ KM_CORE_CT_MARKER, {0}, { 4 } },
{ KM_CORE_CT_CHAR, {0}, { 0x1F923 /* 🤣 */ } },
KM_CORE_CONTEXT_ITEM_END
};
try_status(km_core_context_set(km_core_state_context(test_state), citems));
try_status(km_core_context_set(km_core_state_app_context(test_state), citems));
auto str = km_core_state_context_debug(test_state, KM_CORE_DEBUG_CONTEXT_CACHED);
// std::cout << str << std::endl;
assert(std::u16string(str) == u"|123🤣| (len: 9) [ M(5) U+0031 M(1) U+0032 M(2) U+0033 M(3) M(4) U+1f923 ]");
km_core_cp_dispose(str);
}
void test_context_debug() {
test_context_debug_empty();
test_context_debug_various();
}
//-------------------------------------------------------------------------------------
// Launcher
//-------------------------------------------------------------------------------------
@ -216,4 +260,5 @@ main(int argc, char *argv[]) {
test_context_set_if_needed();
test_context_clear();
test_context_debug();
}

333
core/tests/unit/kmnkbd/test_actions_normalize.cpp Normal file
View file

@ -0,0 +1,333 @@
/*
Copyright: © 2018 SIL International.
Description: Tests for the context API family of functions.
Create Date: 23 Oct 2023
Authors: Marc Durdin
History: 23 Oct 2023 - MCD - Initial implementation.
*/
#include <string>
#include "keyman_core.h"
#include "path.hpp"
#include "action.hpp"
#include "context.hpp"
#include <test_assert.h>
#include "../emscripten_filesystem.h"
km_core_option_item test_env_opts[] =
{
KM_CORE_OPTIONS_END
};
km_core_keyboard * test_kb = nullptr;
km_core_state * test_state = nullptr;
km_core_actions * test_actions = nullptr;
std::string arg_path;
void teardown() {
if(test_state) {
km_core_state_dispose(test_state);
test_state = nullptr;
}
if(test_kb) {
km_core_keyboard_dispose(test_kb);
test_kb = nullptr;
}
if(test_actions) {
delete [] test_actions->output;
delete test_actions;
test_actions = nullptr;
}
}
void setup(const km_core_cp *app_context, const km_core_cp *cached_context, int actions_code_points_to_delete, const std::u32string actions_output) {
teardown();
km::core::path path = km::core::path::join(arg_path, "..", "ldml", "keyboards", "k_001_tiny.kmx");
try_status(km_core_keyboard_load(path.native().c_str(), &test_kb));
try_status(km_core_state_create(test_kb, test_env_opts, &test_state));
try_status(set_context_from_string(km_core_state_context(test_state), cached_context));
try_status(set_context_from_string(km_core_state_app_context(test_state), app_context));
test_actions = new km_core_actions;
test_actions->code_points_to_delete = actions_code_points_to_delete;
test_actions->output = new km_core_usv[actions_output.length() + 1];
actions_output.copy(test_actions->output, actions_output.length());
test_actions->output[actions_output.length()] = 0;
}
//-------------------------------------------------------------------------------------
/**
* Run a single test on actions_normalize. This is quite nuanced, because the
* input state is more than a little complex. We have inputs in NFU and NFD, and
* outputs counting NFU and inserting NFC. Be careful!
*
* @param name descriptive name for the test
* @param initial_app_context the app context stored in the state,
* _before_ transform is applied -- NFU
* @param final_cached_context cached context _after_ actions have been
* applied -- guaranteed NFD (essentially,
* this is initial_cached_context -
* actions_code_points_to_delete +
* actions_output)
* @param actions_code_points_to_delete number of NFD code points that the
* keyboard processor has asked to remove
* in its actions
* @param actions_output NFD string that the keyboard processor
* has asked to insert in its actions
* @param expected_delete expected: NFU code points to ask app to
* remove
* @param expected_output expected: adjusted NFC output to insert
* into the app
* @param expected_final_app_context expected: NFU adjusted final app
* context, which will be NFC from the
* boundary of the transform, but will not
* have been modified prior to that. Should
* match char-for-char what the app ends up
* with in its text buffer.
*/
void test(
const char *name,
const km_core_cp *initial_app_context,
const km_core_cp *final_cached_context,
int actions_code_points_to_delete,
const std::u32string actions_output,
const unsigned int expected_delete,
const std::u32string expected_output,
const km_core_cp *expected_final_app_context
) {
std::cout << "test: " << name << std::endl;
setup(initial_app_context, final_cached_context, actions_code_points_to_delete, actions_output);
assert(km::core::actions_normalize(km_core_state_context(test_state), km_core_state_app_context(test_state), test_actions));
std::cout << " (" << name << "): delete: " << expected_delete << " output: |" << std::u32string(test_actions->output) << "|" << std::endl;
std::u32string o(test_actions->output);
for(auto i = o.begin(); i < o.end(); i++) {
std::cout << "U+" << std::hex << (int)(*i) << " ";
}
std::cout << std::endl;
assert(expected_delete == test_actions->code_points_to_delete);
assert(expected_output == test_actions->output);
auto actual_final_app_context = get_context_as_string(km_core_state_app_context(test_state));
auto actual_final_app_context_string = std::u16string(actual_final_app_context);
auto expected_final_app_context_string = std::u16string(expected_final_app_context);
std::cout << " final app context: actual: |" << actual_final_app_context_string << "| expected: |" << expected_final_app_context_string << "|" << std::endl;
assert(actual_final_app_context_string == expected_final_app_context_string);
delete [] actual_final_app_context;
teardown();
}
void run_tests() {
// Null boundary tests
test(
"noop",
/* app context pre transform: */ u"",
/* cached context post transform: */ u"",
/* action del, output: */ 0, U"",
// ---- results ----
/* action del, output: */ 0, U"",
/* app_context: */ u""
);
test(
"no_output",
/* app context pre transform: */ u"abc",
/* cached context post transform: */ u"abc",
/* action del, output: */ 0, U"",
// ---- results ----
/* action del, output: */ 0, U"",
/* app_context: */ u"abc"
);
test(
"no_context",
/* app context pre transform: */ u"",
/* cached context post transform: */ u"def",
/* action del, output: */ 0, U"def",
// ---- results ----
/* action del, output: */ 0, U"def",
/* app_context: */ u"def"
);
// Simple tests -- no deletions involved
test(
"no_normalization",
/* app context pre transform: */ u"abc",
/* cached context post transform: */ u"abcdef",
/* action del, output: */ 0, U"def",
// ---- results ----
/* action del, output: */ 0, U"def",
/* app_context: */ u"abcdef"
);
test(
"output_to_nfc_basic",
/* app context pre transform: */ u"abc",
/* cached context post transform: */ u"abcde\u0300f",
/* action del, output: */ 0, U"de\u0300f",
// ---- results ----
/* action del, output: */ 0, U"dèf",
/* app_context: */ u"abcdèf"
);
test(
"output_to_nfc_hefty",
/* app context pre transform: */ u"abc",
/* cached context post transform: */ u"abcA\u0300" u"e\u0316\u0301" u"\u0073\u0323\u0307" u"\u0041\u030a" u"\U000114B9\U000114B0",
/* action del, output: */ 0, U"A\u0300" U"e\u0316\u0301" U"\u0073\u0323\u0307" U"\u0041\u030a" U"\U000114B9\U000114B0",
// ---- results ----
/* action del, output: */ 0, U"À" U"é̖" U"\u1e69" U"\u00c5" U"\U000114BC",
/* app_context: */ u"abcÀé̖\u1e69\u00c5\U000114BC"
);
// Interaction with input context when not on normalization boundary
test(
"Backtrack one character to combine as NFC",
/* app context pre transform: */ u"XYZA",
/* cached context post transform: */ u"XYZA\u0300abc",
/* action del, output: */ 0, U"\u0300abc",
// ---- results ----
/* action del, output: */ 1, U"Àabc",
/* app_context: */ u"XYZÀabc"
);
test(
"Backtrack e+comb circ (2 chars) to combine as NFC",
/* app context pre transform: */ u"abce\u0302",
/* cached context post transform: */ u"abce\u0323\u0302",
/* action del, output: */ 1, U"\u0323\u0302",
// ---- results ----
/* action del, output: */ 2, U"ệ",
/* app_context: */ u"abcệ"
);
test(
"One backspace for NFD converts into one char in NFC (ê) and recombine",
/* app context pre transform: */ u"abcê",
/* cached context post transform: */ u"abce\u0323\u0302",
/* action del, output: */ 1, U"\u0323\u0302", // NFD input; delete 1: \u0302
// ---- results ----
/* action del, output: */ 1, U"ệ", // NFC output; delete 1: ê
/* app_context: */ u"abcệ"
);
// a\u0300 should not be normalized because it is not otherwise impacted by
// the action.
test(
"Avoid editing too far back in context when finding normalization boundary",
/* app context pre transform: */ u"a\u0300bcê",
/* cached context post transform: */ u"a\u0300bce\u0323\u0302",
/* action del, output: */ 1, U"\u0323\u0302", // NFD input; delete 1: \u0302
// ---- results ----
/* action del, output: */ 1, U"ệ", // NFC output; delete 1: ê
/* app_context: */ u"a\u0300bcệ"
);
// If we don't reach a normalization boundary, we still should continue to work
test(
"Normalizable letters at start of context",
/* app context pre transform: */ u"\u0300",
/* cached context post transform: */ u"\u0323\u0300\u0302",
/* action del, output: */ 1, U"\u0323\u0300\u0302", // NFD input;
// ---- results ----
/* action del, output: */ 1, U"\u0323\u0300\u0302", // NFC output is still decomposed because there is no base
/* app_context: */ u"\u0323\u0300\u0302"
);
// Modifies the base as well as diacritic
test(
"Two backspaces for NFD converts into one char in NFC (ê) and recombine",
/* app context pre transform: */ u"abcê",
/* cached context post transform: */ u"abca\u0323\u0302",
/* action del, output: */ 2, U"a\u0323\u0302", // NFD input; delete 2: e\u0302
// ---- results ----
/* action del, output: */ 1, U"ậ", // NFC output; delete 1: ê
/* app_context: */ u"abcậ"
);
// surrogate pair tests
test(
"Surrogate pair in context",
/* app context pre transform: */ u"abc\U0001F607ê",
/* cached context post transform: */ u"abc\U0001F607a\u0323\u0302",
/* action del, output: */ 2, U"a\u0323\u0302",
// ---- results ----
/* action del, output: */ 1, U"ậ",
/* app_context: */ u"abc\U0001F607ậ"
);
test(
"Surrogate pair in output",
/* app context pre transform: */ u"abc",
/* cached context post transform: */ u"abc\U0001F607",
/* action del, output: */ 0, U"\U0001F607",
// ---- results ----
/* action del, output: */ 0, U"\U0001F607",
/* app_context: */ u"abc\U0001F607"
);
test(
"Surrogate pairs in both context and output",
/* app context pre transform: */ u"a\U0001F607bcê",
/* cached context post transform: */ u"a\U0001F607bca\U0001F60E",
/* action del, output: */ 2, U"a\U0001F60E",
// ---- results ----
/* action del, output: */ 1, U"a\U0001F60E",
/* app_context: */ u"a\U0001F607bca\U0001F60E"
);
}
//-------------------------------------------------------------------------------------
// Launcher
//-------------------------------------------------------------------------------------
constexpr const auto help_str = "\
test_actions_normalize [--color] <BUILD_PATH>\n\
\n\
--color Force color output\n\
BUILD_PATH Path where test_actions_normalize.exe is found; kmx files are\n\
located relative to this path.\n";
int error_args() {
std::cerr << "test_actions_normalize: Invalid arguments." << std::endl;
std::cout << help_str;
return 1;
}
int main(int argc, char *argv []) {
if(argc < 2) {
return error_args();
}
auto arg_color = std::string(argv[1]) == "--color";
if(arg_color && argc < 3) {
return error_args();
}
console_color::enabled = console_color::isaterminal() || arg_color;
#ifdef __EMSCRIPTEN__
arg_path = get_wasm_file_path(argv[arg_color ? 2 : 1]);
#else
arg_path = argv[arg_color ? 2 : 1];
#endif
run_tests();
}

2
core/tests/unit/kmx/kmx.cpp
View file

@ -17,6 +17,8 @@
#include <string>
#include <type_traits>
#include "keyman_core.h"
#include <kmx/kmx_processevent.h>
#include <kmx/kmx_xstring.h>

2
core/tests/unit/ldml/ldml.cpp
View file

@ -24,6 +24,8 @@
#include <test_assert.h>
#include <test_color.h>
#include "keyman_core.h"
#include <kmx/kmx_xstring.h> // for surrogate pair macros
#include "ldml_test_source.hpp"

8
core/tests/unit/ldml/ldml_test_source.cpp
View file

@ -33,15 +33,9 @@
#include "ldml_test_source.hpp"
#include "ldml_test_utils.hpp"
#if defined(HAVE_ICU4C)
// TODO-LDML: Needed this for some compiler warnings
#define U_FALLTHROUGH
#include "unicode/utypes.h"
#include "core_icu.h"
#include "unicode/uniset.h"
#include "unicode/usetiter.h"
#else
#error icu4c is required for this test
#endif
#define assert_or_return(expr) if(!(expr)) { \
std::wcerr << __FILE__ << ":" << __LINE__ << ": " << \

3
core/tests/unit/ldml/ldml_test_utils.hpp
View file

@ -13,7 +13,8 @@
#include <kmx/kmx_processevent.h> // for char to vk mapping tables
#include <kmx/kmx_xstring.h> // for surrogate pair macros
#include <kmx/kmx_plus.h>
#include <keyman/keyman_core_api.h>
#include "keyman_core.h"
namespace km {
namespace tests {

17
core/tests/unit/ldml/meson.build
View file

@ -83,6 +83,23 @@ t = executable('test_transforms', 'test_transforms.cpp',
objects: lib.extract_all_objects(recursive: false))
test('test_transforms', t, suite: 'ldml')
# run test_context_normalization ldml unit test
normalization_tests_flags = tests_flags
if cpp_compiler.get_id() == 'emscripten'
normalization_tests_flags += ['-lnodefs.js', '-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\']']
endif
t = executable('test_context_normalization',
['test_context_normalization.cpp', '../emscripten_filesystem.cpp'],
cpp_args: defns + warns,
include_directories: [inc, libsrc, '../../../../developer/src/ext/json'],
link_args: links + normalization_tests_flags,
dependencies: [icu_uc, icu_i18n],
objects: lib.extract_all_objects(recursive: false))
test('test_context_normalization', t, suite: 'ldml')
# Run tests on all keyboards (`tests` defined in keyboards/meson.build)
foreach kbd : tests

162
core/tests/unit/ldml/test_context_normalization.cpp Normal file
View file

@ -0,0 +1,162 @@
/*
Copyright: © 2018 SIL International.
Description: Tests for normalization in the context API.
Create Date: 15 Jan 2024
Authors: Marc Durdin
History: 15 Jan 2024 - MCD - Initial implementation.
*/
#include <string>
#include "keyman_core.h"
#include "path.hpp"
#include "action.hpp"
#include <test_assert.h>
#include "../emscripten_filesystem.h"
//-------------------------------------------------------------------------------------
// Context normalization tests
//-------------------------------------------------------------------------------------
km_core_option_item test_env_opts[] =
{
KM_CORE_OPTIONS_END
};
km_core_keyboard * test_kb = nullptr;
km_core_state * test_state = nullptr;
std::string arg_path;
void teardown() {
if(test_state) {
km_core_state_dispose(test_state);
test_state = nullptr;
}
if(test_kb) {
km_core_keyboard_dispose(test_kb);
test_kb = nullptr;
}
}
void setup(const char *keyboard) {
teardown();
km::core::path path = km::core::path::join(arg_path, "keyboards", keyboard);
try_status(km_core_keyboard_load(path.native().c_str(), &test_kb));
try_status(km_core_state_create(test_kb, test_env_opts, &test_state));
}
void debug_context(km_core_debug_context_type context_type) {
auto context = km_core_state_context_debug(test_state, context_type);
if(context_type == KM_CORE_DEBUG_CONTEXT_APP) {
std::cout << "app context: " << context << std::endl;
} else {
std::cout << "cached context: " << context << std::endl;
}
km_core_cp_dispose(context);
}
bool is_identical_context(km_core_cp const *cached_context, km_core_debug_context_type context_type) {
size_t buf_size;
km_core_context_item * citems = nullptr;
debug_context(context_type);
if(context_type == KM_CORE_DEBUG_CONTEXT_APP) {
try_status(km_core_context_get(km_core_state_app_context(test_state), &citems));
} else {
try_status(km_core_context_get(km_core_state_context(test_state), &citems));
}
try_status(km_core_context_items_to_utf16(citems, nullptr, &buf_size));
km_core_cp* new_cached_context = new km_core_cp[buf_size];
try_status(km_core_context_items_to_utf16(citems, new_cached_context, &buf_size));
km_core_context_items_dispose(citems);
bool result = std::u16string(cached_context) == new_cached_context;
delete[] new_cached_context;
return result;
}
void test_context_normalization_already_nfd() {
km_core_cp const *app_context_nfd = u"A\u0300";
setup("k_001_tiny.kmx");
assert(km_core_state_context_set_if_needed(test_state, app_context_nfd) == KM_CORE_CONTEXT_STATUS_UPDATED);
assert(is_identical_context(app_context_nfd, KM_CORE_DEBUG_CONTEXT_APP));
assert(is_identical_context(app_context_nfd, KM_CORE_DEBUG_CONTEXT_CACHED));
teardown();
}
void test_context_normalization_basic() {
km_core_cp const *application_context = u"This is a test À";
km_core_cp const *cached_context = u"This is a test A\u0300";
setup("k_001_tiny.kmx");
assert(km_core_state_context_set_if_needed(test_state, application_context) == KM_CORE_CONTEXT_STATUS_UPDATED);
assert(is_identical_context(application_context, KM_CORE_DEBUG_CONTEXT_APP));
assert(is_identical_context(cached_context, KM_CORE_DEBUG_CONTEXT_CACHED));
teardown();
}
void test_context_normalization_hefty() {
// Latin Latin "ṩ" "Å" Tirhuta U+114bc -> U+114B9 U+114B0
km_core_cp const *application_context = u"À" u"é̖" u"\u1e69" u"\u212b" u"\U000114BC";
km_core_cp const *cached_context = u"A\u0300" u"e\u0316\u0301" u"\u0073\u0323\u0307" u"\u0041\u030a" u"\U000114B9\U000114B0";
setup("k_001_tiny.kmx");
assert(km_core_state_context_set_if_needed(test_state, application_context) == KM_CORE_CONTEXT_STATUS_UPDATED);
assert(is_identical_context(application_context, KM_CORE_DEBUG_CONTEXT_APP));
assert(is_identical_context(cached_context, KM_CORE_DEBUG_CONTEXT_CACHED));
teardown();
}
void test_context_normalization_invalid_unicode() {
// unpaired surrogate illegal
km_core_cp const application_context[] = { 0xDC01, 0x0020, 0x0020, 0xFFFF, 0x0000 };
km_core_cp const cached_context[] = { 0xDC01, 0x0020, 0x0020, 0xFFFF, 0x0000 };
setup("k_001_tiny.kmx");
assert(km_core_state_context_set_if_needed(test_state, application_context) == KM_CORE_CONTEXT_STATUS_UPDATED);
assert(is_identical_context(application_context, KM_CORE_DEBUG_CONTEXT_APP));
assert(is_identical_context(cached_context, KM_CORE_DEBUG_CONTEXT_CACHED));
teardown();
}
void test_context_normalization() {
test_context_normalization_already_nfd();
test_context_normalization_basic();
test_context_normalization_hefty();
// TODO: we need to strip illegal chars: test_context_normalization_invalid_unicode(); // -- unpaired surrogate, illegals
}
//-------------------------------------------------------------------------------------
// Launcher
//-------------------------------------------------------------------------------------
constexpr const auto help_str = "\
context_normalization [--color]\n\
\n\
--color Force color output\n";
int error_args() {
std::cerr << "context_normalization: Invalid arguments." << std::endl;
std::cout << help_str;
return 1;
}
int main(int argc, char *argv []) {
auto arg_color = argc > 1 && std::string(argv[1]) == "--color";
console_color::enabled = console_color::isaterminal() || arg_color;
// Get the path of the current executable
arg_path = argv[0];
auto last = arg_path.find_last_of("/\\");
if(last == std::string::npos) {
std::cerr << "could not parse argv[0]: " << argv[0] << std::endl;
return 1;
}
arg_path.resize(last+1);
#ifdef __EMSCRIPTEN__
arg_path = get_wasm_file_path(arg_path);
#endif
test_context_normalization();
}

3
linux/debian/libkeymancore.symbols
View file

@ -16,6 +16,7 @@ libkeymancore.so.1 libkeymancore #MINVER#
km_core_context_length@Base 17.0.195
km_core_context_set@Base 17.0.195
km_core_context_shrink@Base 17.0.195
km_core_cp_dispose@Base 17.0.244
km_core_event@Base 17.0.195
km_core_get_engine_attrs@Base 17.0.195
km_core_keyboard_dispose@Base 17.0.195
@ -29,9 +30,11 @@ libkeymancore.so.1 libkeymancore #MINVER#
km_core_process_event@Base 17.0.195
km_core_process_queued_actions@Base 17.0.195
km_core_state_action_items@Base 17.0.195
km_core_state_app_context@Base 17.0.245
km_core_state_clone@Base 17.0.195
km_core_state_context@Base 17.0.195
km_core_state_context_clear@Base 17.0.197
km_core_state_context_debug@Base 17.0.244
km_core_state_context_set_if_needed@Base 17.0.197
km_core_state_create@Base 17.0.195
km_core_state_debug_get@Base 17.0.195

36
linux/ibus-keyman/src/engine.c
View file

@ -205,27 +205,17 @@ ibus_keyman_engine_class_init (IBusKeymanEngineClass *klass)
}
static gchar *
get_context_debug(IBusEngine *engine, km_core_context *context) {
get_context_debug(IBusEngine *engine) {
IBusKeymanEngine *keyman = (IBusKeymanEngine *)engine;
km_core_context *state_context = context ? context : km_core_state_context(keyman->state);
size_t buf_size8 = 512, buf_size16 = 512;
km_core_context_item *context_items;
gchar *current_context_utf8 = g_new0(gchar, buf_size8);
km_core_cp *current_context_utf16 = g_new0(km_core_cp, buf_size16);
if (km_core_context_get(state_context, &context_items) == KM_CORE_STATUS_OK) {
km_core_context_items_to_utf8(context_items, current_context_utf8, &buf_size8);
km_core_context_items_to_utf16(context_items, current_context_utf16, &buf_size16);
km_core_cp *buf = km_core_state_context_debug(keyman->state, KM_CORE_DEBUG_CONTEXT_CACHED);
gchar *result = g_utf16_to_utf8((gunichar2 *)buf, -1, NULL, NULL, NULL);
km_core_cp_dispose(buf);
if(result) {
return result;
}
GString *output = g_string_new("");
g_string_append_printf(output, "|%s| (len:%zu) [", current_context_utf8, km_core_context_length(state_context));
for (int i = 0; i < buf_size16 - 1; i++) {
g_string_append_printf(output, "U+%04x ", current_context_utf16[i]);
}
g_string_append(output, "]");
km_core_context_items_dispose(context_items);
g_free(current_context_utf16);
g_free(current_context_utf8);
GString *output = g_string_new("Error converting debug context");
#if GLIB_CHECK_VERSION(2, 76, 0)
return g_string_free_and_steal(output);
#else
@ -264,7 +254,7 @@ set_context_if_needed(IBusEngine *engine) {
guint cursor_pos, anchor_pos, context_start, context_end;
g_autofree gchar *debug_context = NULL;
g_message("%s: current core context : %s", __FUNCTION__, debug_context = get_context_debug(engine, NULL));
g_message("%s: current core context : %s", __FUNCTION__, debug_context = get_context_debug(engine));
ibus_engine_get_surrounding_text(engine, &text, &cursor_pos, &anchor_pos);
@ -934,10 +924,10 @@ ibus_keyman_engine_process_key_event(
km_mod_state |= KM_CORE_MODIFIER_CAPS;
}
g_message("DAR: %s - km_mod_state=0x%x", __FUNCTION__, km_mod_state);
g_autofree gchar *debug_context = NULL;
g_message("%s: before process key event: %s", __FUNCTION__, debug_context = get_context_debug(engine, NULL));
g_autofree gchar *debug_context0 = NULL, *debug_context1 = NULL, *debug_context2 = NULL;
g_message("%s: before process key event: %s", __FUNCTION__, debug_context0 = get_context_debug(engine));
km_core_process_event(keyman->state, keycode_to_vk[keycode], km_mod_state, isKeyDown, KM_CORE_EVENT_FLAG_DEFAULT);
g_message("%s: after process key event : %s", __FUNCTION__, debug_context = get_context_debug(engine, NULL));
g_message("%s: after process key event : %s", __FUNCTION__, debug_context1 = get_context_debug(engine));
// km_core_state_action_items to get action items
size_t num_action_items;
@ -953,7 +943,7 @@ ibus_keyman_engine_process_key_event(
return FALSE;
}
g_message("%s: after processing all actions: %s", __FUNCTION__, debug_context = get_context_debug(engine, NULL));
g_message("%s: after processing all actions: %s", __FUNCTION__, debug_context2 = get_context_debug(engine));
return TRUE;
}

6
mac/Keyman4MacIM/KeymanTests/TestAppDelegate.m
View file

@ -18,7 +18,7 @@
if (_kme == nil) {
_kme = [[KMEngine alloc] initWithKMX:nil context:self.contextBuffer verboseLogging:self.debugMode];
}
return _kme;
}
@ -31,7 +31,7 @@
if (_contextBuffer == nil) {
_contextBuffer = [[NSMutableString alloc] initWithString:@""];
}
return _contextBuffer;
}
@ -39,7 +39,7 @@
_contextBuffer = [contextBuffer mutableCopy];
if (_contextBuffer.length)
[_contextBuffer replaceOccurrencesOfString:@"\0" withString:[NSString nullChar] options:0 range:NSMakeRange(0, 1)];
[self.kme setCoreContext:self.contextBuffer];
[self.kme setCoreContextIfNeeded:self.contextBuffer];
}
- (BOOL)debugMode {

7
mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.h
View file

@ -1,9 +1,9 @@
/**
* Keyman is copyright (C) SIL International. MIT License.
*
*
* CoreWrapper.h
* Keyman
*
*
* Created by Shawn Schantz on 2022-12-12.
*/
@ -28,8 +28,7 @@ NS_ASSUME_NONNULL_BEGIN
withModifiers:(NSEventModifierFlags)modifierState
withKeyDown:(BOOL)isKeyDown;
-(void)setContextIfNeeded:(NSString*)context;
-(void)setContext:(NSString*)context;
-(NSString*)context;
-(NSString*)contextDebug;
-(void)clearCoreContext;
-(void)dealloc;

145
mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m
View file

@ -225,124 +225,6 @@ const int CORE_ENVIRONMENT_ARRAY_LENGTH = 6;
return capsLock;
}
-(NSArray*)loadActionsUsingCore {
size_t actionCount = 0;
km_core_action_item const * actionList =
km_core_state_action_items(self.coreState, &actionCount);
NSMutableArray *eventArray = [NSMutableArray arrayWithCapacity:actionCount];
for (int i = 0; i < actionCount; i++) {
km_core_action_item action = actionList[i];
CoreAction *coreAction = [self createCoreActionForActionStruct:&action];
[eventArray insertObject:coreAction atIndex:i];
}
return eventArray;
}
-(CoreAction*)createCoreActionForActionStruct:(km_core_action_item*)actionStruct {
CoreAction* action = nil;
switch (actionStruct->type)
{
case KM_CORE_IT_END: {
action = [[CoreAction alloc] initWithType: EndAction actionContent:@"" backspaceCount:0 key:@"" value:@"" scope:0];
break;
}
case KM_CORE_IT_CHAR: {
NSString *characterString = [self.coreHelper utf32ValueToString:actionStruct->character];
action = [[CoreAction alloc] initWithType: CharacterAction actionContent:characterString backspaceCount:0 key:@"" value:@"" scope:0];
[self.coreHelper logDebugMessage:@"createCoreActionForActionStruct actionStruct->character decimal: %u, hex: %X", actionStruct->character, actionStruct->character];
[self.coreHelper logDebugMessage:@"createCoreActionForActionStruct converted unicode string: '%@' length=%lu", characterString, characterString.length];
break;
}
case KM_CORE_IT_MARKER: {
action = [[CoreAction alloc] initWithType: MarkerAction actionContent:@"" backspaceCount:0 key:@"" value:@"" scope:0];
break;
}
case KM_CORE_IT_ALERT: {
action = [[CoreAction alloc] initWithType: AlertAction actionContent:@"" backspaceCount:0 key:@"" value:@"" scope:0];
break;
}
case KM_CORE_IT_BACK: {
km_core_backspace_item backspace = actionStruct->backspace;
if (backspace.expected_type == KM_CORE_BT_CHAR) {
NSString *charString = [self.coreHelper utf32ValueToString:backspace.expected_value];
[self.coreHelper logDebugMessage:@"createCoreActionForActionStruct charString = %@", charString];
action = [[CoreAction alloc] initCharacterBackspaceAction:charString];
[self.coreHelper logDebugMessage:@"createCoreActionForActionStruct converted character backspace, expected value =%lu, expected type =%u", backspace.expected_value, backspace.expected_type];
} else if(backspace.expected_type == KM_CORE_BT_MARKER) {
action = [[CoreAction alloc] initMarkerBackspaceAction:actionStruct->backspace.expected_value];
[self.coreHelper logDebugMessage:@"createCoreActionForActionStruct converted marker backspace, expected value =%lu, expected type =%u", backspace.expected_value, backspace.expected_type];
} else {
[self.coreHelper logDebugMessage:@"createCoreActionForActionStruct did not convert unknown backspace, expected value =%lu, expected type =%u", backspace.expected_value, backspace.expected_type];
}
break;
}
case KM_CORE_IT_PERSIST_OPT: {
[self.coreHelper logDebugMessage:@"***createCoreActionForActionStruct Persist Options encountered."];
km_core_option_item const * option = actionStruct->option;
NSString *keyString = [self.coreHelper createNSStringFromUnicharString:option->key];
NSString *valueString = [self.coreHelper createNSStringFromUnicharString:option->value];
[self.coreHelper logDebugMessage:@"***createCoreActionForActionStruct converted Persist Options, key = %@, value = %@, scope = %d", keyString, valueString, option->scope];
action = [[CoreAction alloc] initPersistOptionAction:keyString value:valueString scope:option->scope];
break;
}
case KM_CORE_IT_EMIT_KEYSTROKE: {
action = [[CoreAction alloc] initWithType: EmitKeystrokeAction actionContent:@"" backspaceCount:0 key:@"" value:@"" scope:0];
break;
}
case KM_CORE_IT_INVALIDATE_CONTEXT: {
action = [[CoreAction alloc] initWithType: InvalidateContextAction actionContent:@"" backspaceCount:0 key:@"" value:@"" scope:0];
break;
}
case KM_CORE_IT_CAPSLOCK: {
action = [[CoreAction alloc] initWithType: CapsLockAction actionContent:@"" backspaceCount:0 key:@"" value:@"" scope:0];
break;
}
default: {
NSLog(@"createCoreActionForActionStruct unrecognized type of km_core_action_item = %u\n", actionStruct->type);
}
}
return action;
}
-(NSString *)getContextAsStringUsingCore {
km_core_context * context = km_core_state_context(self.coreState);
km_core_context_item * contextItemsArray = nil;
size_t contextLength = km_core_context_length(context);
NSMutableString *contextString = [[NSMutableString alloc]init];
if (contextLength==0) {
[self.coreHelper logDebugMessage:@"CoreWrapper getContextAsStringUsingCore, context is empty."];
} else {
km_core_status result = km_core_context_get(context, &contextItemsArray);
if (result==KM_CORE_STATUS_OK) {
for (int i = 0; i < contextLength; i++) {
km_core_context_item contextItem = contextItemsArray[i];
if (contextItem.type == KM_CORE_CT_CHAR) {
NSString *unicodeString = [self.coreHelper utf32ValueToString:contextItem.character];
[contextString appendString:unicodeString];
}
}
}
}
NSString *immutableString = [NSString stringWithString:contextString];
// dispose of context items array
if (contextItemsArray) {
km_core_context_items_dispose(contextItemsArray);
}
[self.coreHelper logDebugMessage:@"CoreWrapper getContextAsStringUsingCore = %@", immutableString];
return immutableString;
}
-(void)clearContextUsingCore {
km_core_state_context_clear(self.coreState);
[self.coreHelper logDebugMessage:@"km_core_state_context_clear called"];
@ -354,28 +236,13 @@ const int CORE_ENVIRONMENT_ARRAY_LENGTH = 6;
[self.coreHelper logDebugMessage:@"CoreWrapper setContextIfNeeded, context=%@, km_core_state_context_set_if_needed result=%i", context, result];
}
-(void)setContext:(NSString*)context {
if (context.length == 0) {
[self clearContextUsingCore];
} else {
char const *coreString = [context cStringUsingEncoding:NSUTF8StringEncoding];
km_core_context_item *contextItemArray;
-(NSString*)contextDebug {
km_core_cp * context = km_core_state_context_debug(self.coreState, KM_CORE_DEBUG_CONTEXT_CACHED);
NSString *debugString = [self.coreHelper createNSStringFromUnicharString:context];
km_core_cp_dispose(context);
// create array of context items
km_core_status result = km_core_context_items_from_utf8(coreString, &contextItemArray);
[self.coreHelper logDebugMessage:@"km_core_context_items_from_utf8, result=%i", result];
// set the context in core using the array
km_core_context * coreContext = km_core_state_context(self.coreState);
km_core_context_set(coreContext, contextItemArray);
// dispose
km_core_context_items_dispose(contextItemArray);
}
}
-(NSString*)context {
return [self getContextAsStringUsingCore];
[self.coreHelper logDebugMessage:@"CoreWrapper contextDebug = %@", debugString];
return debugString;
}
//TODO: create and save as static

3
mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.h
View file

@ -21,10 +21,9 @@
@property (assign, nonatomic) BOOL debugMode;
- (id)initWithKMX:(KMXFile *)kmx context:(NSString *)ctxBuf verboseLogging:(BOOL)enableDebugLogging;
- (NSString *)getCoreContext;
- (NSString *)getCoreContextDebug;
- (void)clearCoreContext;
- (void)setCoreContextIfNeeded:(NSString *)context;
- (void)setCoreContext:(NSString *)context;
- (void)setCoreOptions:(NSString *)key withValue:(NSString *)value;
- (CoreKeyOutput *)processEvent:(NSEvent *)event;

14
mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m
View file

@ -36,10 +36,10 @@ const NSString* kEasterEggKmxName = @"EnglishSpanish.kmx";
self.debugMode = enableDebugLogging;
_kmx = kmx;
_coreHelper = [[CoreHelper alloc] initWithDebugMode:enableDebugLogging];
if (kmx) {
[self loadCoreWrapperFromKmxFile:self.kmx.filePath];
[self.coreWrapper setContext:contextString];
[self.coreWrapper setContextIfNeeded:contextString];
}
}
@ -71,7 +71,7 @@ const NSString* kEasterEggKmxName = @"EnglishSpanish.kmx";
if (self.coreHelper) {
self.coreHelper.debugMode = useVerboseLogging;
}
if (useVerboseLogging) {
NSLog(@"KMEngine - Turning verbose logging on");
// In Keyman Engine if "debugMode" is turned on (explicitly) with "English plus Spanish" as the current keyboard and you type "Sentrycrash#KME",
@ -89,8 +89,8 @@ const NSString* kEasterEggKmxName = @"EnglishSpanish.kmx";
NSLog(@"KMEngine - Turning verbose logging off");
}
- (NSString *)getCoreContext {
return self.coreWrapper.context;
- (NSString *)getCoreContextDebug {
return self.coreWrapper.contextDebug;
}
- (void)clearCoreContext {
@ -101,10 +101,6 @@ const NSString* kEasterEggKmxName = @"EnglishSpanish.kmx";
[self.coreWrapper setContextIfNeeded:context];
}
- (void)setCoreContext:(NSString *)context {
[self.coreWrapper setContext:context];
}
- (void)setCoreOptions:(NSString *)key withValue:(NSString *)value {
BOOL success = [self.coreWrapper setOptionsForCore:key value:value];
[self.coreHelper logDebugMessage:@"setCoreOptions for key: %@, value: %@ succeeded = %@", key, value, success ? @"YES" : @"NO"];

18
mac/KeymanEngine4Mac/KeymanEngine4MacTests/CoreWrapperTests.m
View file

@ -1,11 +1,11 @@
/**
* Keyman is copyright (C) SIL International. MIT License.
*
*
* CoreWrapperTest.m
* CoreWrapperTests
*
*
* Created by Shawn Schantz on 2023-02-17.
*
*
* Description...
*/
@ -29,7 +29,7 @@ CoreWrapper *mockWrapper;
NSString *khmerKeyboardPath = [[[NSBundle mainBundle] resourcePath] stringByAppendingPathComponent:@"khmer_angkor.kmx"];
NSLog(@"mockKmxFilePath = %@\n", mockKmxFilePath);
mockWrapper = [[CoreWrapper alloc] initWithHelper: [CoreTestStaticHelperMethods helper] kmxFilePath:mockKmxFilePath];
}
@ -57,7 +57,7 @@ CoreWrapper *mockWrapper;
- (void)testprocessEvent_lowercaseA_returnsExpectedCharacterForKmx {
NSString *kmxPath = [CoreTestStaticHelperMethods getKmxFilePathTestMacEngine];
CoreWrapper *core = [[CoreWrapper alloc] initWithHelper: [CoreTestStaticHelperMethods helper] kmxFilePath:kmxPath];
// expecting character with 'Ç'
CoreKeyOutput *coreOutput = [core processMacVirtualKey:MVK_A withModifiers:0 withKeyDown:YES];
XCTAssert([coreOutput.textToInsert isEqualToString:@"\u00C7"], @"Expected capital C cedille (U+00C7)");
@ -66,9 +66,11 @@ CoreWrapper *mockWrapper;
- (void)testgetContextAsString_ContextContainsEmojis_ReturnsSameContext {
NSString *kmxPath = [CoreTestStaticHelperMethods getKmxFilePathTestMacEngine];
CoreWrapper *core = [[CoreWrapper alloc] initWithHelper: [CoreTestStaticHelperMethods helper] kmxFilePath:kmxPath];
[core setContext:@"🤔?👍🏻✅"];
NSString *finalContext = core.context;
XCTAssert([finalContext isEqualToString:@"🤔?👍🏻✅"], @"Expected '🤔?👍🏻✅' in context buffer");
[core setContextIfNeeded:@"🤔?👍🏻✅"];
NSString *finalContext = core.contextDebug;
// Note: relying on km_core_state_context_debug output format is just barely
// acceptable for a unit test
XCTAssert([finalContext isEqualToString:@"|🤔?👍🏻✅| (len: 5) [ U+1f914 U+003f U+1f44d U+1f3fb U+2705 ]"], @"Expected '🤔?👍🏻✅' in context buffer");
}
@end

48
mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMEngineTests.m
View file

@ -41,21 +41,27 @@ NSString * names[nCombinations];
KMXFile *kmxFile = [KeymanEngineTestsStaticHelperMethods getKmxFileTestMacEngine];
KMEngine *engine = [[KMEngine alloc] initWithKMX:kmxFile context:@"" verboseLogging:YES];
XCTAssert(engine != nil, @"Expected non-nil engine");
XCTAssert(engine.getCoreContext.length == 0, @"Expected empty context buffer");
// Note: relying on km_core_state_context_debug output format is just barely
// acceptable for a unit test
XCTAssert([engine.getCoreContextDebug isEqualToString:@"|| (len: 0) [ ]"], @"Expected empty context buffer");
}
- (void)testinitWithKMX_ValidKmxNonEmptyContext_InitializedWithContext {
KMXFile *kmxFile = [KeymanEngineTestsStaticHelperMethods getKmxFileTestMacEngine];
KMEngine *engine = [[KMEngine alloc] initWithKMX:kmxFile context:@"abc" verboseLogging:YES];
XCTAssert(engine != nil, @"Expected non-nil engine");
XCTAssert([engine.getCoreContext isEqualToString:@"abc"], @"Expected 'abc' in context buffer");
// Note: relying on km_core_state_context_debug output format is just barely
// acceptable for a unit test
XCTAssert([engine.getCoreContextDebug isEqualToString:@"|abc| (len: 3) [ U+0061 U+0062 U+0063 ]"], @"Expected 'abc' in context buffer");
}
- (void)testsetCoreContextIfNeeded_NonEmptyContext_InitialContextUpdated {
KMXFile *kmxFile = [KeymanEngineTestsStaticHelperMethods getKmxFileTestMacEngine];
KMEngine *engine = [[KMEngine alloc] initWithKMX:kmxFile context:@"a" verboseLogging:YES];
[engine setCoreContextIfNeeded:@"xyz"];
XCTAssert([engine.getCoreContext isEqualToString:@"xyz"], @"Expected 'xyz' in context buffer");
// Note: relying on km_core_state_context_debug output format is just barely
// acceptable for a unit test
XCTAssert([engine.getCoreContextDebug isEqualToString:@"|xyz| (len: 3) [ U+0078 U+0079 U+007a ]"], @"Expected 'xyz' in context buffer");
}
// TODO: re-enable this one after the core API km_core_state_context_set_if_needed is fixed
@ -64,7 +70,9 @@ NSString * names[nCombinations];
KMXFile *kmxFile = [KeymanEngineTestsStaticHelperMethods getKmxFileTestMacEngine];
KMEngine *engine = [[KMEngine alloc] initWithKMX:kmxFile context:@"" verboseLogging:YES];
[engine setCoreContextIfNeeded:@"xyz"];
XCTAssert([engine.getCoreContext isEqualToString:@"xyz"], @"Expected 'xyz' in context buffer");
// Note: relying on km_core_state_context_debug output format is just barely
// acceptable for a unit test
XCTAssert([engine.getCoreContextDebug isEqualToString:@"|xyz| (len: 3) [ U+0078 U+0079 U+007a ]"], @"Expected 'xyz' in context buffer");
}
*/
@ -278,7 +286,7 @@ NSString * names[nCombinations];
NSString * characters = charactersIgnoringModifiers;
if (modifiers[i] & (LEFT_ALT_FLAG | RIGHT_ALT_FLAG))
characters = [characters stringByAppendingString:@"\u030A"];
NSLog(@"Test case: %lu", (NSUInteger)modifiers[i]);
// NOTE: 'a' happens to be keyCode 0 (see initVirtualKeyMapping in CoreHelper)
NSEvent *event = [NSEvent keyEventWithType:NSEventTypeKeyDown location:NSMakePoint(0, 0) modifierFlags:modifiers[i] timestamp:0 windowNumber:0 context:nil characters:characters charactersIgnoringModifiers:charactersIgnoringModifiers isARepeat:NO keyCode:0];
@ -335,10 +343,10 @@ NSString * names[nCombinations];
- (void)testprocessEvent_eventForSWithModifiers_ReturnsCharacterActionWithExpectedCharacterBasedOnKmx {
int i = 0;
[KMEngineTests fillInNamesAndModifiersForAllChiralCombinations];
KMXFile *kmxFile = [KeymanEngineTestsStaticHelperMethods getKmxFileTestMacEngine];
KMEngine *engine = [[KMEngine alloc] initWithKMX:kmxFile context:@"" verboseLogging:YES];
for (i = 0; i < nCombinations; i++) {
[engine clearCoreContext];
NSString *charactersIgnoringModifiers = (modifiers[i] & (LEFT_SHIFT_FLAG | RIGHT_SHIFT_FLAG)) ? @"S" : @"s";
@ -349,7 +357,7 @@ NSString * names[nCombinations];
else
characters = @"ß";
}
NSLog(@"Test case: %lu", (NSUInteger)modifiers[i]);
// NOTE: 's' happens to be keyCode 1 (see initVirtualKeyMapping in CoreHelper)
NSEvent *event = [NSEvent keyEventWithType:NSEventTypeKeyDown location:NSMakePoint(0, 0) modifierFlags:modifiers[i] timestamp:0 windowNumber:0 context:nil characters:characters charactersIgnoringModifiers:charactersIgnoringModifiers isARepeat:NO keyCode:1];
@ -554,8 +562,10 @@ NSString * names[nCombinations];
NSLog(@"output: %@", output);
XCTAssert(output.codePointsToDeleteBeforeInsert == 1, @"Expected output to delete one code point");
XCTAssert(!output.hasTextToInsert, @"expected to insert nothing");
NSString *context = engine.getCoreContext;
XCTAssert([context isEqualToString:@""], @"Context should be empty.");
NSString *context = engine.getCoreContextDebug;
// Note: relying on km_core_state_context_debug output format is just barely
// acceptable for a unit test
XCTAssert([context isEqualToString:@"|| (len: 0) [ ]"], @"Context should be empty.");
}
- (void)testCoreProcessEvent_eventReturnWithElNuerKmx_EmitWithContextEmpty {
@ -564,8 +574,10 @@ NSString * names[nCombinations];
NSEvent *event = [NSEvent keyEventWithType:NSEventTypeKeyDown location:NSMakePoint(0, 0) modifierFlags:0 timestamp:0 windowNumber:0 context:nil characters:@"\n" charactersIgnoringModifiers:@"\n" isARepeat:NO keyCode:kVK_Return];
CoreKeyOutput *output = [engine processEvent:event];
XCTAssert(output.emitKeystroke, @"Expected emitKeystroke==YES");
NSString *context = engine.getCoreContext;
XCTAssert([context isEqualToString:@""], @"Context should be cleared.");
NSString *context = engine.getCoreContextDebug;
// Note: relying on km_core_state_context_debug output format is just barely
// acceptable for a unit test
XCTAssert([context isEqualToString:@"|| (len: 0) [ ]"], @"Context should be cleared.");
}
- (void)testCoreProcessEvent_eventTabWithElNuerKmx_EmitWithContextEmpty {
@ -574,8 +586,10 @@ NSString * names[nCombinations];
NSEvent *event = [NSEvent keyEventWithType:NSEventTypeKeyDown location:NSMakePoint(0, 0) modifierFlags:0 timestamp:0 windowNumber:0 context:nil characters:@"\t" charactersIgnoringModifiers:@"\t" isARepeat:NO keyCode:kVK_Tab];
CoreKeyOutput *output = [engine processEvent:event];
XCTAssert(output.emitKeystroke, @"Expected emitKeystroke==YES");
NSString *context = engine.getCoreContext;
XCTAssert([context isEqualToString:@""], @"Context should be cleared.");
NSString *context = engine.getCoreContextDebug;
// Note: relying on km_core_state_context_debug output format is just barely
// acceptable for a unit test
XCTAssert([context isEqualToString:@"|| (len: 0) [ ]"], @"Context should be cleared.");
}
- (void)testCoreProcessEvent_eventSingleQuoteWithElNuerKmx_ReturnsDiacritic {
@ -585,8 +599,10 @@ NSString * names[nCombinations];
NSEvent *event = [NSEvent keyEventWithType:NSEventTypeKeyDown location:NSMakePoint(0, 0) modifierFlags:0 timestamp:0 windowNumber:0 context:nil characters:@"'" charactersIgnoringModifiers:@"'" isARepeat:NO keyCode:kVK_ANSI_Quote];
CoreKeyOutput *output = [engine processEvent:event];
XCTAssert(output.hasTextToInsert, @"returns text to insert");
context = engine.getCoreContext;
XCTAssert([context isEqualToString:@"\u025B\u0308"], @"Context updated with diacritic.");
context = engine.getCoreContextDebug;
// Note: relying on km_core_state_context_debug output format is just barely
// acceptable for a unit test
XCTAssert([context isEqualToString:@"|\u025B\u0308| (len: 2) [ U+025b U+0308 ]"], @"Context updated with diacritic.");
}
@end

41
windows/src/engine/keyman32/calldll.cpp
View file

@ -154,38 +154,17 @@ LogContext(km_core_state *lpCoreKeyboardState, uint8_t context_type) {
if (!lpCoreKeyboardState) {
return FALSE;
}
km_core_context_item *citems = nullptr;
km_core_status error_status;
char *log_str_title = nullptr;
char* const int_context = "Intermediate Context";
char* const core_context = "Core Context";
switch(context_type){
case CONTEXT_CORE:
error_status = (km_core_status_codes)km_core_context_get(km_core_state_context(lpCoreKeyboardState), &citems);
log_str_title = int_context;
break;
case CONTEXT_INT:
error_status = (km_core_status_codes)km_core_state_get_intermediate_context(lpCoreKeyboardState, &citems);
log_str_title = core_context;
break;
default:
error_status = KM_CORE_STATUS_INVALID_ARGUMENT;
}
if (error_status != KM_CORE_STATUS_OK){
km_core_context_items_dispose(citems);
return FALSE;
}
DWORD context_length = (DWORD)km_core_context_item_list_size(citems);
WCHAR *buf = new WCHAR[(context_length * 3) +1 ]; // *3 if every context item was a deadkey
if (!ContextItemToAppContext(citems, buf, context_length)) {
km_core_context_items_dispose(citems);
delete[] buf;
return FALSE;
}
km_core_context_items_dispose(citems);
SendDebugMessageFormat(0, sdmKeyboard, 0, "%s: [%s]", log_str_title, Debug_UnicodeString(buf));
delete[] buf;
km_core_cp* buffer = km_core_state_context_debug(
lpCoreKeyboardState,
context_type == CONTEXT_CORE ? KM_CORE_DEBUG_CONTEXT_CACHED : KM_CORE_DEBUG_CONTEXT_INTERMEDIATE
);
char* const log_str_title = context_type == CONTEXT_CORE ? "Core Context" : "Intermediate Context";
SendDebugMessageFormat(0, sdmKeyboard, 0, "%s: %ls", log_str_title, buffer);
km_core_cp_dispose(buffer);
return TRUE;
}

1
windows/src/engine/keyman32/keymanengine.h
View file

@ -40,6 +40,7 @@
#include <msctf.h>
#include "../../../../common/windows/cpp/include/legacy_kmx_file.h"
#include <keyman/keyman_core_api.h>
#include <keyman/keyman_core_api_context.h> // for intermediate context
#include <keyman/keyman_core_api_consts.h>
/***************************************************************************/

39
windows/src/engine/keyman32/kmprocess.cpp
View file

@ -70,31 +70,6 @@
BOOL fOutputKeystroke;
char *getcontext_debug() {
PKEYMAN64THREADDATA _td = ThreadGlobals();
if (!_td || !_td->lpActiveKeyboard || !_td->lpActiveKeyboard->lpCoreKeyboardState){
return "";
}
WCHAR buf[(MAXCONTEXT * 3) + 1]; // *3 if every context item was a deadkey
km_core_context_item *citems = nullptr;
if (KM_CORE_STATUS_OK != km_core_context_get(
km_core_state_context(_td->lpActiveKeyboard->lpCoreKeyboardState), &citems)) {
return "";
}
DWORD context_length = (DWORD)km_core_context_item_list_size(citems);
if (!ContextItemToAppContext(citems, buf, context_length)) {
km_core_context_items_dispose(citems);
return "";
}
km_core_context_items_dispose(citems);
return Debug_UnicodeString(buf);
}
/**
* Process the key stroke using the core processor
*
@ -148,8 +123,18 @@ BOOL ProcessHook()
if(_td->state.msg.message == wm_keymankeydown) { // I4827
if (ShouldDebug(sdmKeyboard)) {
SendDebugMessageFormat(_td->state.msg.hwnd, sdmKeyboard, 0, "Key pressed: %s Context '%s'",
Debug_VirtualKey(_td->state.vkey), getcontext_debug());
if(!_td->lpActiveKeyboard || !_td->lpActiveKeyboard->lpCoreKeyboardState) {
SendDebugMessageFormat(_td->state.msg.hwnd, sdmKeyboard, 0, "Key pressed: %s Context <unavailable>",
Debug_VirtualKey(_td->state.vkey));
} else {
km_core_cp* debug_context = km_core_state_context_debug(
_td->lpActiveKeyboard->lpCoreKeyboardState,
KM_CORE_DEBUG_CONTEXT_CACHED
);
SendDebugMessageFormat(_td->state.msg.hwnd, sdmKeyboard, 0, "Key pressed: %s Context '%ls'",
Debug_VirtualKey(_td->state.vkey), debug_context);
km_core_cp_dispose(debug_context);
}
}
}