diff --git a/HISTORY.md b/HISTORY.md
index b5d6ea2caa..c5e1f2e780 100644
--- a/HISTORY.md
+++ b/HISTORY.md
@@ -1,5 +1,11 @@
# Keyman Version History
+## 18.0.55 alpha 2024-06-13
+
+* fix(developer): handle missing OSK when importing a Windows keyboard into a touch-only project (#11720)
+* fix(developer): verify email addresses in .kps and .keyboard_info (#11735)
+* change(web): prep for better asynchronous prediction handling (#10343)
+
## 18.0.54 alpha 2024-06-12
* fix(common): remove subpackage entries for older TS version (#11745)
diff --git a/VERSION.md b/VERSION.md
index 1d6c319a44..b38400af1f 100644
--- a/VERSION.md
+++ b/VERSION.md
@@ -1 +1 @@
-18.0.55
\ No newline at end of file
+18.0.56
\ No newline at end of file
diff --git a/core/src/actions_normalize.cpp b/core/src/actions_normalize.cpp
index 3384fda975..78bada7b6f 100644
--- a/core/src/actions_normalize.cpp
+++ b/core/src/actions_normalize.cpp
@@ -15,12 +15,12 @@
#include "state.hpp"
#include "option.hpp"
#include "debuglog.h"
-#include "core_icu.h"
+#include "util_normalize.hpp"
+#include "utfcodec.hpp"
+#include "kmx/kmx_xstring.h"
-// forward declarations
-
-icu::UnicodeString context_items_to_unicode_string(km::core::context const *context);
-km_core_usv *unicode_string_to_usv(icu::UnicodeString& src);
+// forward declaration
+bool context_items_to_unicode_string(km::core::context const *context, std::u32string &str);
/**
* Normalize the output from an action to NFC, across the context | output
@@ -65,32 +65,12 @@ bool km::core::actions_normalize(
cached_context.
*/
- /*
- Initialization
- */
-
- UErrorCode icu_status = U_ZERO_ERROR;
- const icu::Normalizer2 *nfc = icu::Normalizer2::getNFCInstance(icu_status);
- assert(U_SUCCESS(icu_status));
- if(!U_SUCCESS(icu_status)) {
- DebugLog("getNFCInstance failed with %x", icu_status);
+ std::u32string output(actions.output);
+ std::u32string cached_context_string, app_context_string;
+ if (!context_items_to_unicode_string(cached_context, cached_context_string)) {
return false;
}
-
- const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(icu_status);
- assert(U_SUCCESS(icu_status));
- if(!U_SUCCESS(icu_status)) {
- DebugLog("getNFDInstance failed with %x", icu_status);
- return false;
- }
-
- icu::UnicodeString output = icu::UnicodeString::fromUTF32(reinterpret_cast(actions.output), -1);
- icu::UnicodeString cached_context_string = context_items_to_unicode_string(cached_context);
- icu::UnicodeString app_context_string = context_items_to_unicode_string(app_context);
- assert(!output.isBogus());
- assert(!cached_context_string.isBogus());
- assert(!app_context_string.isBogus());
- if(output.isBogus() || cached_context_string.isBogus() || app_context_string.isBogus()) {
+ if (!context_items_to_unicode_string(app_context, app_context_string)) {
return false;
}
int nfu_to_delete = 0;
@@ -98,20 +78,25 @@ bool km::core::actions_normalize(
/*
Further debug assertion of inputs
*/
-
- assert(nfd->isNormalized(output, icu_status) && U_SUCCESS(icu_status));
- assert(nfd->isNormalized(cached_context_string, icu_status) && U_SUCCESS(icu_status));
+ assert(km::core::util::is_nfd(output));
+ assert(km::core::util::is_nfd(cached_context_string));
/*
The keyboard processor will have updated the cached_context already,
- applying the transform to it, so we need to rewind this. Remove the output
- from cached_context_string to start
+ applying the transform to it, so we need to rewind this.
+
+ Assert that 'cached_context_string' ends with 'output'
+
+ Remove the output
+ from cached_context_string to start.
*/
assert(cached_context_string.length() >= output.length());
- int n = cached_context_string.length() - output.length();
+ size_t n = cached_context_string.length() - output.length();
+ // auto end_cached = cached_context_string.substr(n, output.length());
+ // assert(end_cached == output);
assert(cached_context_string.compare(n, output.length(), output) == 0);
- cached_context_string.remove(n);
+ cached_context_string.resize(n);
/*
While cached_context is guaranteed to be normalized, actions->output may not
@@ -119,21 +104,22 @@ bool km::core::actions_normalize(
normalization in our output, we now need to look for a normalization
boundary prior to the intersection of the cached_context and the output.
*/
- if(!output.isEmpty()) {
- while(n > 0 && !nfd->hasBoundaryBefore(output[0])) {
+ if(!output.empty()) {
+ while(n > 0 && !km::core::util::has_nfd_boundary_before(output[0])) {
// The output may interact with the context further in normalization. We
// need to copy characters back further until we reach a normalization
// boundary.
// Remove last code point from the context ...
- n = cached_context_string.moveIndex32(n, -1);
- UChar32 chr = cached_context_string.char32At(n);
- cached_context_string.remove(n);
+ auto len = cached_context_string.length();
+ assert(len>0);
+ auto chr = cached_context_string.at(len-1);
+ cached_context_string.resize(len-1);
// And prepend it to the output ...
- output.insert(0, chr);
+ output.insert(0, 1, chr);
}
}
@@ -148,25 +134,20 @@ bool km::core::actions_normalize(
its normalized form matches the cached_context normalized form.
*/
- while(app_context_string.countChar32()) {
- icu::UnicodeString app_context_nfd;
- nfd->normalize(app_context_string, app_context_nfd, icu_status);
- assert(U_SUCCESS(icu_status));
- if(!U_SUCCESS(icu_status)) {
- DebugLog("nfd->normalize failed with %x", icu_status);
+ while(!app_context_string.empty()) {
+ auto app_context_nfd = app_context_string;
+ if(!km::core::util::normalize_nfd(app_context_nfd)) {
+ DebugLog("nfd->normalize failed");
return false;
}
- if(app_context_nfd.compare(cached_context_string) == 0) {
+ if(app_context_nfd == cached_context_string) {
break;
}
- // remove the last UChar32
- int32_t lastUChar32 = app_context_string.length()-1;
- // adjust pointer to get the entire char (i.e. so we don't slice a non-BMP char)
- lastUChar32 = app_context_string.getChar32Start(lastUChar32);
- // remove the UChar32 (1 or 2 code units)
- app_context_string.remove(lastUChar32);
+ size_t len = app_context_string.length();
+ // remove the cp at end
+ app_context_string.resize(len-1);
nfu_to_delete++;
}
@@ -174,17 +155,15 @@ bool km::core::actions_normalize(
Normalize our output string
*/
- icu::UnicodeString output_nfc;
- nfc->normalize(output, output_nfc, icu_status);
- assert(U_SUCCESS(icu_status));
- if(!U_SUCCESS(icu_status)) {
- DebugLog("nfc->normalize failed with %x", icu_status);
+ auto output_nfc = output;
+ if(!km::core::util::normalize_nfc(output_nfc)) {
+ DebugLog("nfc->normalize failed");
return false;
}
- auto new_output = unicode_string_to_usv(output_nfc);
- if(!new_output) {
- // error logging handled in unicode_string_to_usv
+ auto new_output = km::core::util::string_to_usv(output_nfc);
+ assert(new_output != nullptr);
+ if(new_output == nullptr) {
return false;
}
@@ -197,8 +176,8 @@ bool km::core::actions_normalize(
app_context_string.append(output_nfc);
km_core_context_item *app_context_items = nullptr;
km_core_status status = KM_CORE_STATUS_OK;
- if((status = context_items_from_utf16(app_context_string.getTerminatedBuffer(), &app_context_items)) != KM_CORE_STATUS_OK) {
- DebugLog("context_items_from_utf16 failed with %x", status);
+ if((status = context_items_from_utf32(app_context_string.c_str(), &app_context_items)) != KM_CORE_STATUS_OK) {
+ DebugLog("context_items_from_string failed with %x", status);
delete [] new_output;
return false;
}
@@ -224,21 +203,19 @@ bool km::core::actions_normalize(
/**
* Helper to convert km_core_context list into a icu::UnicodeString
*/
-icu::UnicodeString context_items_to_unicode_string(km::core::context const *context) {
- icu::UnicodeString nullString;
- nullString.setToBogus();
+bool context_items_to_unicode_string(km::core::context const *context, std::u32string &str) {
km_core_context_item *items = nullptr;
km_core_status status;
if((status = km_core_context_get(static_cast(context), &items)) != KM_CORE_STATUS_OK) {
DebugLog("Failed to retrieve context with %s", status);
- return nullString;
+ return false;
}
size_t buf_size = 0;
if((status = context_items_to_utf32(items, nullptr, &buf_size)) != KM_CORE_STATUS_OK) {
DebugLog("Failed to retrieve context size with %s", status);
km_core_context_items_dispose(items);
- return nullString;
+ return false;
}
km_core_usv *buf = new km_core_usv[buf_size];
@@ -246,38 +223,15 @@ icu::UnicodeString context_items_to_unicode_string(km::core::context const *cont
DebugLog("Failed to retrieve context with %s", status);
km_core_context_items_dispose(items);
delete [] buf;
- return nullString;
+ return false;
}
- auto result = icu::UnicodeString::fromUTF32(reinterpret_cast(buf), -1);
+ str = std::u32string(buf, buf_size - 1); // don't include terminating null
km_core_context_items_dispose(items);
delete [] buf;
- return result;
+ return true;
}
-/**
- * Helper to convert icu::UnicodeString to a UTF-32 km_core_usv buffer,
- * nul-terminated
- */
-km_core_usv *unicode_string_to_usv(icu::UnicodeString& src) {
- UErrorCode icu_status = U_ZERO_ERROR;
-
- km_core_usv *dst = new km_core_usv[src.length() + 1];
-
- src.toUTF32(reinterpret_cast(dst), src.length(), icu_status);
-
- assert(U_SUCCESS(icu_status));
- if(!U_SUCCESS(icu_status)) {
- DebugLog("toUTF32 failed with %x", icu_status);
- delete[] dst;
- return nullptr;
- }
-
- dst[src.length()] = 0;
- return dst;
-}
-
-
/**
* Refresh app_context to match the cached_context. Does not do normalization,
diff --git a/core/src/context.hpp b/core/src/context.hpp
index 91cc9688cd..f56d637eff 100644
--- a/core/src/context.hpp
+++ b/core/src/context.hpp
@@ -87,6 +87,30 @@ km_core_status
context_items_from_utf16(km_core_cu const *text,
km_core_context_item **out_ptr);
+/**
+ * Convert a UTF32 encoded Unicode string into an array of `km_core_context_item`
+ * structures. Allocates memory as needed.
+ *
+ * @return km_core_status
+ * * `KM_CORE_STATUS_OK`: On success.
+ * * `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are
+ * null.
+ * * `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be
+ * allocated for the output buffer.
+ * * `KM_CORE_STATUS_INVALID_UTF`: In the event the UTF32 string cannot
+ * be decoded.
+ *
+ * @param text a pointer to a null terminated array of utf32 encoded data.
+ * @param out_ptr a pointer to the result variable: A pointer to the start of
+ * the `km_core_context_item` array containing the representation
+ * of the input string. Terminated with a type of
+ * `KM_CORE_CT_END`. Must be disposed of with
+ * `km_core_context_items_dispose`.
+ */
+km_core_status
+context_items_from_utf32(km_core_usv const *text,
+ km_core_context_item **out_ptr);
+
/**
* Convert a context item array into a UTF-16 encoded string placing it into the
* supplied buffer of specified size, and return the number of code units
diff --git a/core/src/core_icu.cpp b/core/src/core_icu.cpp
new file mode 100644
index 0000000000..cfaac47c50
--- /dev/null
+++ b/core/src/core_icu.cpp
@@ -0,0 +1,72 @@
+/*
+ Copyright: © SIL International.
+ Description: Common LDML utilities
+ Create Date: 24 May 2024
+ Authors: Steven R. Loomis
+*/
+
+#include "core_icu.h"
+
+#if !KMN_NO_ICU
+
+namespace km {
+namespace core {
+namespace util {
+
+/**
+ * Helper to convert icu::UnicodeString to a UTF-32 km_core_usv buffer,
+ * nul-terminated
+ */
+km_core_usv *unicode_string_to_usv(icu::UnicodeString& src) {
+ UErrorCode icu_status = U_ZERO_ERROR;
+
+ km_core_usv *dst = new km_core_usv[src.length() + 1];
+
+ src.toUTF32(reinterpret_cast(dst), src.length(), icu_status);
+
+ if(!UASSERT_SUCCESS(icu_status)) {
+ DebugLog("toUTF32 failed with %x", icu_status);
+ delete[] dst;
+ return nullptr;
+ }
+
+ dst[src.length()] = 0;
+ return dst;
+}
+
+/**
+ * Internal function to normalize with a specified mode.
+ * Note: that this function _does_ assert failure, so it is not
+ * required to assert its return code. The return is provided so
+ * that callers can exit (such as making no change) if there was failure.
+ *
+ * Also note that "failure" here is something catastrophic: ICU not initialized,
+ * or, more likely, some low memory situation. Does not fail on "bad" data.
+ * @param n the ICU Normalizer to use
+ * @param str input/output string
+ * @param status error code, must be initialized on input
+ * @return false if failure
+ */
+bool normalize(const icu::Normalizer2 *n, std::u16string &str, UErrorCode &status) {
+ if(!UASSERT_SUCCESS(status)) {
+ return false;
+ }
+ assert(n != nullptr);
+ icu::UnicodeString dest;
+ icu::UnicodeString src = icu::UnicodeString(str.data(), (int32_t)str.length());
+ n->normalize(src, dest, status);
+ // the next line here will assert
+ if (!UASSERT_SUCCESS(status)) {
+ return false;
+ } else {
+ str.assign(dest.getBuffer(), dest.length());
+ return true;
+ }
+}
+
+
+}
+}
+} /* end km::core::util */
+
+#endif
diff --git a/core/src/core_icu.h b/core/src/core_icu.h
index d95d198637..edfc209802 100644
--- a/core/src/core_icu.h
+++ b/core/src/core_icu.h
@@ -3,8 +3,30 @@
*/
#pragma once
+#ifdef __EMSCRIPTEN__
+// define this in tests to keep ICU around
+# if !defined(KMN_IN_LDML_TESTS)
+# if !defined(KMN_NO_ICU)
+// under wasm, turn off ICU except in tests.
+# define KMN_NO_ICU 1
+# endif
+# endif
+#elif !defined(KMN_NO_ICU)
+# define KMN_NO_ICU 0
+#endif
+
+#if KMN_NO_ICU
+
+// NO ICU
+
+// any shims needed here for disabling ICU
+
+#else
+
+// YES ICU
+
#if !defined(HAVE_ICU4C)
-#error icu4c is required for this code
+# error icu4c is required for this code
#endif
#define U_FALLTHROUGH
@@ -12,6 +34,7 @@
#include "unicode/unistr.h"
#include "unicode/normalizer2.h"
+#include "keyman_core.h"
#include "debuglog.h"
#include
@@ -29,5 +52,40 @@ inline bool uassert_success(const char *file, int line, const char *function, UE
/**
* Assert an ICU4C UErrorCode
* the first assert is for debug builds, the second triggers the debuglog and has the return value.
+ * @returns true on success
* */
#define UASSERT_SUCCESS(status) (assert(U_SUCCESS(status)), uassert_success(__FILE__, __LINE__, __FUNCTION__, status))
+
+// ------------------ some ICU C++ utilities ----------------------------
+
+namespace km {
+namespace core {
+namespace util {
+
+/**
+ * Convert a UnicodeString to a km_core_usv array
+ * @return the 0-terminated array. Caller owns storage.
+ */
+km_core_usv *unicode_string_to_usv(icu::UnicodeString& src);
+
+/**
+ * Internal function to normalize with a specified mode.
+ * Note: that this function _does_ assert failure, so it is not
+ * required to assert its return code. The return is provided so
+ * that callers can exit (such as making no change) if there was failure.
+ *
+ * Also note that "failure" here is something catastrophic: ICU not initialized,
+ * or, more likely, some low memory situation. Does not fail on "bad" data.
+ * @param n the ICU Normalizer to use
+ * @param str input/output string
+ * @param status error code, must be initialized on input
+ * @return false if failure
+ */
+bool normalize(const icu::Normalizer2 *n, std::u16string &str, UErrorCode &status);
+
+
+}
+}
+}
+
+#endif /* KMN_NO_ICU */
diff --git a/core/src/km_core_context_api.cpp b/core/src/km_core_context_api.cpp
index dd27ab7e5c..594080d8d0 100644
--- a/core/src/km_core_context_api.cpp
+++ b/core/src/km_core_context_api.cpp
@@ -115,6 +115,13 @@ context_items_from_utf16(km_core_cu const *text,
}
+km_core_status
+context_items_from_utf32(km_core_usv const *text,
+ km_core_context_item **out_ptr)
+{
+ return _context_items_from(reinterpret_cast(text), out_ptr);
+}
+
km_core_status context_items_to_utf8(km_core_context_item const *ci,
char *buf, size_t * sz_ptr)
{
diff --git a/core/src/kmx/kmx_xstring.cpp b/core/src/kmx/kmx_xstring.cpp
index a1d5ff518a..794ba26889 100644
--- a/core/src/kmx/kmx_xstring.cpp
+++ b/core/src/kmx/kmx_xstring.cpp
@@ -50,6 +50,15 @@ size_t km::core::kmx::u16len(const km_core_cu *p) {
return i;
}
+size_t km::core::kmx::u32len(const km_core_usv *p) {
+ int i = 0;
+ while (*p) {
+ p++;
+ i++;
+ }
+ return i;
+}
+
int km::core::kmx::u16cmp(const km_core_cu *p, const km_core_cu *q) {
while (*p && *q) {
if (*p != *q) return *p - *q;
@@ -107,6 +116,11 @@ km_core_cu *km::core::kmx::u16dup(km_core_cu *src) {
memcpy(dup, src, (u16len(src) + 1) * sizeof(km_core_cu));
return dup;
}
+km_core_usv *km::core::kmx::u32dup(const km_core_usv *src) {
+ km_core_usv *dup = new km_core_usv[u32len(src) + 1];
+ memcpy(dup, src, (u32len(src) + 1) * sizeof(src[0]));
+ return dup;
+}
/*
* int xstrlen( PKMX_BYTE p );
diff --git a/core/src/kmx/kmx_xstring.h b/core/src/kmx/kmx_xstring.h
index a82e959e56..e17c003091 100644
--- a/core/src/kmx/kmx_xstring.h
+++ b/core/src/kmx/kmx_xstring.h
@@ -117,6 +117,9 @@ int u16ncmp(const km_core_cu *p, const km_core_cu *q, size_t count);
km_core_cu *u16tok(km_core_cu *p, km_core_cu ch, km_core_cu **ctx);
km_core_cu *u16dup(km_core_cu *src);
+size_t u32len(const km_core_usv *p);
+km_core_usv *u32dup(const km_core_usv *src);
+
//KMX_BOOL MapUSCharToVK(KMX_WORD ch, PKMX_WORD puKey, PKMX_DWORD puShiftFlags);
// --- implementation ---
diff --git a/core/src/ldml/ldml_markers.cpp b/core/src/ldml/ldml_markers.cpp
index b1d4604dbc..5d4fb4ff46 100644
--- a/core/src/ldml/ldml_markers.cpp
+++ b/core/src/ldml/ldml_markers.cpp
@@ -268,15 +268,14 @@ add_pending_markers(
marker_map *markers,
marker_list &last_markers,
const std::u32string::const_iterator &last,
- const std::u32string::const_iterator &end,
- const icu::Normalizer2 *nfd) {
+ const std::u32string::const_iterator &end) {
// quick check to see if there's no work to do.
if(markers == nullptr) {
return;
}
/** which character this marker is 'glued' to. */
char32_t marker_ch;
- icu::UnicodeString decomposition;
+ std::u32string decomposition;
if (last == end) {
// at end of text, so use a special value to indicate 'EOT'.
marker_ch = MARKER_BEFORE_EOT;
@@ -285,17 +284,13 @@ add_pending_markers(
// if the character is composed, we need to use the first decomposed char
// as the 'glue'.
- if(!nfd->getDecomposition(ch, decomposition)) {
+ if(!km::core::util::normalize_nfd(ch, decomposition)) {
// char does not have a decomposition - so it may be used for the glue
- marker_ch = ch;
- decomposition.remove(); // no other entries needed
- } else {
- // 'glue' is the first codepoint of the decomposition.
- marker_ch = decomposition.char32At(0);
- if (decomposition.countChar32() == 1) {
- decomposition.remove(); // no other entries needed
- } // else: will add the remainder below
+ // the 'if' is only for the assertions here.
+ assert(decomposition.length() == 1); // should be a single UTF-32 char
+ assert(decomposition.at(0) == ch); // should be the same char
}
+ marker_ch = decomposition.at(0); // always the first char
}
markers->emplace_back(marker_ch);
// now, update the map with these markers (in order) on this character.
@@ -304,10 +299,10 @@ add_pending_markers(
markers->emplace_back(marker_ch, *i);
}
// add any further entries due to decomposition
- if (!decomposition.isEmpty()) {
- // We already added the base char above, add teh rest
- for (auto i=1; iemplace_back(decomposition.char32At(i));
+ if (decomposition.length() > 1) {
+ // We already added the base char above, add the rest
+ for (size_t i=1; iemplace_back(decomposition.at(i));
}
}
// clear the list
@@ -318,10 +313,6 @@ std::u32string
remove_markers(const std::u32string &str, marker_map *markers, marker_encoding encoding) {
std::u32string out;
marker_list last_markers;
- UErrorCode status = U_ZERO_ERROR;
- const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(status);
- UASSERT_SUCCESS(status);
-
auto last = str.begin(); // points to the part of the string after the last matched marker
for (auto i = str.begin(); i != str.end();) {
auto marker_no = parse_next_marker(i, str.end(), encoding);
@@ -329,7 +320,7 @@ remove_markers(const std::u32string &str, marker_map *markers, marker_encoding e
// add any markers found before this entry, but only if there is intervening
// text. This prevents the sentinel or the '\u' from becoming the attachment char.
if (i != last) {
- add_pending_markers(markers, last_markers, last, str.end(), nfd);
+ add_pending_markers(markers, last_markers, last, str.end());
out.append(last, i); // append any non-marker text since the end of the last marker
last = i; // advance over text we've already appended
}
@@ -347,7 +338,7 @@ remove_markers(const std::u32string &str, marker_map *markers, marker_encoding e
// add any remaining pending markers.
// if last == str.end() then this wil be MARKER_BEFORE_EOT
// otherwise it will be the glue character
- add_pending_markers(markers, last_markers, last, str.end(), nfd);
+ add_pending_markers(markers, last_markers, last, str.end());
// get the suffix between the last marker and the end (could be nothing)
out.append(last, str.end());
return out;
diff --git a/core/src/ldml/ldml_transforms.cpp b/core/src/ldml/ldml_transforms.cpp
index 7b161ff4f9..42953c604e 100644
--- a/core/src/ldml/ldml_transforms.cpp
+++ b/core/src/ldml/ldml_transforms.cpp
@@ -1,3 +1,5 @@
+// TEMP
+#define KMN_NO_ICU 0
/*
Copyright: © SIL International.
Description: This is an implementation of the LDML keyboard spec 3.0.
diff --git a/core/src/meson.build b/core/src/meson.build
index e8008fb78a..a4160842eb 100644
--- a/core/src/meson.build
+++ b/core/src/meson.build
@@ -40,6 +40,29 @@ if icu_uc.found()
defns += '-DHAVE_ICU4C'
endif
+# On wasm, generate util_normalize_table.h automatically from ICU
+generated_headers = []
+
+if cpp_compiler.get_id() == 'emscripten'
+
+util_normalize_table_generator = executable('util_normalize_table_generator',
+ ['util_normalize_table_generator.cpp'],
+ cpp_args: defns + warns,
+ include_directories: [inc],
+ link_args: links,
+ dependencies: [icu_uc, icu_i18n],
+ )
+
+util_normalize_table_h = custom_target('util_normalize_table.h',
+ output: 'util_normalize_table.h',
+ command: [util_normalize_table_generator],
+ capture:true)
+
+generated_headers += util_normalize_table_h
+
+
+endif
+
kmx_files = files(
'actions_normalize.cpp',
@@ -60,6 +83,7 @@ kmx_files = files(
'km_core_processevent_api.cpp',
'jsonpp.cpp',
'util_normalize.cpp',
+ 'core_icu.cpp',
'ldml/ldml_processor.cpp',
'ldml/ldml_transforms.cpp',
'ldml/ldml_markers.cpp',
@@ -110,6 +134,7 @@ lib = library('keymancore',
kmx_files,
mock_files,
version_res,
+ generated_headers,
cpp_args: defns + warns + flags,
link_args: links,
version: lib_version,
diff --git a/core/src/util_normalize.cpp b/core/src/util_normalize.cpp
index c21b75ea5b..bcdfc1bb3f 100644
--- a/core/src/util_normalize.cpp
+++ b/core/src/util_normalize.cpp
@@ -13,6 +13,7 @@
#ifdef __EMSCRIPTEN__
#include
#include "utfcodec.hpp"
+#include
// JS implementations
EM_JS(char*, NormalizeNFD, (const char* input), {
@@ -21,6 +22,17 @@ EM_JS(char*, NormalizeNFD, (const char* input), {
const nfd = instr.normalize("NFD");
return stringToNewUTF8(nfd);
});
+
+EM_JS(char*, NormalizeNFC, (const char* input), {
+ if (!input) return input; // pass through null
+ const instr = Module.UTF8ToString(input);
+ const nfd = instr.normalize("NFC");
+ return stringToNewUTF8(nfd);
+});
+
+// pull in the generated table
+#include "util_normalize_table.h"
+
#endif
namespace km {
@@ -28,31 +40,15 @@ namespace core {
namespace util {
#ifndef __EMSCRIPTEN__
-
-/**
- * Internal function to normalize with a specified mode.
- * Note: that this function _does_ assert failure, so it is not
- * required to assert its return code. The return is provided so
- * that callers can exit (such as making no change) if there was failure.
- *
- * Also note that "failure" here is something catastrophic: ICU not initialized,
- * or, more likely, some low memory situation. Does not fail on "bad" data.
- * @param n the ICU Normalizer to use
- * @param str input/output string
- * @param status error code, must be initialized on input
- * @return false if failure
- */
-static bool normalize(const icu::Normalizer2 *n, std::u16string &str, UErrorCode &status) {
+inline const icu::Normalizer2 *getNFD(UErrorCode &status) {
+ const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(status);
UASSERT_SUCCESS(status);
- assert(n != nullptr);
- icu::UnicodeString dest;
- icu::UnicodeString src = icu::UnicodeString(str.data(), (int32_t)str.length());
- n->normalize(src, dest, status);
- // the next line here will assert
- if (UASSERT_SUCCESS(status)) {
- str.assign(dest.getBuffer(), dest.length());
- }
- return U_SUCCESS(status);
+ return nfd;
+}
+inline const icu::Normalizer2 *getNFC(UErrorCode &status) {
+ const icu::Normalizer2 *nfc = icu::Normalizer2::getNFCInstance(status);
+ UASSERT_SUCCESS(status);
+ return nfc;
}
#endif
@@ -66,6 +62,16 @@ bool normalize_nfd(std::u32string &str) {
}
}
+bool normalize_nfc(std::u32string &str) {
+ std::u16string rstr = km::core::kmx::u32string_to_u16string(str);
+ if(!km::core::util::normalize_nfc(rstr)) {
+ return false;
+ } else {
+ str = km::core::kmx::u16string_to_u32string(rstr);
+ return true;
+ }
+}
+
bool normalize_nfd(std::u16string &str) {
#ifdef __EMSCRIPTEN__
std::string instr = convert(str);
@@ -81,9 +87,26 @@ bool normalize_nfd(std::u16string &str) {
return true;
#else
UErrorCode status = U_ZERO_ERROR;
- const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(status);
- UASSERT_SUCCESS(status);
- return normalize(nfd, str, status);
+ return normalize(getNFD(status), str, status);
+#endif
+}
+
+bool normalize_nfc(std::u16string &str) {
+#ifdef __EMSCRIPTEN__
+ std::string instr = convert(str);
+ const char *in = instr.c_str();
+ char *out = NormalizeNFC(in);
+ if (out == nullptr) {
+ assert(out != nullptr);
+ return false;
+ }
+ std::string outstr(out);
+ str = convert(outstr);
+ free(out);
+ return true;
+#else
+ UErrorCode status = U_ZERO_ERROR;
+ return normalize(getNFC(status), str, status);
#endif
}
@@ -91,26 +114,130 @@ bool normalize_nfd(std::u16string &str) {
* Normalize the input string using ICU, out of place
*/
bool normalize_nfd(km_core_cu const * src, std::u16string &dst) {
- UErrorCode icu_status = U_ZERO_ERROR;
- const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(icu_status);
- assert(U_SUCCESS(icu_status));
- if(!U_SUCCESS(icu_status)) {
- // TODO: log the failure code
+#ifdef __EMSCRIPTEN__
+ dst = std::u16string(src);
+ return normalize_nfd(dst); // vector to above fcn
+#else
+ UErrorCode status = U_ZERO_ERROR;
+ auto nfd = getNFD(status);
+ if (nfd == nullptr) {
return false;
}
icu::UnicodeString udst;
icu::UnicodeString usrc = icu::UnicodeString(src);
- nfd->normalize(usrc, udst, icu_status);
- assert(U_SUCCESS(icu_status));
- if(!U_SUCCESS(icu_status)) {
- // TODO: log the failure code
+ nfd->normalize(usrc, udst, status);
+ if(!UASSERT_SUCCESS(status)) {
return false;
}
dst.assign(udst.getBuffer(), udst.length());
return true;
+#endif
}
+bool
+normalize_nfd(km_core_usv cp, std::u32string &dst) {
+ // set the output string to the original string
+ dst.clear();
+ dst.append(1, cp);
+#ifdef __EMSCRIPTEN__
+ auto str16 = convert(dst);
+ if (!normalize_nfd(str16)) {
+ return false; // failed, retain original str
+ } else {
+ dst = convert(str16);
+ return true;
+ }
+#else
+ UErrorCode icu_status = U_ZERO_ERROR;
+ const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(icu_status);
+ assert(U_SUCCESS(icu_status));
+ if (!U_SUCCESS(icu_status)) {
+ // TODO: log the failure code
+ return false;
+ }
+ icu::UnicodeString decomposition;
+ if (!nfd->getDecomposition(cp, decomposition)) {
+ return false; // no error, just no decomposition
+ } else {
+ dst.clear();
+ auto len = decomposition.countChar32();
+ for (int i = 0; i < len; i++) {
+ dst.append(1, decomposition.char32At(i));
+ }
+ return true;
+ }
+#endif
+}
+
+bool is_nfd(const std::u16string& str) {
+#ifdef __EMSCRIPTEN__
+ std::u16string o = str;
+ normalize_nfd(o);
+ return (o == str); // false if changed
+#else
+ UErrorCode status = U_ZERO_ERROR;
+ auto nfd = getNFD(status);
+ if (nfd == nullptr) return false;
+ auto ustr = icu::UnicodeString(false, str.c_str(), (int)str.length());
+ auto result = nfd->isNormalized(ustr, status);
+ if (!UASSERT_SUCCESS(status)) {
+ return false;
+ } else {
+ return result;
+ }
+#endif
+}
+
+bool is_nfd(const std::u32string& str) {
+#ifdef __EMSCRIPTEN__
+ std::u32string o = str;
+ normalize_nfd(o);
+ return (o == str); // false if changed
+#else
+ UErrorCode status = U_ZERO_ERROR;
+ auto nfd = getNFD(status);
+ if (nfd == nullptr) return false;
+ auto ustr = icu::UnicodeString::fromUTF32(reinterpret_cast(str.c_str()), (int)str.length());
+ auto result = nfd->isNormalized(ustr, status);
+ if (!UASSERT_SUCCESS(status)) {
+ return false;
+ } else {
+ return result;
+ }
+#endif
+}
+
+bool has_nfd_boundary_before(km_core_usv cp) {
+#ifdef __EMSCRIPTEN__
+// it's a negative table. entries in the table mean returning false. non-entries return true.
+ for (auto i=0;i<(km_noBoundaryBefore_entries*2);i+=2) {
+ auto start = km_noBoundaryBefore[i+0];
+ if (start > cp) return true;
+ auto count = km_noBoundaryBefore[i+1];
+ auto limit = start+count;
+ if (cp >= start && cp < limit) return false;
+ }
+ return true; // fallthrough
+#else
+ UErrorCode status = U_ZERO_ERROR;
+ auto nfd = getNFD(status);
+ if (nfd == nullptr) return false;
+ return nfd->hasBoundaryBefore(cp);
+#endif
+}
+
+/**
+ * Helper to convert std::u32string to a UTF-32 km_core_usv buffer,
+ * nul-terminated.
+ * Parallel to unicode_string_to_usv()
+ * @returns new buffer, caller owns storage
+ */
+km_core_usv *string_to_usv(const std::u32string& src) {
+ return km::core::kmx::u32dup(src.c_str());
+}
+
+
}
}
}
diff --git a/core/src/util_normalize.hpp b/core/src/util_normalize.hpp
index 90ed650b08..6321ffacda 100644
--- a/core/src/util_normalize.hpp
+++ b/core/src/util_normalize.hpp
@@ -14,6 +14,12 @@ namespace km {
namespace core {
namespace util {
+/** Normalize a u32string inplace to NFC. @return false on failure */
+bool normalize_nfc(std::u32string &str);
+
+/** Normalize a u16string inplace to NFC. @return false on failure */
+bool normalize_nfc(std::u16string &str);
+
/** Normalize a u32string inplace to NFD. @return false on failure */
bool normalize_nfd(std::u32string &str);
@@ -23,6 +29,21 @@ bool normalize_nfd(std::u16string &str);
/** normalize src to dst in NFD. @return false on failure */
bool normalize_nfd(km_core_cu const * src, std::u16string &dst);
+/** normalize (decompose) a single cp to string. @return false on failure */
+bool normalize_nfd(km_core_usv cp, std::u32string &dst);
+
+/** @return true if string is already NFD */
+bool is_nfd(const std::u16string& str);
+
+/** @return true if string is already NFD */
+bool is_nfd(const std::u32string& str);
+
+/** @return true if cp can interacts with prior chars */
+bool has_nfd_boundary_before(km_core_usv cp);
+
+/** convenience function, caller owns storage */
+km_core_usv *string_to_usv(const std::u32string& src);
+
}
}
}
diff --git a/core/src/util_normalize_table_generator.cpp b/core/src/util_normalize_table_generator.cpp
new file mode 100644
index 0000000000..994f16d8b3
--- /dev/null
+++ b/core/src/util_normalize_table_generator.cpp
@@ -0,0 +1,108 @@
+/*
+ Copyright: © SIL International.
+ Description: Generator for util_normalize_table.h
+ Create Date: 5 Jun 2024
+ Authors: Steven R. Loomis
+
+ util_normalize_table.h is used under wasm by utilities in util_normalize.cpp to implement
+ normalization functions without needing ICU4C linked.
+
+ This generator is invoked automatically by meson as part of the build.
+*/
+
+#include "kmx/kmx_plus.h"
+#include "kmx/kmx_xstring.h"
+
+#define KMN_NO_ICU 0 // we will need ICU..
+
+#include "core_icu.h"
+
+#include
+#include
+#include
+#include
+
+#include
+
+#include
+
+
+
+int
+write_nfd_table() {
+#ifndef __EMSCRIPTEN__
+ std::cerr << "Note: This is unusual - this generator is usually only run under emscripten!" << std::endl;
+#endif
+
+ // We write to stdout instead of to a file to avoid dealing with the filesystem under emscripten.
+
+ std::cerr << "Writing to stdout." << std::endl;
+
+ // write preamble
+ std::cout << "// GENERATED FILE: DO NOT EDIT" << std::endl;
+ std::cout << "//" << std::endl;
+ std::cout << "// util_normalize_table.h is generated by util_normalize_table_generator.cpp" << std::endl;
+ std::cout << "// and used by util_normalize.cpp" << std::endl;
+ std::cout << std::endl;
+ std::cout << "#pragma once" << std::endl;
+ std::cout << "#define KM_HASBOUNDARYBEFORE_UNICODE_VERSION \"" << U_UNICODE_VERSION << "\"" << std::endl;
+ std::cout << "#define KM_HASBOUNDARYBEFORE_ICU_VERSION \"" << U_ICU_VERSION << "\"" << std::endl;
+ std::cout << std::endl;
+ // we're going to need an NFD normalizer
+ UErrorCode status = U_ZERO_ERROR;
+ const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(status);
+ assert(U_SUCCESS(status));
+
+ // collect the raw list of chars that do NOT have a boundary before them.
+ std::vector noBoundary;
+ for (km_core_usv ch = 0; ch < km::core::kmx::Uni_MAX_CODEPOINT; ch++) {
+ bool bb = nfd->hasBoundaryBefore(ch);
+ assert(!(ch == 0 && !bb)); // assert that we can use U+0000 as a terminator
+ if (bb) continue; //only emit nonboundary
+ noBoundary.push_back(ch);
+ }
+
+ // now, compress these into runs
+ std::vector> runs; // start,len
+
+ km_core_usv first = 0;
+ km_core_usv last = 0;
+ for(auto i = noBoundary.begin(); i <= noBoundary.end(); i++) {
+ if (first == 0) {
+ first = last = *i;
+ } else {
+ last++;
+ if(i == noBoundary.end() || *i != last) {
+ // end of a run
+ runs.emplace_back(first, last - first);
+ if (i != noBoundary.end()) {
+ // setup for next
+ first = last = *i;
+ }
+ }
+ }
+ }
+
+ // finally, write out metadata and the runs themselves.
+ std::cout << "#define km_noBoundaryBefore_entries " << runs.size() << "\n";
+
+ std::cout << "static char32_t km_noBoundaryBefore[km_noBoundaryBefore_entries * 2 ] = {" << std::endl;
+
+ std::cout << "/* start codepoint, count (inclusive), ...range end */" << std::endl;
+
+ for (auto i = runs.begin(); i < runs.end(); i++) {
+ std::cout << "\t0x" << std::hex << i->first << std::dec << ",\t " << i->second << ", // ...0x" << std::hex << (i->first+i->second-1) << std::endl;
+ }
+
+ // termination
+ std::cout << "};" << std::endl;
+ std::cout << "// end" << std::endl;
+ std::cerr << "Wrote " << runs.size() << " runs representing " << noBoundary.size() << " entries." << std::endl;
+ return 0;
+}
+
+int
+main(int /*argc*/, const char * /*argv*/[]) {
+ write_nfd_table();
+ return 0;
+}
diff --git a/core/tests/unit/kmnkbd/meson.build b/core/tests/unit/kmnkbd/meson.build
index 0adda0a35e..7285b9bf11 100644
--- a/core/tests/unit/kmnkbd/meson.build
+++ b/core/tests/unit/kmnkbd/meson.build
@@ -34,7 +34,7 @@ test_path = join_paths(meson.current_build_dir(), '..', 'kmx')
tests_flags = []
if cpp_compiler.get_id() == 'emscripten'
- tests_flags += ['-lnodefs.js', '-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\']']
+ tests_flags += ['-lnodefs.js', wasm_exported_runtime_methods]
endif
foreach t : tests
diff --git a/core/tests/unit/ldml/ldml_test_source.cpp b/core/tests/unit/ldml/ldml_test_source.cpp
index 439236fd77..f2cca9ee5f 100644
--- a/core/tests/unit/ldml/ldml_test_source.cpp
+++ b/core/tests/unit/ldml/ldml_test_source.cpp
@@ -18,6 +18,9 @@
#include
+// Ensure that ICU gets included even on wasm.
+#define KMN_IN_LDML_TESTS
+
#include // for char to vk mapping tables
#include // for surrogate pair macros
#include
diff --git a/core/tests/unit/ldml/meson.build b/core/tests/unit/ldml/meson.build
index f164e90f12..81f63033fe 100644
--- a/core/tests/unit/ldml/meson.build
+++ b/core/tests/unit/ldml/meson.build
@@ -126,7 +126,7 @@ test('test_context_normalization', test_context_normalization, suite: 'ldml')
# Build and run additional test_unicode test
test_unicode = executable('test_unicode', 'test_unicode.cpp',
- ['test_unicode.cpp', common_test_files],
+ ['test_unicode.cpp', common_test_files, generated_headers],
cpp_args: defns + warns,
include_directories: [inc, libsrc, '../../../../developer/src/ext/json'],
link_args: links + tests_flags,
diff --git a/core/tests/unit/ldml/test_unicode.cpp b/core/tests/unit/ldml/test_unicode.cpp
index 1c2da5444d..b36f1a3e38 100644
--- a/core/tests/unit/ldml/test_unicode.cpp
+++ b/core/tests/unit/ldml/test_unicode.cpp
@@ -10,6 +10,9 @@
#include
#include
+// Ensure that ICU gets included even on wasm.
+#define KMN_IN_LDML_TESTS
+
#include "keyman_core.h"
#include "path.hpp"
@@ -22,6 +25,8 @@
#include
#include
#include "json.hpp"
+#include "util_normalize.hpp"
+#include "kmx/kmx_xstring.h"
#include
#include
@@ -40,6 +45,11 @@
} \
}
+#ifdef __EMSCRIPTEN__
+// Pull this in to verify versions
+#include "util_normalize_table.h"
+#endif
+
//-------------------------------------------------------------------------------------
// Unicode version tests
//-------------------------------------------------------------------------------------
@@ -172,6 +182,42 @@ const std::string &block_unicode_ver) {
std::cout << std::endl;
}
+#ifdef __EMSCRIPTEN__
+inline const char *boolstr(bool b) {
+ return b?"T":"f";
+}
+
+void test_has_boundary_before() {
+ std::cout << "= " << __FUNCTION__ << std::endl;
+ std::cout << "I see we are on Emscripten / wasm! Now we will do some additional tests." << std::endl;
+ std::string icu4c_unicode(U_UNICODE_VERSION), header_unicode(KM_HASBOUNDARYBEFORE_UNICODE_VERSION),
+ icu4c_icu(U_ICU_VERSION), header_icu(KM_HASBOUNDARYBEFORE_ICU_VERSION);
+ std::cout << "Unicode: " << U_UNICODE_VERSION << ", and from the table file: " << KM_HASBOUNDARYBEFORE_UNICODE_VERSION << std::endl;
+ std::cout << "It would be very strange for these versions to be out of sync. Some sort of build or tool problem." << std::endl;
+ assert_basic_equal(icu4c_unicode, header_unicode);
+ assert_basic_equal(icu4c_icu, header_icu);
+
+ std::cout << std::endl << "Now, let's make sure has_nfd_boundary_before() matches ICU." << std::endl;
+
+ UErrorCode status = U_ZERO_ERROR;
+ const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(status);
+ UASSERT_SUCCESS(status);
+
+ // now, test that hasBoundaryBefore is the same
+ for (km_core_usv cp = 0; cp < km::core::kmx::Uni_MAX_CODEPOINT; cp++) {
+ auto km_hbb = km::core::util::has_nfd_boundary_before(cp);
+ auto icu_hbb = nfd->hasBoundaryBefore(cp);
+
+ if (km_hbb != icu_hbb) {
+ std::cerr << "Error: util_normalize_table.h said " << boolstr(km_hbb) << " but ICU said " << boolstr(icu_hbb) << " for "
+ << "has_nfd_boundary_before(0x" << std::hex << cp << std::dec << ")" << std::endl;
+ }
+ assert(km_hbb == icu_hbb);
+ }
+ std::cout << "All OK!" << std::endl;
+}
+#endif
+
int test_all(const char *jsonpath, const char *packagepath, const char *blockspath) {
std::cout << "= " << __FUNCTION__ << std::endl;
@@ -187,6 +233,10 @@ int test_all(const char *jsonpath, const char *packagepath, const char *blockspa
test_unicode_versions(versions, package, block_unicode_ver);
+#ifdef __EMSCRIPTEN__
+ test_has_boundary_before();
+#endif
+
return EXIT_SUCCESS;
}
diff --git a/developer/src/tike/xml/help/contexthelp.xml b/developer/src/tike/xml/help/contexthelp.xml
index 0c338aba98..4e3d9158e4 100644
--- a/developer/src/tike/xml/help/contexthelp.xml
+++ b/developer/src/tike/xml/help/contexthelp.xml
@@ -102,6 +102,18 @@
======================================================================== -->
+
@@ -139,6 +151,24 @@
This corresponds to the following source line:
store(&message) 'Here is a message about a keyboard'
+
+
+ The Keyboard Version documents the version of the keyboard.
+ A keyboard version should be updated whenever there are changes to a keyboard. The good principles to follow are:
+
+ - Increment the major version number for a
+ keyboard that has significant new functionality.
+ - Increment the minor version number for changes that impact functionality but not
+ in a significant manner.
+ - Optionally, use a third number for bug fixes.
+
+
+ This corresponds to the following source line:
+ store(&keyboardversion) '1.1.2'
+ Note: there is a difference between &keyboardversion, which documents the keyboard version, and &version,
+ which determines which version of Keyman a keyboard will run with.
+
+
In this field, enter information about the keyboard for your own reference. These comments will only be visible in the
source file, and not to users of your keyboard.
@@ -183,6 +213,41 @@
Tests your KeymanWeb keyboard in an Internet Explorer embedded window
+
+ The Features grid controls which additional file components are included in
+ the keyboard. Each of the features relates to a system store. Here are the file
+ components:
+
+ - Embedded JavaScript
+ - Embedded CSS
+ - Web Help
+ - Include Codes
+ - Desktop On-Screen Keyboard (auto-included if Targets is any)
+ - Touch-Optimised Keyboard (auto-included if Targets is any)
+
+ Icon will be automatically included when a new keyboard project is created.
+
+
+
+
+ This will open a selection dialog allowing you to choose a feature to add to
+ the keyboard project. Adding a feature will add an extra tab to the editor,
+ and add the corresponding store to the keyboard source
+
+
+
+ Depending on what is included in the Feature Grid, you can select a feature from
+ the grid then click on Edit... This will take you to the corresponding tab and
+ let you make changes.
+
+
+
+
+ Removing a feature will not delete the component file,
+ but will just remove the store from the keyboard source.
+
+
+
@@ -197,8 +262,40 @@
The toolbox allows for the changing of the colours, addition of text and shapes. It also allows for the moving of the icon around the canvas and also a preview of the icon is displayed.
+
+
+ Click on a colour from the box to apply the colour onto the Keyboard's
+ icon. You can see the Foreground Color displays the colour you chose. To deselect
+ the colour, click on the X mark on the corner left of the colour box, or choose
+ another colour.
+
+
+
+ This tab allows you to edit the visual representation of your keyboard
+ layout. The content on this tab is stored in the .kvks file associated with
+ your keyboard. The visual representation is used only in desktop and desktop
+ web; however if no touch layout is defined, this layout will be synthesized
+ into a touch layout automatically.
+
+ An On-Screen keyboard is optional but in most keyboards is recommended.
+ The On-Screen keyboard may not always match the actual layout identically,
+ because you may choose to hide some of the details of encoding from the
+ interface presented to the user.
+
+
+ This keyboard layout can also be printed or included in HTML or other documentation.
+ The editor allows you to export the file to HTML, PNG or BMP formats.
+
+
+
+
+ If this option is checked, when the Fill from layout button is clicked,
+ then keys without corresponding rules in the Layout will be filled with the
+ base layout character.
+
+
@@ -331,33 +428,185 @@
The debugger can be used without the debug information by clicking on Test without debugger.
-
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/developer/src/tike/xml/layoutbuilder/builder.xsl b/developer/src/tike/xml/layoutbuilder/builder.xsl
index 3b75cd81a3..d43d90c574 100644
--- a/developer/src/tike/xml/layoutbuilder/builder.xsl
+++ b/developer/src/tike/xml/layoutbuilder/builder.xsl
@@ -271,6 +271,10 @@
+
+