erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-08-11 11:25:34 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 26

Merge branch 'master' into change/web/drop-correction-batching

Browse source
This commit is contained in:
Marc Durdin 2024-06-18 07:49:26 +10:00 • committed by GitHub
commit 26328463c8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
54 changed files with 1925 additions and 457 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
HISTORY.md
View file

@ -1,5 +1,37 @@
# Keyman Version History
## 18.0.57 alpha 2024-06-17
* fix(web): fix id of longpress keys with modifier set in touch layout (#11783)
* fix(web): prevent desktop OSK crash when addKeyboards is called before engine init (#11786)
* fix(core): serialize tests for core/wasm on mac agents (#11795)
* fix(developer): refactor kmcmplib compiler messages to use map (#11738)
* fix(developer): make native compilation of kmcmplib under Linux possible (#11779)
* feat(core): devolve regex to javascript (#11777)
* feat(core): remove ICU from core under wasm (#11778)
* fix(linux): restart ibus after manual integration test run (#11775)
## 18.0.56 alpha 2024-06-14
* feat(core): devolve normalization to js (#11541)
* fix(developer): show message if no more platforms to add to touch layout editor (#11759)
* docs(developer): context help in package-editor and put the existing context help in their own tab comments (#11760)
* docs(developer): context help in keyboard-editor section (#11754)
* docs(developer): context help in new-project section (#11767)
* docs(developer): context help for new-project-parameters in keyman developer (#11769)
* docs(developer): context help for Select BCP 47 tag in Keyman Developer (#11770)
* change(common): update esbuild to 0.18.9 (#11693)
* change(web): more prep for better async prediction handling (#10347)
* fix(web): set new-context rules' device to match that of the active OSK (#11743)
* chore(linux): Update debian changelog (#11671)
* fix(web): add limited Array.from polyfill for lm-worker use (#11732)
## 18.0.55 alpha 2024-06-13
* fix(developer): handle missing OSK when importing a Windows keyboard into a touch-only project (#11720)
* fix(developer): verify email addresses in .kps and .keyboard_info (#11735)
* change(web): prep for better asynchronous prediction handling (#10343)
## 18.0.54 alpha 2024-06-12
* fix(common): remove subpackage entries for older TS version (#11745)

2
VERSION.md
View file

@ -1 +1 @@
18.0.55
18.0.58

2
common/models/wordbreakers/src/default/index.ts
View file

@ -274,7 +274,7 @@ export class BreakerContext {
* @param chunk a chunk of text. Starts and ends at word boundaries.
*/
function isNonSpace(chunk: string, options?: DefaultWordBreakerOptions): boolean {
return !Array.from(chunk).map((char) => property(char, options)).every(wb => (
return !chunk.split('').map((char) => property(char, options)).every(wb => (
wb === WordBreakProperty.CR ||
wb === WordBreakProperty.LF ||
wb === WordBreakProperty.Newline ||

3
common/web/lm-worker/build-polyfiller.js
View file

@ -144,6 +144,9 @@ let sourceFileSet = [
// Needed for Android / Chromium browser pre-41.
loadPolyfill('../../../node_modules/string.prototype.codepointat/codepointat.js', 'src/polyfills/string.codepointat.js'),
// Needed for Android / Chromium browser pre-45.
// Not used in this codebase, but used by some compiled model defaults.
loadPolyfill('src/polyfills/array.from.js', 'src/polyfills/array.from.js'),
// Needed for Android / Chromium browser pre-45.
loadPolyfill('src/polyfills/array.fill.js', 'src/polyfills/array.fill.js'),
// Needed for Android / Chromium browser pre-45.
loadPolyfill('src/polyfills/array.findIndex.js', 'src/polyfills/array.findIndex.js'),

44
common/web/lm-worker/src/polyfills/array.from.js Normal file
View file

@ -0,0 +1,44 @@
(function() {
if(!Array.from) {
function isHighSurrogate(codeUnit) {
codeUnit = codeUnit.charCodeAt(0);
return codeUnit >= 0xD800 && codeUnit <= 0xDBFF;
}
function isLowSurrogate(codeUnit) {
codeUnit = codeUnit.charCodeAt(0);
return codeUnit >= 0xDC00 && codeUnit <= 0xDFFF;
}
Array.from = function (obj) {
if(Array.isArray(obj)) {
// Simple array clone
return obj.slice();
} else if(typeof obj == 'string') {
// Array.from is surrogate-aware and will not split surrogate pairs.
// We can start with a full split and then remerge the pairs.
var simpleSplit = obj.split('');
/** @type {string[]} */
var finalSplit = [];
/** @type {number} */
var i;
while(simpleSplit.length > 0) {
// Do we have a surrogate pair?
var a = simpleSplit.shift();
if(isHighSurrogate(a) && (isLowSurrogate(simpleSplit[0] || ''))) {
// yes, so merge them before pushing.
a = a + simpleSplit.shift();
console.log(a);
} // else: 'no', so just push the current char to the array and continue
finalSplit.push(a);
}
return finalSplit;
} else {
throw "Unexpected + nonpolyfilled use of Array.from encountered; aborting";
}
}
}
}());

9
core/commands.inc.sh
View file

@ -91,7 +91,14 @@ do_test() {
if [[ $target =~ ^(x86|x64)$ ]]; then
cmd //C build.bat $target $BUILDER_CONFIGURATION test $testparams
else
meson test -C "$MESON_PATH" $testparams
if [[ $target == wasm ]] && [[ $BUILDER_OS == mac ]]; then
# 11794 -- parallel tests failing on some mac build agents; temporary
# mitigation until we diagnose root cause
meson test -j 1 -C "$MESON_PATH" $testparams
else
meson test -C "$MESON_PATH" $testparams
fi
fi
builder_finish_action success test:$target
}

146
core/src/actions_normalize.cpp
View file

@ -15,12 +15,12 @@
#include "state.hpp"
#include "option.hpp"
#include "debuglog.h"
#include "core_icu.h"
#include "util_normalize.hpp"
#include "utfcodec.hpp"
#include "kmx/kmx_xstring.h"
// forward declarations
icu::UnicodeString context_items_to_unicode_string(km::core::context const *context);
km_core_usv *unicode_string_to_usv(icu::UnicodeString& src);
// forward declaration
bool context_items_to_unicode_string(km::core::context const *context, std::u32string &str);
/**
* Normalize the output from an action to NFC, across the context | output
@ -65,32 +65,12 @@ bool km::core::actions_normalize(
cached_context.
*/
/*
Initialization
*/
UErrorCode icu_status = U_ZERO_ERROR;
const icu::Normalizer2 *nfc = icu::Normalizer2::getNFCInstance(icu_status);
assert(U_SUCCESS(icu_status));
if(!U_SUCCESS(icu_status)) {
DebugLog("getNFCInstance failed with %x", icu_status);
std::u32string output(actions.output);
std::u32string cached_context_string, app_context_string;
if (!context_items_to_unicode_string(cached_context, cached_context_string)) {
return false;
}
const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(icu_status);
assert(U_SUCCESS(icu_status));
if(!U_SUCCESS(icu_status)) {
DebugLog("getNFDInstance failed with %x", icu_status);
return false;
}
icu::UnicodeString output = icu::UnicodeString::fromUTF32(reinterpret_cast<const UChar32*>(actions.output), -1);
icu::UnicodeString cached_context_string = context_items_to_unicode_string(cached_context);
icu::UnicodeString app_context_string = context_items_to_unicode_string(app_context);
assert(!output.isBogus());
assert(!cached_context_string.isBogus());
assert(!app_context_string.isBogus());
if(output.isBogus() || cached_context_string.isBogus() || app_context_string.isBogus()) {
if (!context_items_to_unicode_string(app_context, app_context_string)) {
return false;
}
int nfu_to_delete = 0;
@ -98,20 +78,25 @@ bool km::core::actions_normalize(
/*
Further debug assertion of inputs
*/
assert(nfd->isNormalized(output, icu_status) && U_SUCCESS(icu_status));
assert(nfd->isNormalized(cached_context_string, icu_status) && U_SUCCESS(icu_status));
assert(km::core::util::is_nfd(output));
assert(km::core::util::is_nfd(cached_context_string));
/*
The keyboard processor will have updated the cached_context already,
applying the transform to it, so we need to rewind this. Remove the output
from cached_context_string to start
applying the transform to it, so we need to rewind this.
Assert that 'cached_context_string' ends with 'output'
Remove the output
from cached_context_string to start.
*/
assert(cached_context_string.length() >= output.length());
int n = cached_context_string.length() - output.length();
size_t n = cached_context_string.length() - output.length();
// auto end_cached = cached_context_string.substr(n, output.length());
// assert(end_cached == output);
assert(cached_context_string.compare(n, output.length(), output) == 0);
cached_context_string.remove(n);
cached_context_string.resize(n);
/*
While cached_context is guaranteed to be normalized, actions->output may not
@ -119,21 +104,22 @@ bool km::core::actions_normalize(
normalization in our output, we now need to look for a normalization
boundary prior to the intersection of the cached_context and the output.
*/
if(!output.isEmpty()) {
while(n > 0 && !nfd->hasBoundaryBefore(output[0])) {
if(!output.empty()) {
while(n > 0 && !km::core::util::has_nfd_boundary_before(output[0])) {
// The output may interact with the context further in normalization. We
// need to copy characters back further until we reach a normalization
// boundary.
// Remove last code point from the context ...
n = cached_context_string.moveIndex32(n, -1);
UChar32 chr = cached_context_string.char32At(n);
cached_context_string.remove(n);
auto len = cached_context_string.length();
assert(len>0);
auto chr = cached_context_string.at(len-1);
cached_context_string.resize(len-1);
// And prepend it to the output ...
output.insert(0, chr);
output.insert(0, 1, chr);
}
}
@ -148,25 +134,20 @@ bool km::core::actions_normalize(
its normalized form matches the cached_context normalized form.
*/
while(app_context_string.countChar32()) {
icu::UnicodeString app_context_nfd;
nfd->normalize(app_context_string, app_context_nfd, icu_status);
assert(U_SUCCESS(icu_status));
if(!U_SUCCESS(icu_status)) {
DebugLog("nfd->normalize failed with %x", icu_status);
while(!app_context_string.empty()) {
auto app_context_nfd = app_context_string;
if(!km::core::util::normalize_nfd(app_context_nfd)) {
DebugLog("nfd->normalize failed");
return false;
}
if(app_context_nfd.compare(cached_context_string) == 0) {
if(app_context_nfd == cached_context_string) {
break;
}
// remove the last UChar32
int32_t lastUChar32 = app_context_string.length()-1;
// adjust pointer to get the entire char (i.e. so we don't slice a non-BMP char)
lastUChar32 = app_context_string.getChar32Start(lastUChar32);
// remove the UChar32 (1 or 2 code units)
app_context_string.remove(lastUChar32);
size_t len = app_context_string.length();
// remove the cp at end
app_context_string.resize(len-1);
nfu_to_delete++;
}
@ -174,17 +155,15 @@ bool km::core::actions_normalize(
Normalize our output string
*/
icu::UnicodeString output_nfc;
nfc->normalize(output, output_nfc, icu_status);
assert(U_SUCCESS(icu_status));
if(!U_SUCCESS(icu_status)) {
DebugLog("nfc->normalize failed with %x", icu_status);
auto output_nfc = output;
if(!km::core::util::normalize_nfc(output_nfc)) {
DebugLog("nfc->normalize failed");
return false;
}
auto new_output = unicode_string_to_usv(output_nfc);
if(!new_output) {
// error logging handled in unicode_string_to_usv
auto new_output = km::core::util::string_to_usv(output_nfc);
assert(new_output != nullptr);
if(new_output == nullptr) {
return false;
}
@ -197,8 +176,8 @@ bool km::core::actions_normalize(
app_context_string.append(output_nfc);
km_core_context_item *app_context_items = nullptr;
km_core_status status = KM_CORE_STATUS_OK;
if((status = context_items_from_utf16(app_context_string.getTerminatedBuffer(), &app_context_items)) != KM_CORE_STATUS_OK) {
DebugLog("context_items_from_utf16 failed with %x", status);
if((status = context_items_from_utf32(app_context_string.c_str(), &app_context_items)) != KM_CORE_STATUS_OK) {
DebugLog("context_items_from_string failed with %x", status);
delete [] new_output;
return false;
}
@ -224,21 +203,19 @@ bool km::core::actions_normalize(
/**
* Helper to convert km_core_context list into a icu::UnicodeString
*/
icu::UnicodeString context_items_to_unicode_string(km::core::context const *context) {
icu::UnicodeString nullString;
nullString.setToBogus();
bool context_items_to_unicode_string(km::core::context const *context, std::u32string &str) {
km_core_context_item *items = nullptr;
km_core_status status;
if((status = km_core_context_get(static_cast<km_core_context const *>(context), &items)) != KM_CORE_STATUS_OK) {
DebugLog("Failed to retrieve context with %s", status);
return nullString;
return false;
}
size_t buf_size = 0;
if((status = context_items_to_utf32(items, nullptr, &buf_size)) != KM_CORE_STATUS_OK) {
DebugLog("Failed to retrieve context size with %s", status);
km_core_context_items_dispose(items);
return nullString;
return false;
}
km_core_usv *buf = new km_core_usv[buf_size];
@ -246,38 +223,15 @@ icu::UnicodeString context_items_to_unicode_string(km::core::context const *cont
DebugLog("Failed to retrieve context with %s", status);
km_core_context_items_dispose(items);
delete [] buf;
return nullString;
return false;
}
auto result = icu::UnicodeString::fromUTF32(reinterpret_cast<const UChar32*>(buf), -1);
str = std::u32string(buf, buf_size - 1); // don't include terminating null
km_core_context_items_dispose(items);
delete [] buf;
return result;
return true;
}
/**
* Helper to convert icu::UnicodeString to a UTF-32 km_core_usv buffer,
* nul-terminated
*/
km_core_usv *unicode_string_to_usv(icu::UnicodeString& src) {
UErrorCode icu_status = U_ZERO_ERROR;
km_core_usv *dst = new km_core_usv[src.length() + 1];
src.toUTF32(reinterpret_cast<UChar32*>(dst), src.length(), icu_status);
assert(U_SUCCESS(icu_status));
if(!U_SUCCESS(icu_status)) {
DebugLog("toUTF32 failed with %x", icu_status);
delete[] dst;
return nullptr;
}
dst[src.length()] = 0;
return dst;
}
/**
* Refresh app_context to match the cached_context. Does not do normalization,

24
core/src/context.hpp
View file

@ -87,6 +87,30 @@ km_core_status
context_items_from_utf16(km_core_cu const *text,
km_core_context_item **out_ptr);
/**
* Convert a UTF32 encoded Unicode string into an array of `km_core_context_item`
* structures. Allocates memory as needed.
*
* @return km_core_status
* * `KM_CORE_STATUS_OK`: On success.
* * `KM_CORE_STATUS_INVALID_ARGUMENT`: If non-optional parameters are
* null.
* * `KM_CORE_STATUS_NO_MEM`: In the event not enough memory can be
* allocated for the output buffer.
* * `KM_CORE_STATUS_INVALID_UTF`: In the event the UTF32 string cannot
* be decoded.
*
* @param text a pointer to a null terminated array of utf32 encoded data.
* @param out_ptr a pointer to the result variable: A pointer to the start of
* the `km_core_context_item` array containing the representation
* of the input string. Terminated with a type of
* `KM_CORE_CT_END`. Must be disposed of with
* `km_core_context_items_dispose`.
*/
km_core_status
context_items_from_utf32(km_core_usv const *text,
km_core_context_item **out_ptr);
/**
* Convert a context item array into a UTF-16 encoded string placing it into the
* supplied buffer of specified size, and return the number of code units

72
core/src/core_icu.cpp Normal file
View file

@ -0,0 +1,72 @@
/*
Copyright: © SIL International.
Description: Common LDML utilities
Create Date: 24 May 2024
Authors: Steven R. Loomis
*/
#include "core_icu.h"
#if !KMN_NO_ICU
namespace km {
namespace core {
namespace util {
/**
* Helper to convert icu::UnicodeString to a UTF-32 km_core_usv buffer,
* nul-terminated
*/
km_core_usv *unicode_string_to_usv(icu::UnicodeString& src) {
UErrorCode icu_status = U_ZERO_ERROR;
km_core_usv *dst = new km_core_usv[src.length() + 1];
src.toUTF32(reinterpret_cast<UChar32*>(dst), src.length(), icu_status);
if(!UASSERT_SUCCESS(icu_status)) {
DebugLog("toUTF32 failed with %x", icu_status);
delete[] dst;
return nullptr;
}
dst[src.length()] = 0;
return dst;
}
/**
* Internal function to normalize with a specified mode.
* Note: that this function _does_ assert failure, so it is not
* required to assert its return code. The return is provided so
* that callers can exit (such as making no change) if there was failure.
*
* Also note that "failure" here is something catastrophic: ICU not initialized,
* or, more likely, some low memory situation. Does not fail on "bad" data.
* @param n the ICU Normalizer to use
* @param str input/output string
* @param status error code, must be initialized on input
* @return false if failure
*/
bool normalize(const icu::Normalizer2 *n, std::u16string &str, UErrorCode &status) {
if(!UASSERT_SUCCESS(status)) {
return false;
}
assert(n != nullptr);
icu::UnicodeString dest;
icu::UnicodeString src = icu::UnicodeString(str.data(), (int32_t)str.length());
n->normalize(src, dest, status);
// the next line here will assert
if (!UASSERT_SUCCESS(status)) {
return false;
} else {
str.assign(dest.getBuffer(), dest.length());
return true;
}
}
}
}
} /* end km::core::util */
#endif

64
core/src/core_icu.h
View file

@ -3,15 +3,42 @@
*/
#pragma once
#ifdef __EMSCRIPTEN__
// define this in tests to keep ICU around
# if !defined(KMN_IN_LDML_TESTS)
# if !defined(KMN_NO_ICU)
// under wasm, turn off ICU except in tests.
# define KMN_NO_ICU 1
# endif
# endif
#elif !defined(KMN_NO_ICU)
# define KMN_NO_ICU 0
#endif
#if KMN_NO_ICU
// NO ICU
// any shims needed here for disabling ICU
#else
// YES ICU
#if !defined(HAVE_ICU4C)
#error icu4c is required for this code
# error icu4c is required for this code
#endif
#define U_FALLTHROUGH
#include "unicode/utypes.h"
#include "unicode/unistr.h"
#include "unicode/normalizer2.h"
#include "unicode/uniset.h"
#include "unicode/usetiter.h"
#include "unicode/regex.h"
#include "unicode/utext.h"
#include "keyman_core.h"
#include "debuglog.h"
#include <assert.h>
@ -29,5 +56,40 @@ inline bool uassert_success(const char *file, int line, const char *function, UE
/**
* Assert an ICU4C UErrorCode
* the first assert is for debug builds, the second triggers the debuglog and has the return value.
* @returns true on success
* */
#define UASSERT_SUCCESS(status) (assert(U_SUCCESS(status)), uassert_success(__FILE__, __LINE__, __FUNCTION__, status))
// ------------------ some ICU C++ utilities ----------------------------
namespace km {
namespace core {
namespace util {
/**
* Convert a UnicodeString to a km_core_usv array
* @return the 0-terminated array. Caller owns storage.
*/
km_core_usv *unicode_string_to_usv(icu::UnicodeString& src);
/**
* Internal function to normalize with a specified mode.
* Note: that this function _does_ assert failure, so it is not
* required to assert its return code. The return is provided so
* that callers can exit (such as making no change) if there was failure.
*
* Also note that "failure" here is something catastrophic: ICU not initialized,
* or, more likely, some low memory situation. Does not fail on "bad" data.
* @param n the ICU Normalizer to use
* @param str input/output string
* @param status error code, must be initialized on input
* @return false if failure
*/
bool normalize(const icu::Normalizer2 *n, std::u16string &str, UErrorCode &status);
}
}
}
#endif /* KMN_NO_ICU */

7
core/src/km_core_context_api.cpp
View file

@ -115,6 +115,13 @@ context_items_from_utf16(km_core_cu const *text,
}
km_core_status
context_items_from_utf32(km_core_usv const *text,
km_core_context_item **out_ptr)
{
return _context_items_from<utf32>(reinterpret_cast<utf32::codeunit_t const *>(text), out_ptr);
}
km_core_status context_items_to_utf8(km_core_context_item const *ci,
char *buf, size_t * sz_ptr)
{

14
core/src/kmx/kmx_xstring.cpp
View file

@ -50,6 +50,15 @@ size_t km::core::kmx::u16len(const km_core_cu *p) {
return i;
}
size_t km::core::kmx::u32len(const km_core_usv *p) {
int i = 0;
while (*p) {
p++;
i++;
}
return i;
}
int km::core::kmx::u16cmp(const km_core_cu *p, const km_core_cu *q) {
while (*p && *q) {
if (*p != *q) return *p - *q;
@ -107,6 +116,11 @@ km_core_cu *km::core::kmx::u16dup(km_core_cu *src) {
memcpy(dup, src, (u16len(src) + 1) * sizeof(km_core_cu));
return dup;
}
km_core_usv *km::core::kmx::u32dup(const km_core_usv *src) {
km_core_usv *dup = new km_core_usv[u32len(src) + 1];
memcpy(dup, src, (u32len(src) + 1) * sizeof(src[0]));
return dup;
}
/*
* int xstrlen( PKMX_BYTE p );

3
core/src/kmx/kmx_xstring.h
View file

@ -117,6 +117,9 @@ int u16ncmp(const km_core_cu *p, const km_core_cu *q, size_t count);
km_core_cu *u16tok(km_core_cu *p, km_core_cu ch, km_core_cu **ctx);
km_core_cu *u16dup(km_core_cu *src);
size_t u32len(const km_core_usv *p);
km_core_usv *u32dup(const km_core_usv *src);
//KMX_BOOL MapUSCharToVK(KMX_WORD ch, PKMX_WORD puKey, PKMX_DWORD puShiftFlags);
// --- implementation ---

35
core/src/ldml/ldml_markers.cpp
View file

@ -268,15 +268,14 @@ add_pending_markers(
marker_map *markers,
marker_list &last_markers,
const std::u32string::const_iterator &last,
const std::u32string::const_iterator &end,
const icu::Normalizer2 *nfd) {
const std::u32string::const_iterator &end) {
// quick check to see if there's no work to do.
if(markers == nullptr) {
return;
}
/** which character this marker is 'glued' to. */
char32_t marker_ch;
icu::UnicodeString decomposition;
std::u32string decomposition;
if (last == end) {
// at end of text, so use a special value to indicate 'EOT'.
marker_ch = MARKER_BEFORE_EOT;
@ -285,17 +284,13 @@ add_pending_markers(
// if the character is composed, we need to use the first decomposed char
// as the 'glue'.
if(!nfd->getDecomposition(ch, decomposition)) {
if(!km::core::util::normalize_nfd(ch, decomposition)) {
// char does not have a decomposition - so it may be used for the glue
marker_ch = ch;
decomposition.remove(); // no other entries needed
} else {
// 'glue' is the first codepoint of the decomposition.
marker_ch = decomposition.char32At(0);
if (decomposition.countChar32() == 1) {
decomposition.remove(); // no other entries needed
} // else: will add the remainder below
// the 'if' is only for the assertions here.
assert(decomposition.length() == 1); // should be a single UTF-32 char
assert(decomposition.at(0) == ch); // should be the same char
}
marker_ch = decomposition.at(0); // always the first char
}
markers->emplace_back(marker_ch);
// now, update the map with these markers (in order) on this character.
@ -304,10 +299,10 @@ add_pending_markers(
markers->emplace_back(marker_ch, *i);
}
// add any further entries due to decomposition
if (!decomposition.isEmpty()) {
// We already added the base char above, add teh rest
for (auto i=1; i<decomposition.countChar32(); i++) {
markers->emplace_back(decomposition.char32At(i));
if (decomposition.length() > 1) {
// We already added the base char above, add the rest
for (size_t i=1; i<decomposition.length(); i++) {
markers->emplace_back(decomposition.at(i));
}
}
// clear the list
@ -318,10 +313,6 @@ std::u32string
remove_markers(const std::u32string &str, marker_map *markers, marker_encoding encoding) {
std::u32string out;
marker_list last_markers;
UErrorCode status = U_ZERO_ERROR;
const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(status);
UASSERT_SUCCESS(status);
auto last = str.begin(); // points to the part of the string after the last matched marker
for (auto i = str.begin(); i != str.end();) {
auto marker_no = parse_next_marker(i, str.end(), encoding);
@ -329,7 +320,7 @@ remove_markers(const std::u32string &str, marker_map *markers, marker_encoding e
// add any markers found before this entry, but only if there is intervening
// text. This prevents the sentinel or the '\u' from becoming the attachment char.
if (i != last) {
add_pending_markers(markers, last_markers, last, str.end(), nfd);
add_pending_markers(markers, last_markers, last, str.end());
out.append(last, i); // append any non-marker text since the end of the last marker
last = i; // advance over text we've already appended
}
@ -347,7 +338,7 @@ remove_markers(const std::u32string &str, marker_map *markers, marker_encoding e
// add any remaining pending markers.
// if last == str.end() then this wil be MARKER_BEFORE_EOT
// otherwise it will be the glue character
add_pending_markers(markers, last_markers, last, str.end(), nfd);
add_pending_markers(markers, last_markers, last, str.end());
// get the suffix between the last marker and the end (could be nothing)
out.append(last, str.end());
return out;

4
core/src/ldml/ldml_markers.hpp
View file

@ -17,10 +17,6 @@
#include "debuglog.h"
#include "core_icu.h"
#include "unicode/uniset.h"
#include "unicode/usetiter.h"
#include "unicode/regex.h"
#include "unicode/utext.h"
namespace km {
namespace core {

174
core/src/ldml/ldml_transforms.cpp
View file

@ -437,20 +437,14 @@ reorder_group::apply(std::u32string &str) const {
}
transform_entry::transform_entry(const transform_entry &other)
: fFrom(other.fFrom), fTo(other.fTo), fFromPattern(nullptr), fMapFromStrId(other.fMapFromStrId),
: fFrom(other.fFrom), fTo(other.fTo), fFromPattern(other.fFromPattern), fMapFromStrId(other.fMapFromStrId),
fMapToStrId(other.fMapToStrId), fMapFromList(other.fMapFromList), fMapToList(other.fMapToList),
normalization_disabled(other.normalization_disabled) {
if (other.fFromPattern) {
// clone pattern
fFromPattern.reset(other.fFromPattern->clone());
}
}
transform_entry::transform_entry(const std::u32string &from, const std::u32string &to)
: fFrom(from), fTo(to), fFromPattern(nullptr), fMapFromStrId(), fMapToStrId(), fMapFromList(), fMapToList(), normalization_disabled(false) {
: fFrom(from), fTo(to), fFromPattern(from), fMapFromStrId(), fMapToStrId(), fMapFromList(), fMapToList(), normalization_disabled(false) {
assert(!fFrom.empty());
init();
}
transform_entry::transform_entry(
@ -461,7 +455,7 @@ transform_entry::transform_entry(
const kmx::kmx_plus &kplus,
bool &valid,
bool norm_disabled)
: fFrom(from), fTo(to), fFromPattern(nullptr), fMapFromStrId(mapFrom), fMapToStrId(mapTo), normalization_disabled(norm_disabled) {
: fFrom(from), fTo(to), fFromPattern(), fMapFromStrId(mapFrom), fMapToStrId(mapTo), normalization_disabled(norm_disabled) {
if (!valid)
return; // exit early
assert(!fFrom.empty()); // TODO-LDML: should not happen?
@ -469,7 +463,12 @@ transform_entry::transform_entry(
assert(kplus.strs != nullptr);
assert(kplus.vars != nullptr);
assert(kplus.elem != nullptr);
if(!init()) {
std::u32string from2 = fFrom;
if (!normalization_disabled) {
// normalize, including markers, for regex
normalize_nfd_markers(from2, regex_sentinel);
}
if (!fFromPattern.init(from2)) {
valid = false;
}
@ -506,160 +505,17 @@ transform_entry::transform_entry(
}
}
bool
transform_entry::init() {
if (fFrom.empty()) {
return false;
}
// TODO-LDML: if we have mapFrom, may need to do other processing.
std::u32string from2 = fFrom;
if (!normalization_disabled) {
// normalize, including markers, for regex
normalize_nfd_markers(from2, regex_sentinel);
}
std::u16string patstr = km::core::kmx::u32string_to_u16string(from2);
UErrorCode status = U_ZERO_ERROR;
/* const */ icu::UnicodeString patustr = icu::UnicodeString(patstr.data(), (int32_t)patstr.length());
// add '$' to match to end
patustr.append(u'$'); // TODO-LDML: may need to escape some markers. Marker #91 will look like a `[` to the pattern
fFromPattern.reset(icu::RegexPattern::compile(patustr, 0, status));
return (UASSERT_SUCCESS(status));
}
size_t
transform_entry::apply(const std::u32string &input, std::u32string &output) const {
assert(fFromPattern);
// TODO-LDML: Really? can't go from u32 to UnicodeString?
// TODO-LDML: Also, we could cache the u16 string at the transformGroup level or higher.
UErrorCode status = U_ZERO_ERROR;
const std::u16string matchstr = km::core::kmx::u32string_to_u16string(input);
icu::UnicodeString matchustr = icu::UnicodeString(matchstr.data(), (int32_t)matchstr.length());
// TODO-LDML: create a new Matcher every time. These could be cached and reset.
std::unique_ptr<icu::RegexMatcher> matcher(fFromPattern->matcher(matchustr, status));
if (!UASSERT_SUCCESS(status)) {
return 0; // TODO-LDML: return error
}
if (!matcher->find(status)) { // i.e. matches somewhere, in this case at end of str
return 0; // no match
}
// TODO-LDML: this is UTF-16 len, not UTF-32 len!!
// TODO-LDML: if we had an underlying UText this would be simpler.
int32_t matchStart = matcher->start(status);
int32_t matchEnd = matcher->end(status);
if (!UASSERT_SUCCESS(status)) {
return 0; // TODO-LDML: return error
}
// extract..
const icu::UnicodeString substr = matchustr.tempSubStringBetween(matchStart, matchEnd);
// preflight to UTF-32 to get length
UErrorCode substrStatus = U_ZERO_ERROR; // throwaway status
// we need the UTF-32 matchLen for our return.
auto matchLen = substr.toUTF32(nullptr, 0, substrStatus);
// should have matched something.
assert(matchLen > 0);
// now, do the replace.
/** this is the 'to' or other replacement string.*/
icu::UnicodeString rustr;
if (fMapFromStrId == 0) {
// Normal case: not a map.
// This replace will apply $1, $2 etc.
// Convert the fTo into u16 TODO-LDML (we could cache this?)
const std::u16string rstr = km::core::kmx::u32string_to_u16string(fTo);
rustr = icu::UnicodeString(rstr.data(), (int32_t)rstr.length());
} else {
// Set map case: mapping from/to
// we actually need the group(1) string here.
// this is only the content in parenthesis ()
icu::UnicodeString group1 = matcher->group(1, status);
if (!UASSERT_SUCCESS(status)) {
// TODO-LDML: could be a malformed from pattern
return 0; // TODO-LDML: return error
}
// now, how long is group1 in UTF-32, hmm?
UErrorCode preflightStatus = U_ZERO_ERROR; // throwaway status
auto group1Len = group1.toUTF32(nullptr, 0, preflightStatus);
char32_t *s = new char32_t[group1Len + 1];
assert(s != nullptr); // TODO-LDML: OOM
// convert
group1.toUTF32((UChar32 *)s, group1Len + 1, status);
if (!UASSERT_SUCCESS(status)) {
return 0; // TODO-LDML: memory issue
}
std::u32string match32(s, group1Len); // taken from just group1
// clean up buffer
delete [] s;
// Now we're ready to do the actual mapping.
// 1., we need to find the index in the source set.
auto matchIndex = findIndexFrom(match32);
assert(matchIndex != -1L); // TODO-LDML: not matching shouldn't happen, the regex wouldn't have matched.
// we already asserted on load that the from and to sets have the same cardinality.
// 2. get the target string, convert to utf-16
// we use the same matchIndex that was just found
const std::u16string rstr = km::core::kmx::u32string_to_u16string(fMapToList.at(matchIndex));
// 3. update the UnicodeString for replacement
rustr = icu::UnicodeString(rstr.data(), (int32_t)rstr.length());
// and we return to the regular code flow.
}
// here we replace the match output. No normalization, yet.
icu::UnicodeString entireOutput = matcher->replaceFirst(rustr, status);
if (!UASSERT_SUCCESS(status)) {
// TODO-LDML: could fail here due to bad input (syntax err)
return 0;
}
// entireOutput includes all of 'input', but modified. Need to substring it.
icu::UnicodeString outu = entireOutput.tempSubString(matchStart);
// Special case if there's no output, save some allocs
if (outu.length() == 0) {
output.clear();
} else {
// TODO-LDML: All we are trying to do is to extract the output string. Probably too many steps.
UErrorCode preflightStatus = U_ZERO_ERROR;
// calculate how big the buffer is
auto out32len = outu.toUTF32(nullptr, 0, preflightStatus); // preflightStatus will be an err, because we know the buffer overruns zero bytes
// allocate
std::unique_ptr<char32_t[]> s(new char32_t[out32len + 1]);
assert(s);
if (!s) {
return 0; // TODO-LDML: allocation failed
}
// convert
outu.toUTF32((UChar32 *)(s.get()), out32len + 1, status);
if (!UASSERT_SUCCESS(status)) {
return 0; // TODO-LDML: memory issue
}
output.assign(s.get(), out32len);
// NOW do a marker-safe normalize
if (!normalization_disabled && !normalize_nfd_markers(output)) {
auto result = fFromPattern.apply(input, output, fTo, fMapFromList, fMapToList);
// NOW do a marker-safe normalize
if (result != 0 && !output.empty() && !normalization_disabled) {
if (!normalize_nfd_markers(output)) {
DebugLog("normalize_nfd_markers(output) failed");
return 0; // TODO-LDML: normalization failed.
return 0; // TODO-LDML: normalization failed.
}
}
return matchLen;
}
int32_t transform_entry::findIndexFrom(const std::u32string &match) const {
return findIndex(match, fMapFromList);
}
int32_t transform_entry::findIndex(const std::u32string &match, const std::deque<std::u32string> list) {
int32_t index = 0;
for(auto e = list.begin(); e < list.end(); e++, index++) {
if (match == *e) {
return index;
}
}
return -1; // not found
return result;
}
any_group::any_group(const transform_group &g) : type(any_group_type::transform), transform(g), reorder() {

10
core/src/ldml/ldml_transforms.hpp
View file

@ -16,11 +16,7 @@
#include <utility>
#include "debuglog.h"
#include "core_icu.h"
#include "unicode/uniset.h"
#include "unicode/usetiter.h"
#include "unicode/regex.h"
#include "unicode/utext.h"
#include "util_regex.hpp"
namespace km {
namespace core {
@ -111,14 +107,12 @@ public:
private:
const std::u32string fFrom;
const std::u32string fTo;
std::unique_ptr<icu::RegexPattern> fFromPattern;
km::core::util::km_regex fFromPattern;
const KMX_DWORD fMapFromStrId;
const KMX_DWORD fMapToStrId;
std::deque<std::u32string> fMapFromList;
std::deque<std::u32string> fMapToList;
/** Internal function to setup pattern string @returns true on success */
bool init();
bool normalization_disabled;
/** @returns the index of the item in the fMapFromList list, or -1 */
int32_t findIndexFrom(const std::u32string &match) const;

44
core/src/meson.build
View file

@ -16,11 +16,6 @@ if cpp_compiler.get_id() == 'msvc'
version_res += import('windows').compile_resources('version.rc', args:['/n','/c65001'])
endif
if cpp_compiler.get_id() == 'emscripten'
# TODO: why do we need this defn here?
defns += ['-DKM_CORE_LIBRARY']
endif
# ICU4C is used for repertoire tests and core implementation
if target_machine.system() == 'linux'
@ -36,10 +31,42 @@ else
icu_i18n = icu4c.get_variable('icui18n_dep')
endif
if cpp_compiler.get_id() == 'emscripten'
# TODO: why do we need this defn here?
defns += ['-DKM_CORE_LIBRARY']
icu_if_not_on_wasm = []
else
# only include this if NOT on wasm.
icu_if_not_on_wasm = [icu_uc, icu_i18n]
endif
if icu_uc.found()
defns += '-DHAVE_ICU4C'
endif
# On wasm, generate util_normalize_table.h automatically from ICU
generated_headers = []
if cpp_compiler.get_id() == 'emscripten'
util_normalize_table_generator = executable('util_normalize_table_generator',
['util_normalize_table_generator.cpp'],
cpp_args: defns + warns,
include_directories: [inc],
link_args: links,
dependencies: [icu_uc, icu_i18n],
)
util_normalize_table_h = custom_target('util_normalize_table.h',
output: 'util_normalize_table.h',
command: [util_normalize_table_generator],
capture:true)
generated_headers += util_normalize_table_h
endif
kmx_files = files(
'actions_normalize.cpp',
@ -60,6 +87,8 @@ kmx_files = files(
'km_core_processevent_api.cpp',
'jsonpp.cpp',
'util_normalize.cpp',
'util_regex.cpp',
'core_icu.cpp',
'ldml/ldml_processor.cpp',
'ldml/ldml_transforms.cpp',
'ldml/ldml_markers.cpp',
@ -110,18 +139,19 @@ lib = library('keymancore',
kmx_files,
mock_files,
version_res,
generated_headers,
cpp_args: defns + warns + flags,
link_args: links,
version: lib_version,
include_directories: inc,
pic: true,
install: true,
dependencies: [icu_uc, icu_i18n],
dependencies: icu_if_not_on_wasm,
)
headerdirs = [ '.', 'keyman' ] # subdirectories of ${prefix}/include to add to header path
keymancore = declare_dependency(link_with: lib, include_directories: inc, dependencies: [icu_uc, icu_i18n])
keymancore = declare_dependency(link_with: lib, include_directories: inc, dependencies: icu_if_not_on_wasm)
pkg = import('pkgconfig')
pkg.generate(

199
core/src/util_normalize.cpp
View file

@ -13,6 +13,7 @@
#ifdef __EMSCRIPTEN__
#include <emscripten.h>
#include "utfcodec.hpp"
#include <assert.h>
// JS implementations
EM_JS(char*, NormalizeNFD, (const char* input), {
@ -21,6 +22,17 @@ EM_JS(char*, NormalizeNFD, (const char* input), {
const nfd = instr.normalize("NFD");
return stringToNewUTF8(nfd);
});
EM_JS(char*, NormalizeNFC, (const char* input), {
if (!input) return input; // pass through null
const instr = Module.UTF8ToString(input);
const nfd = instr.normalize("NFC");
return stringToNewUTF8(nfd);
});
// pull in the generated table
#include "util_normalize_table.h"
#endif
namespace km {
@ -28,31 +40,15 @@ namespace core {
namespace util {
#ifndef __EMSCRIPTEN__
/**
* Internal function to normalize with a specified mode.
* Note: that this function _does_ assert failure, so it is not
* required to assert its return code. The return is provided so
* that callers can exit (such as making no change) if there was failure.
*
* Also note that "failure" here is something catastrophic: ICU not initialized,
* or, more likely, some low memory situation. Does not fail on "bad" data.
* @param n the ICU Normalizer to use
* @param str input/output string
* @param status error code, must be initialized on input
* @return false if failure
*/
static bool normalize(const icu::Normalizer2 *n, std::u16string &str, UErrorCode &status) {
inline const icu::Normalizer2 *getNFD(UErrorCode &status) {
const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(status);
UASSERT_SUCCESS(status);
assert(n != nullptr);
icu::UnicodeString dest;
icu::UnicodeString src = icu::UnicodeString(str.data(), (int32_t)str.length());
n->normalize(src, dest, status);
// the next line here will assert
if (UASSERT_SUCCESS(status)) {
str.assign(dest.getBuffer(), dest.length());
}
return U_SUCCESS(status);
return nfd;
}
inline const icu::Normalizer2 *getNFC(UErrorCode &status) {
const icu::Normalizer2 *nfc = icu::Normalizer2::getNFCInstance(status);
UASSERT_SUCCESS(status);
return nfc;
}
#endif
@ -66,6 +62,16 @@ bool normalize_nfd(std::u32string &str) {
}
}
bool normalize_nfc(std::u32string &str) {
std::u16string rstr = km::core::kmx::u32string_to_u16string(str);
if(!km::core::util::normalize_nfc(rstr)) {
return false;
} else {
str = km::core::kmx::u16string_to_u32string(rstr);
return true;
}
}
bool normalize_nfd(std::u16string &str) {
#ifdef __EMSCRIPTEN__
std::string instr = convert<char16_t,char>(str);
@ -81,9 +87,26 @@ bool normalize_nfd(std::u16string &str) {
return true;
#else
UErrorCode status = U_ZERO_ERROR;
const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(status);
UASSERT_SUCCESS(status);
return normalize(nfd, str, status);
return normalize(getNFD(status), str, status);
#endif
}
bool normalize_nfc(std::u16string &str) {
#ifdef __EMSCRIPTEN__
std::string instr = convert<char16_t,char>(str);
const char *in = instr.c_str();
char *out = NormalizeNFC(in);
if (out == nullptr) {
assert(out != nullptr);
return false;
}
std::string outstr(out);
str = convert<char,char16_t>(outstr);
free(out);
return true;
#else
UErrorCode status = U_ZERO_ERROR;
return normalize(getNFC(status), str, status);
#endif
}
@ -91,26 +114,130 @@ bool normalize_nfd(std::u16string &str) {
* Normalize the input string using ICU, out of place
*/
bool normalize_nfd(km_core_cu const * src, std::u16string &dst) {
UErrorCode icu_status = U_ZERO_ERROR;
const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(icu_status);
assert(U_SUCCESS(icu_status));
if(!U_SUCCESS(icu_status)) {
// TODO: log the failure code
#ifdef __EMSCRIPTEN__
dst = std::u16string(src);
return normalize_nfd(dst); // vector to above fcn
#else
UErrorCode status = U_ZERO_ERROR;
auto nfd = getNFD(status);
if (nfd == nullptr) {
return false;
}
icu::UnicodeString udst;
icu::UnicodeString usrc = icu::UnicodeString(src);
nfd->normalize(usrc, udst, icu_status);
assert(U_SUCCESS(icu_status));
if(!U_SUCCESS(icu_status)) {
// TODO: log the failure code
nfd->normalize(usrc, udst, status);
if(!UASSERT_SUCCESS(status)) {
return false;
}
dst.assign(udst.getBuffer(), udst.length());
return true;
#endif
}
bool
normalize_nfd(km_core_usv cp, std::u32string &dst) {
// set the output string to the original string
dst.clear();
dst.append(1, cp);
#ifdef __EMSCRIPTEN__
auto str16 = convert<char32_t, char16_t>(dst);
if (!normalize_nfd(str16)) {
return false; // failed, retain original str
} else {
dst = convert<char16_t, char32_t>(str16);
return true;
}
#else
UErrorCode icu_status = U_ZERO_ERROR;
const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(icu_status);
assert(U_SUCCESS(icu_status));
if (!U_SUCCESS(icu_status)) {
// TODO: log the failure code
return false;
}
icu::UnicodeString decomposition;
if (!nfd->getDecomposition(cp, decomposition)) {
return false; // no error, just no decomposition
} else {
dst.clear();
auto len = decomposition.countChar32();
for (int i = 0; i < len; i++) {
dst.append(1, decomposition.char32At(i));
}
return true;
}
#endif
}
bool is_nfd(const std::u16string& str) {
#ifdef __EMSCRIPTEN__
std::u16string o = str;
normalize_nfd(o);
return (o == str); // false if changed
#else
UErrorCode status = U_ZERO_ERROR;
auto nfd = getNFD(status);
if (nfd == nullptr) return false;
auto ustr = icu::UnicodeString(false, str.c_str(), (int)str.length());
auto result = nfd->isNormalized(ustr, status);
if (!UASSERT_SUCCESS(status)) {
return false;
} else {
return result;
}
#endif
}
bool is_nfd(const std::u32string& str) {
#ifdef __EMSCRIPTEN__
std::u32string o = str;
normalize_nfd(o);
return (o == str); // false if changed
#else
UErrorCode status = U_ZERO_ERROR;
auto nfd = getNFD(status);
if (nfd == nullptr) return false;
auto ustr = icu::UnicodeString::fromUTF32(reinterpret_cast<const UChar32*>(str.c_str()), (int)str.length());
auto result = nfd->isNormalized(ustr, status);
if (!UASSERT_SUCCESS(status)) {
return false;
} else {
return result;
}
#endif
}
bool has_nfd_boundary_before(km_core_usv cp) {
#ifdef __EMSCRIPTEN__
// it's a negative table. entries in the table mean returning false. non-entries return true.
for (auto i=0;i<(km_noBoundaryBefore_entries*2);i+=2) {
auto start = km_noBoundaryBefore[i+0];
if (start > cp) return true;
auto count = km_noBoundaryBefore[i+1];
auto limit = start+count;
if (cp >= start && cp < limit) return false;
}
return true; // fallthrough
#else
UErrorCode status = U_ZERO_ERROR;
auto nfd = getNFD(status);
if (nfd == nullptr) return false;
return nfd->hasBoundaryBefore(cp);
#endif
}
/**
* Helper to convert std::u32string to a UTF-32 km_core_usv buffer,
* nul-terminated.
* Parallel to unicode_string_to_usv()
* @returns new buffer, caller owns storage
*/
km_core_usv *string_to_usv(const std::u32string& src) {
return km::core::kmx::u32dup(src.c_str());
}
}
}
}

21
core/src/util_normalize.hpp
View file

@ -14,6 +14,12 @@ namespace km {
namespace core {
namespace util {
/** Normalize a u32string inplace to NFC. @return false on failure */
bool normalize_nfc(std::u32string &str);
/** Normalize a u16string inplace to NFC. @return false on failure */
bool normalize_nfc(std::u16string &str);
/** Normalize a u32string inplace to NFD. @return false on failure */
bool normalize_nfd(std::u32string &str);
@ -23,6 +29,21 @@ bool normalize_nfd(std::u16string &str);
/** normalize src to dst in NFD. @return false on failure */
bool normalize_nfd(km_core_cu const * src, std::u16string &dst);
/** normalize (decompose) a single cp to string. @return false on failure */
bool normalize_nfd(km_core_usv cp, std::u32string &dst);
/** @return true if string is already NFD */
bool is_nfd(const std::u16string& str);
/** @return true if string is already NFD */
bool is_nfd(const std::u32string& str);
/** @return true if cp can interacts with prior chars */
bool has_nfd_boundary_before(km_core_usv cp);
/** convenience function, caller owns storage */
km_core_usv *string_to_usv(const std::u32string& src);
}
}
}

108
core/src/util_normalize_table_generator.cpp Normal file
View file

@ -0,0 +1,108 @@
/*
Copyright: © SIL International.
Description: Generator for util_normalize_table.h
Create Date: 5 Jun 2024
Authors: Steven R. Loomis
util_normalize_table.h is used under wasm by utilities in util_normalize.cpp to implement
normalization functions without needing ICU4C linked.
This generator is invoked automatically by meson as part of the build.
*/
#include "kmx/kmx_plus.h"
#include "kmx/kmx_xstring.h"
#define KMN_NO_ICU 0 // we will need ICU..
#include "core_icu.h"
#include <fstream>
#include <iostream>
#include <string>
#include <vector>
#include <assert.h>
#include <unicode/uchar.h>
int
write_nfd_table() {
#ifndef __EMSCRIPTEN__
std::cerr << "Note: This is unusual - this generator is usually only run under emscripten!" << std::endl;
#endif
// We write to stdout instead of to a file to avoid dealing with the filesystem under emscripten.
std::cerr << "Writing to stdout." << std::endl;
// write preamble
std::cout << "// GENERATED FILE: DO NOT EDIT" << std::endl;
std::cout << "//" << std::endl;
std::cout << "// util_normalize_table.h is generated by util_normalize_table_generator.cpp" << std::endl;
std::cout << "// and used by util_normalize.cpp" << std::endl;
std::cout << std::endl;
std::cout << "#pragma once" << std::endl;
std::cout << "#define KM_HASBOUNDARYBEFORE_UNICODE_VERSION \"" << U_UNICODE_VERSION << "\"" << std::endl;
std::cout << "#define KM_HASBOUNDARYBEFORE_ICU_VERSION \"" << U_ICU_VERSION << "\"" << std::endl;
std::cout << std::endl;
// we're going to need an NFD normalizer
UErrorCode status = U_ZERO_ERROR;
const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(status);
assert(U_SUCCESS(status));
// collect the raw list of chars that do NOT have a boundary before them.
std::vector<km_core_usv> noBoundary;
for (km_core_usv ch = 0; ch < km::core::kmx::Uni_MAX_CODEPOINT; ch++) {
bool bb = nfd->hasBoundaryBefore(ch);
assert(!(ch == 0 && !bb)); // assert that we can use U+0000 as a terminator
if (bb) continue; //only emit nonboundary
noBoundary.push_back(ch);
}
// now, compress these into runs
std::vector<std::pair<km_core_usv,std::size_t>> runs; // start,len
km_core_usv first = 0;
km_core_usv last = 0;
for(auto i = noBoundary.begin(); i <= noBoundary.end(); i++) {
if (first == 0) {
first = last = *i;
} else {
last++;
if(i == noBoundary.end() || *i != last) {
// end of a run
runs.emplace_back(first, last - first);
if (i != noBoundary.end()) {
// setup for next
first = last = *i;
}
}
}
}
// finally, write out metadata and the runs themselves.
std::cout << "#define km_noBoundaryBefore_entries " << runs.size() << "\n";
std::cout << "static char32_t km_noBoundaryBefore[km_noBoundaryBefore_entries * 2 ] = {" << std::endl;
std::cout << "/* start codepoint, count (inclusive), ...range end */" << std::endl;
for (auto i = runs.begin(); i < runs.end(); i++) {
std::cout << "\t0x" << std::hex << i->first << std::dec << ",\t " << i->second << ", // ...0x" << std::hex << (i->first+i->second-1) << std::endl;
}
// termination
std::cout << "};" << std::endl;
std::cout << "// end" << std::endl;
std::cerr << "Wrote " << runs.size() << " runs representing " << noBoundary.size() << " entries." << std::endl;
return 0;
}
int
main(int /*argc*/, const char * /*argv*/[]) {
write_nfd_table();
return 0;
}

352
core/src/util_regex.cpp Normal file
View file

@ -0,0 +1,352 @@
/*
Copyright: © SIL International.
Description: Core Regex Utilities - abstract out ICU dependencies
Create Date: 5 Jun 2024
Authors: Steven R. Loomis
*/
#include "util_regex.hpp"
#include "core_icu.h"
#include "kmx/kmx_xstring.h"
#ifdef __EMSCRIPTEN__
#include <emscripten.h>
#include "utfcodec.hpp"
#include <assert.h>
// JS implementations
/**
* RegexMatchLen(pattern, input)
* @param pattern the string, sans trailing $, for the pattern
* @param input the text to match against
* @return the length, in code units, of the matched portion
*/
EM_JS(int, RegexMatchLen, (const char* pattern, const char *input), {
const DEBUG_JS = false;
if (!pattern) return -1;
if (!input) return -1;
const patternstr = Module.UTF8ToString(pattern) + '$';
const inputstr = Module.UTF8ToString(input);
const re = new RegExp(patternstr);
const result = re.exec(inputstr);
if (DEBUG_JS) console.dir({patternstr,inputstr,re,result});
if (!result) return 0; // no match
const index = result.index;
// code unit indices
const startIndex = index;
const endIndex = inputstr.length;
const matchedText = inputstr.substring(startIndex);
const matchedCodepoints = [...matchedText];
if (DEBUG_JS) console.dir({index, startIndex,endIndex,matchedText,matchedCodepoints});
return matchedCodepoints.length;
});
/**
* RegexGroup1(pattern, input)
* @param pattern the string, sans trailing $, for the pattern
* @param input the text to match against
* @return string of group 1 match or null
*/
EM_JS(char*, RegexGroup1, (const char* pattern, const char *input), {
const DEBUG_JS = false;
if (!pattern) return 0;
if (!input) return 0;
const patternstr = Module.UTF8ToString(pattern) + '$';
const inputstr = Module.UTF8ToString(input);
const re = new RegExp(patternstr);
const result = re.exec(inputstr);
if (!result) return 0; // no match
const g1 = result[1];
if (DEBUG_JS) console.dir({patternstr,inputstr,re,result,g1});
if (!g1) return 0;
return stringToNewUTF8(g1);
});
/**
* RegexSubstitute
* @param pattern the string, sans trailing $, for the pattern
* @param input the text to match against
* @param to the replacement text
* @return the entire updated output string
*/
EM_JS(char*, RegexSubstitute, (const char* pattern, const char *input, const char *to), {
const DEBUG_JS = false;
if (!pattern) return -1;
if (!input) return -1;
const patternstr = Module.UTF8ToString(pattern) + '$';
const inputstr = Module.UTF8ToString(input);
const tostr = Module.UTF8ToString(to);
const re = new RegExp(patternstr);
const output = inputstr.replace(re, tostr);
return stringToNewUTF8(output);
});
// pull in the generated table
#include "util_normalize_table.h"
#endif
namespace km {
namespace core {
namespace util {
/** find the */
int32_t km_regex::findIndex(const std::u32string &match, const std::deque<std::u32string> &list) {
int32_t index = 0;
for(auto e = list.begin(); e < list.end(); e++, index++) {
if (match == *e) {
return index;
}
}
return -1; // not found
}
km_regex::km_regex()
#if KMN_NO_ICU
#else
: fPattern(nullptr)
#endif
{
}
km_regex::km_regex(const km_regex& other)
#if KMN_NO_ICU
: fPattern(other.fPattern)
#else
: fPattern(nullptr)
#endif
{
#if KMN_NO_ICU
#else
if (other.fPattern) {
// clone pattern
fPattern.reset(other.fPattern->clone());
}
#endif
}
km_regex::km_regex(const std::u32string &pattern)
#if KMN_NO_ICU
: fPattern(pattern)
#else
: fPattern(nullptr)
#endif
{
init(pattern);
}
km_regex::~km_regex() {
}
bool km_regex::valid() const {
#if KMN_NO_ICU
return (!fPattern.empty());
#else
// valid if fPattern is present.
return !!fPattern;
#endif
}
bool km_regex::init(const std::u32string &pattern) {
#if KMN_NO_ICU
// The current implementation makes a new regex every time, so we always return true.
assert(!pattern.empty());
fPattern = pattern;
return true;
#else
if (pattern.empty()) {
return false;
}
std::u16string patstr = km::core::kmx::u32string_to_u16string(pattern);
UErrorCode status = U_ZERO_ERROR;
/* const */ icu::UnicodeString patustr = icu::UnicodeString(patstr.data(), (int32_t)patstr.length());
// add '$' to match to end
patustr.append(u'$');
fPattern.reset(icu::RegexPattern::compile(patustr, 0, status));
return (UASSERT_SUCCESS(status));
#endif
}
size_t km_regex::apply(const std::u32string &input, std::u32string &output,
const std::u32string &to,
const std::deque<std::u32string> &fromList,
const std::deque<std::u32string> &toList ) const {
#if KMN_NO_ICU
// length in code points of match from end
std::string patstr = convert<char32_t,char>(fPattern);
std::string instr = convert<char32_t,char>(input);
/** code units */
const auto matchLen = RegexMatchLen(patstr.c_str(), instr.c_str());
assert(matchLen != -1); // error
if (matchLen == 0) {
return 0; // Normal case return: no match
}
std::u32string rustr; // replacement
if (fromList.empty()) {
// Normal case: not a map.
// This replace will apply $1, $2 etc.
rustr = to;
} else {
// we actually need the group(1) string here.
// this is only the content in parenthesis ()
char *group1 = RegexGroup1(patstr.c_str(), instr.c_str());
assert(group1 != nullptr);
const std::string group1str(group1);
const std::u32string match32 = convert<char, char32_t>(group1str);
free(group1);
// Now we're ready to do the actual mapping.
// 1., we need to find the index in the source set.
auto matchIndex = findIndex(match32, fromList);
assert(matchIndex != -1L); // This indicates that the regex and the fromList are out of sync.
// we already asserted on load that the from and to sets have the same cardinality.
// 2. get the target string
// we use the same matchIndex that was just found
// 3. update the UnicodeString for replacement
rustr = toList.at(matchIndex);
}
std::string rstr = convert<char32_t,char>(rustr);
// here we replace the match output.
char *out = RegexSubstitute(patstr.c_str(), instr.c_str(), rstr.c_str());
assert(out != nullptr);
std::string outstr(out);
free(out);
output = convert<char, char32_t>(outstr);
// output includes all of 'input', but modified. Need to substring it.
/** code units */
const auto matchStart = input.length() - matchLen;
// remove the unmatched prefix.
output.erase(0, matchStart);
return matchLen;
#else
assert(fPattern);
// TODO-LDML: This entire section may have too many conversions and copies. Could be optimized.
UErrorCode status = U_ZERO_ERROR;
const std::u16string matchstr = km::core::kmx::u32string_to_u16string(input);
icu::UnicodeString matchustr = icu::UnicodeString(matchstr.data(), (int32_t)matchstr.length());
// TODO-LDML: create a new Matcher every time. These could be cached and reset.
std::unique_ptr<icu::RegexMatcher> matcher(fPattern->matcher(matchustr, status));
if (!UASSERT_SUCCESS(status)) {
return 0;
}
if (!matcher->find(status)) { // i.e. matches somewhere, in this case at end of str
return 0; // Normal case return: no match
}
// Note: this is UTF-16 len, not UTF-32 len.
int32_t matchStart = matcher->start(status);
int32_t matchEnd = matcher->end(status);
if (!UASSERT_SUCCESS(status)) {
return 0; // TODO-LDML: return error
}
// extract..
const icu::UnicodeString substr = matchustr.tempSubStringBetween(matchStart, matchEnd);
// preflight to UTF-32 to get length
UErrorCode substrStatus = U_ZERO_ERROR; // throwaway status
// we need the UTF-32 matchLen for our return.
auto matchLen = substr.toUTF32(nullptr, 0, substrStatus);
// should have matched something.
assert(matchLen > 0);
// now, do the replace.
/** this is the 'to' or other replacement string.*/
icu::UnicodeString rustr;
if (fromList.empty()) {
// Normal case: not a map.
// This replace will apply $1, $2 etc.
// Convert the fTo into u16 TODO-LDML (we could cache this?)
const std::u16string rstr = km::core::kmx::u32string_to_u16string(to);
rustr = icu::UnicodeString(rstr.data(), (int32_t)rstr.length());
} else {
// Set map case: mapping from/to
// we actually need the group(1) string here.
// this is only the content in parenthesis ()
icu::UnicodeString group1 = matcher->group(1, status);
if (!UASSERT_SUCCESS(status)) {
// TODO-LDML: could be a malformed from pattern
return 0; // TODO-LDML: return error
}
// now, how long is group1 in UTF-32, hmm?
UErrorCode preflightStatus = U_ZERO_ERROR; // throwaway status
auto group1Len = group1.toUTF32(nullptr, 0, preflightStatus);
char32_t *s = new char32_t[group1Len + 1];
assert(s != nullptr); // TODO-LDML: OOM
// convert
group1.toUTF32((UChar32 *)s, group1Len + 1, status);
if (!UASSERT_SUCCESS(status)) {
return 0; // TODO-LDML: memory issue
}
std::u32string match32(s, group1Len); // taken from just group1
// clean up buffer
delete [] s;
// Now we're ready to do the actual mapping.
// 1., we need to find the index in the source set.
auto matchIndex = findIndex(match32, fromList);
assert(matchIndex != -1L); // This indicates that the regex and the fromList are out of sync.
// we already asserted on load that the from and to sets have the same cardinality.
// 2. get the target string, convert to utf-16
// we use the same matchIndex that was just found
const std::u16string rstr = km::core::kmx::u32string_to_u16string(toList.at(matchIndex));
// 3. update the UnicodeString for replacement
rustr = icu::UnicodeString(rstr.data(), (int32_t)rstr.length());
// and we return to the regular code flow.
}
// here we replace the match output.
icu::UnicodeString entireOutput = matcher->replaceFirst(rustr, status);
if (!UASSERT_SUCCESS(status)) {
// TODO-LDML: could fail here due to bad input (syntax err)
return 0;
}
// entireOutput includes all of 'input', but modified. Need to substring it.
icu::UnicodeString outu = entireOutput.tempSubString(matchStart);
// Special case if there's no output, save some allocs
if (outu.length() == 0) {
output.clear();
} else {
// TODO-LDML: All we are trying to do is to extract the output string. Probably too many steps.
UErrorCode preflightStatus = U_ZERO_ERROR;
// calculate how big the buffer is
auto out32len = outu.toUTF32(nullptr, 0, preflightStatus); // preflightStatus will be an err, because we know the buffer overruns zero bytes
// allocate
std::unique_ptr<char32_t[]> s(new char32_t[out32len + 1]);
assert(s);
if (!s) {
return 0;
}
// convert
outu.toUTF32((UChar32 *)(s.get()), out32len + 1, status);
if (!UASSERT_SUCCESS(status)) {
return 0;
}
output.assign(s.get(), out32len);
}
return matchLen;
#endif
}
}
}
}

48
core/src/util_regex.hpp Normal file
View file

@ -0,0 +1,48 @@
/*
Copyright: © SIL International.
Description: Normalization and Regex utilities
Create Date: 23 May 2024
Authors: Steven R. Loomis
*/
#pragma once
#include "core_icu.h"
#include "keyman_core.h"
#include <string>
#include <deque>
namespace km {
namespace core {
namespace util {
class km_regex {
public:
km_regex();
km_regex(const km_regex &other);
km_regex(const std::u32string &pattern);
~km_regex();
bool init(const std::u32string &pattern);
size_t apply(
const std::u32string &input,
std::u32string &output,
const std::u32string &to,
const std::deque<std::u32string> &fromList,
const std::deque<std::u32string> &toList) const;
bool valid() const;
private:
#if KMN_NO_ICU
std::u32string fPattern; // TODO: by value?
#else
std::unique_ptr<icu::RegexPattern> fPattern;
#endif
// utility functions
public:
static int32_t findIndex(const std::u32string &match, const std::deque<std::u32string> &list);
};
} // namespace util
} // namespace core
} // namespace km

2
core/tests/unit/kmnkbd/meson.build
View file

@ -34,7 +34,7 @@ test_path = join_paths(meson.current_build_dir(), '..', 'kmx')
tests_flags = []
if cpp_compiler.get_id() == 'emscripten'
tests_flags += ['-lnodefs.js', '-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\']']
tests_flags += ['-lnodefs.js', wasm_exported_runtime_methods]
endif
foreach t : tests

3
core/tests/unit/ldml/ldml_test_source.cpp
View file

@ -18,6 +18,9 @@
#include <json.hpp>
// Ensure that ICU gets included even on wasm.
#define KMN_IN_LDML_TESTS
#include <kmx/kmx_processevent.h> // for char to vk mapping tables
#include <kmx/kmx_xstring.h> // for surrogate pair macros
#include <kmx/kmx_plus.h>

2
core/tests/unit/ldml/meson.build
View file

@ -126,7 +126,7 @@ test('test_context_normalization', test_context_normalization, suite: 'ldml')
# Build and run additional test_unicode test
test_unicode = executable('test_unicode', 'test_unicode.cpp',
['test_unicode.cpp', common_test_files],
['test_unicode.cpp', common_test_files, generated_headers],
cpp_args: defns + warns,
include_directories: [inc, libsrc, '../../../../developer/src/ext/json'],
link_args: links + tests_flags,

127
core/tests/unit/ldml/test_transforms.cpp
View file

@ -1,11 +1,13 @@
#include "../../../src/ldml/ldml_markers.hpp"
#include "../../../src/ldml/ldml_transforms.hpp"
#include "../../../src/util_regex.hpp"
#include "kmx/kmx_plus.h"
#include "kmx/kmx_xstring.h"
#include "test_color.h"
#include <iostream>
#include <string>
#include <test_assert.h>
#include "../../../src/util_regex.hpp"
// TODO-LDML: normal asserts wern't working, so using some hacks.
// #include "ldml_test_utils.hpp"
@ -13,14 +15,14 @@
// #include "debuglog.h"
#ifndef zassert_string_equal
#define zassert_string_equal(actual, expected) \
{ \
if (actual != expected) { \
std::wcerr << __FILE__ << ":" << __LINE__ << ": " << console_color::fg(console_color::BRIGHT_RED) \
<< "got: " << km::core::kmx::Debug_UnicodeString(actual, 0) << " expected " \
<< km::core::kmx::Debug_UnicodeString(expected, 1) << console_color::reset() << std::endl; \
return EXIT_FAILURE; \
} \
#define zassert_string_equal(actual, expected) \
{ \
if (actual != expected) { \
std::wcerr << __FILE__ << ":" << __LINE__ << ": " << console_color::fg(console_color::BRIGHT_RED) << "got: " << actual \
<< " " << km::core::kmx::Debug_UnicodeString(actual, 0) << " expected " << expected << " " \
<< km::core::kmx::Debug_UnicodeString(expected, 1) << console_color::reset() << std::endl; \
return EXIT_FAILURE; \
} \
}
#endif
@ -624,16 +626,16 @@ test_map() {
std::cout << __FILE__ << ":" << __LINE__ << " transform_entry::findIndex" << std::endl;
{
std::deque<std::u32string> list;
assert_equal(transform_entry::findIndex(U"Does Not Exist", list), -1);
assert_equal(km::core::util::km_regex::findIndex(U"Does Not Exist", list), -1);
list.emplace_back(U"0th");
list.emplace_back(U"First");
list.emplace_back(U"Second");
assert_equal(transform_entry::findIndex(U"First", list), 1);
assert_equal(transform_entry::findIndex(U"0th", list), 0);
assert_equal(transform_entry::findIndex(U"Second", list), 2);
assert_equal(transform_entry::findIndex(U"Nowhere", list), -1);
assert_equal(km::core::util::km_regex::findIndex(U"First", list), 1);
assert_equal(km::core::util::km_regex::findIndex(U"0th", list), 0);
assert_equal(km::core::util::km_regex::findIndex(U"Second", list), 2);
assert_equal(km::core::util::km_regex::findIndex(U"Nowhere", list), -1);
}
return EXIT_SUCCESS;
@ -1135,6 +1137,102 @@ test_normalize() {
return EXIT_SUCCESS;
}
/** test for the util_regex.hpp functions */
int
test_util_regex() {
std::cout << "== " << __FUNCTION__ << std::endl;
{
std::cout << __FILE__ << ":" << __LINE__ << " * util_regex.hpp null tests" << std::endl;
km::core::util::km_regex r;
assert(!r.valid()); // not valid because of an empty string
}
{
std::cout << __FILE__ << ":" << __LINE__ << " * util_regex.hpp simple tests" << std::endl;
km::core::util::km_regex r(U"ion");
assert(r.valid());
const std::u32string to(U"ivity");
const std::deque<std::u32string> fromList;
const std::deque<std::u32string> toList;
std::u32string output;
auto apply0 = r.apply(U"not present", output, to, fromList, toList);
assert_equal(apply0, 0); // not found
const std::u32string input(U"action");
auto apply1 = r.apply(input, output, to, fromList, toList);
assert_equal(apply1, 3); // matched last 3 codepoints
std::u32string expect(U"ivity");
zassert_string_equal(output, expect)
}
{
std::cout << __FILE__ << ":" << __LINE__ << " * util_regex.hpp wide tests" << std::endl;
km::core::util::km_regex r(U"e𐒻");
assert(r.valid());
const std::u32string to(U"𐓏");
const std::deque<std::u32string> fromList;
const std::deque<std::u32string> toList;
std::u32string output;
const std::u32string input(U":e𐒻");
auto apply1 = r.apply(input, output, to, fromList, toList);
assert_equal(apply1, 2); // matched last 2 codepoints
std::u32string expect(U"𐓏");
zassert_string_equal(output, expect)
}
{
std::cout << __FILE__ << ":" << __LINE__ << " * util_regex.hpp simple map tests" << std::endl;
km::core::util::km_regex r(U"(A|B|C)");
assert(r.valid());
const std::u32string to(U"$[1:alpha2]"); // ignored
std::deque<std::u32string> fromList;
fromList.emplace_back(U"A");
fromList.emplace_back(U"B");
fromList.emplace_back(U"C");
std::deque<std::u32string> toList;
toList.emplace_back(U"N");
toList.emplace_back(U"O");
toList.emplace_back(U"P");
std::u32string output;
auto apply0 = r.apply(U"not present", output, to, fromList, toList);
assert_equal(apply0, 0); // not found
const std::u32string input(U"WHOA");
auto apply1 = r.apply(input, output, to, fromList, toList);
assert_equal(apply1, 1); // matched last 1 codepoint
std::u32string expect(U"N");
zassert_string_equal(output, expect)
}
{
std::cout << __FILE__ << ":" << __LINE__ << " * util_regex.hpp wide map tests" << std::endl;
km::core::util::km_regex r(U"(𐒷|𐒻|𐓏𐓏|x)");
assert(r.valid());
const std::u32string to(U"$[1:alpha2]"); // ignored
std::deque<std::u32string> fromList;
fromList.emplace_back(U"𐒷");
fromList.emplace_back(U"𐒻");
fromList.emplace_back(U"𐓏𐓏");
fromList.emplace_back(U"x");
std::deque<std::u32string> toList;
toList.emplace_back(U"x");
toList.emplace_back(U"𐒷");
toList.emplace_back(U"𐒻");
toList.emplace_back(U"𐓏");
std::u32string output;
auto apply0 = r.apply(U"not present", output, to, fromList, toList);
assert_equal(apply0, 0); // not found
assert_equal(r.apply(U"WHO𐓏𐒷", output, to, fromList, toList), 1);
zassert_string_equal(output, U"x");
assert_equal(r.apply(U"WHO𐓏x", output, to, fromList, toList), 1);
zassert_string_equal(output, U"𐓏");
assert_equal(r.apply(U"WHO𐓏𐓏", output, to, fromList, toList), 2); // 2 codepoints
zassert_string_equal(output, U"𐒻");
}
return EXIT_SUCCESS;
}
int
main(int argc, const char *argv[]) {
int rc = EXIT_SUCCESS;
@ -1152,6 +1250,9 @@ main(int argc, const char *argv[]) {
console_color::enabled = console_color::isaterminal() || arg_color;
if (test_util_regex() != EXIT_SUCCESS) {
rc = EXIT_FAILURE;
}
if (test_transforms() != EXIT_SUCCESS) {
rc = EXIT_FAILURE;

50
core/tests/unit/ldml/test_unicode.cpp
View file

@ -10,6 +10,9 @@
#include <fstream>
#include <iostream>
// Ensure that ICU gets included even on wasm.
#define KMN_IN_LDML_TESTS
#include "keyman_core.h"
#include "path.hpp"
@ -22,6 +25,8 @@
#include <unicode/uversion.h>
#include <unicode/uchar.h>
#include "json.hpp"
#include "util_normalize.hpp"
#include "kmx/kmx_xstring.h"
#include <test_assert.h>
#include <test_color.h>
@ -40,6 +45,11 @@
} \
}
#ifdef __EMSCRIPTEN__
// Pull this in to verify versions
#include "util_normalize_table.h"
#endif
//-------------------------------------------------------------------------------------
// Unicode version tests
//-------------------------------------------------------------------------------------
@ -172,6 +182,42 @@ const std::string &block_unicode_ver) {
std::cout << std::endl;
}
#ifdef __EMSCRIPTEN__
inline const char *boolstr(bool b) {
return b?"T":"f";
}
void test_has_boundary_before() {
std::cout << "= " << __FUNCTION__ << std::endl;
std::cout << "I see we are on Emscripten / wasm! Now we will do some additional tests." << std::endl;
std::string icu4c_unicode(U_UNICODE_VERSION), header_unicode(KM_HASBOUNDARYBEFORE_UNICODE_VERSION),
icu4c_icu(U_ICU_VERSION), header_icu(KM_HASBOUNDARYBEFORE_ICU_VERSION);
std::cout << "Unicode: " << U_UNICODE_VERSION << ", and from the table file: " << KM_HASBOUNDARYBEFORE_UNICODE_VERSION << std::endl;
std::cout << "It would be very strange for these versions to be out of sync. Some sort of build or tool problem." << std::endl;
assert_basic_equal(icu4c_unicode, header_unicode);
assert_basic_equal(icu4c_icu, header_icu);
std::cout << std::endl << "Now, let's make sure has_nfd_boundary_before() matches ICU." << std::endl;
UErrorCode status = U_ZERO_ERROR;
const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(status);
UASSERT_SUCCESS(status);
// now, test that hasBoundaryBefore is the same
for (km_core_usv cp = 0; cp < km::core::kmx::Uni_MAX_CODEPOINT; cp++) {
auto km_hbb = km::core::util::has_nfd_boundary_before(cp);
auto icu_hbb = nfd->hasBoundaryBefore(cp);
if (km_hbb != icu_hbb) {
std::cerr << "Error: util_normalize_table.h said " << boolstr(km_hbb) << " but ICU said " << boolstr(icu_hbb) << " for "
<< "has_nfd_boundary_before(0x" << std::hex << cp << std::dec << ")" << std::endl;
}
assert(km_hbb == icu_hbb);
}
std::cout << "All OK!" << std::endl;
}
#endif
int test_all(const char *jsonpath, const char *packagepath, const char *blockspath) {
std::cout << "= " << __FUNCTION__ << std::endl;
@ -187,6 +233,10 @@ int test_all(const char *jsonpath, const char *packagepath, const char *blockspa
test_unicode_versions(versions, package, block_unicode_ver);
#ifdef __EMSCRIPTEN__
test_has_boundary_before();
#endif
return EXIT_SUCCESS;
}

1
developer/src/kmcmplib/include/kmcompx.h
View file

@ -29,6 +29,7 @@ typedef KMX_WCHAR* PKMX_WCHAR ;
#ifndef _MSC_VER
#include <type_traits>
#include <cstddef>
template < typename T, size_t N >
size_t _countof( T ( & /*arr*/ )[ N ] )

20
developer/src/kmcmplib/src/CompMsg.cpp
View file

@ -1,11 +1,7 @@
#include <CompMsg.h>
#include <map>
struct CompilerError {
KMX_DWORD ErrorCode;
const KMX_CHAR* Text;
};
const struct CompilerError CompilerErrors[] = {
std::map<KMX_DWORD, const KMX_CHAR*> CompilerErrorMap = {
{ CERR_InvalidLayoutLine , "Invalid 'layout' command"},
{ CERR_NoVersionLine , "No version line found for file"},
{ CERR_InvalidGroupLine , "Invalid 'group' command"},
@ -150,14 +146,8 @@ const struct CompilerError CompilerErrors[] = {
{ CWARN_VirtualKeyInOutput , "Virtual keys are not supported in output"},
{ 0, nullptr }
};
};
KMX_CHAR *GetCompilerErrorString(KMX_DWORD code)
{
for(int i = 0; CompilerErrors[i].ErrorCode; i++) {
if(CompilerErrors[i].ErrorCode == code) {
return ( KMX_CHAR*) CompilerErrors[i].Text;
}
}
return nullptr;
const KMX_CHAR *GetCompilerErrorString(KMX_DWORD code) {
return CompilerErrorMap[code];
}

2
developer/src/kmcmplib/src/CompMsg.h
View file

@ -2,4 +2,4 @@
#include "km_types.h"
#include <kmn_compiler_errors.h>
KMX_CHAR *GetCompilerErrorString(KMX_DWORD code) ;
const KMX_CHAR *GetCompilerErrorString(KMX_DWORD code) ;

6
developer/src/kmcmplib/src/CompileKeyboardBuffer.cpp
View file

@ -174,7 +174,7 @@ namespace kmcmp {
void CopyExtraData(PFILE_KEYBOARD fk) {
/* Copy stores */
PFILE_STORE store = fk->dpStoreArray;
for(int i = 0; i < fk->cxStoreArray; i++, store++) {
for(KMX_DWORD i = 0; i < fk->cxStoreArray; i++, store++) {
KMCMP_COMPILER_RESULT_EXTRA_STORE extraStore;
extraStore.storeType =
(store->fIsStore ? STORETYPE_STORE : 0) |
@ -188,11 +188,11 @@ namespace kmcmp {
}
PFILE_GROUP group = fk->dpGroupArray;
for(int i = 0; i < fk->cxGroupArray; i++, group++) {
for(KMX_DWORD i = 0; i < fk->cxGroupArray; i++, group++) {
KMCMP_COMPILER_RESULT_EXTRA_GROUP extraGroup;
extraGroup.isReadOnly = group->fReadOnly;
extraGroup.name = string_from_u16string(group->szName);
fk->extra->groups.push_back(extraGroup);
}
}
}
}

19
developer/src/kmcmplib/src/Compiler.cpp
View file

@ -276,7 +276,7 @@ KMX_BOOL kmcmp::AddCompileWarning(PKMX_CHAR buf)
KMX_BOOL AddCompileError(KMX_DWORD msg)
{
KMX_CHAR szText[COMPILE_ERROR_MAX_LEN];
KMX_CHAR* szTextp = NULL;
const KMX_CHAR* szTextp = NULL;
if (msg & CERR_FATAL)
{
@ -806,12 +806,13 @@ bool resizeStoreArray(PFILE_KEYBOARD fk) {
* reallocates the key array in increments of 100
*/
bool resizeKeyArray(PFILE_GROUP gp, int increment) {
if((gp->cxKeyArray + increment - 1) % 100 < increment) {
PFILE_KEY kp = new FILE_KEY[((gp->cxKeyArray + increment)/100 + 1) * 100];
const int cxKeyArray = (int)gp->cxKeyArray;
if((cxKeyArray + increment - 1) % 100 < increment) {
PFILE_KEY kp = new FILE_KEY[((cxKeyArray + increment)/100 + 1) * 100];
if (!kp) return false;
if (gp->dpKeyArray)
{
memcpy(kp, gp->dpKeyArray, gp->cxKeyArray * sizeof(FILE_KEY));
memcpy(kp, gp->dpKeyArray, cxKeyArray * sizeof(FILE_KEY));
delete[] gp->dpKeyArray;
}
@ -1191,11 +1192,11 @@ int GetCompileTargetsFromTargetsStore(const KMX_WCHAR* store) {
if(AnyTarget == token) {
result |= COMPILETARGETS_KMX | COMPILETARGETS_JS;
}
for(auto p: KMXKeymanTargets) {
if(p == token) result |= COMPILETARGETS_KMX;
for(auto target: KMXKeymanTargets) {
if(target == token) result |= COMPILETARGETS_KMX;
}
for(auto p: KMWKeymanTargets) {
if(p == token) result |= COMPILETARGETS_JS;
for(auto target: KMWKeymanTargets) {
if(target == token) result |= COMPILETARGETS_JS;
}
token = u16tok(nullptr, u" ", &ctx);
@ -3516,7 +3517,7 @@ bool UTF16TempFromUTF8(KMX_BYTE* infile, int sz, KMX_BYTE** tempfile, int *sz16)
try {
std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> converter;
result = converter.from_bytes((char*)infile, (char*)infile+sz);
} catch(std::range_error e) {
} catch(std::range_error& e) {
AddCompileError(CHINT_NonUnicodeFile);
result.resize(sz);
for(int i = 0; i < sz; i++) {

2
developer/src/kmcmplib/src/NamedCodeConstants.cpp
View file

@ -268,7 +268,7 @@ int IsHangulSyllable(const KMX_WCHAR *codename, int *code)
if(strchr("GNDRMBSJCKTPH", ch))
{
/* Has an initial syllable */
int isDoubled = towupper(*(codename+1)) == ch;
int isDoubled = towupper(*(codename+1)) == (wint_t)ch;
LIndex = -1;
for(i = 0; i < HangulLCount; i++) {

8
developer/src/kmcmplib/src/kmx_u16.cpp
View file

@ -9,6 +9,7 @@
#include <codecvt>
#include <locale>
#include <stdarg.h>
#include <algorithm>
//String <- wstring
std::string string_from_wstring(std::wstring const str) {
@ -96,7 +97,7 @@ std::string toHex(int num1) {
s += (87 + temp);
num = num / 16;
}
reverse(s.begin(), s.end());
std::reverse(s.begin(), s.end());
return s;
}
@ -274,10 +275,7 @@ double u16tof( KMX_WCHAR* str)
char digit;
PKMX_WCHAR q = (PKMX_WCHAR)u16chr(str, '.');
size_t pos_dot = q-str ;
if (pos_dot < 0)
pos_dot = u16len(str);
size_t pos_dot = (q-str < 0) ? u16len(str) : q-str;
for (size_t i = 0; i < u16len(str); i++)
{

1
developer/src/kmcmplib/src/kmx_u16.h
View file

@ -4,6 +4,7 @@
#include <vector>
#include <ctype.h>
#include <string>
#include <cstring>
#include "kmcompx.h"
std::string string_from_wstring(std::wstring const str);

4
developer/src/kmcmplib/src/meson.build
View file

@ -12,8 +12,10 @@ lib_links = []
if cpp_compiler.get_id() == 'gcc' or cpp_compiler.get_id() == 'clang'
warns += [
'-Wall',
'-Wextra'
'-Wextra',
'-Wno-ignored-qualifiers'
]
flags += ['-D__cdecl= ']
endif
if cpp_compiler.get_id() == 'msvc'

8
developer/src/kmcmplib/tests/gtest-compiler-test.cpp
View file

@ -223,13 +223,13 @@ TEST_F(CompilerTest, ProcessBeginLine_test) {
TEST_F(CompilerTest, ValidateMatchNomatchOutput_test) {
EXPECT_EQ(CERR_None, ValidateMatchNomatchOutput(NULL));
EXPECT_EQ(CERR_None, ValidateMatchNomatchOutput((PKMX_WCHAR)u""));
const KMX_WCHAR context[] = { 'a', 'b', 'c', UC_SENTINEL, CODE_CONTEXT, 'd', 'e', 'f' };
const KMX_WCHAR context[] = { 'a', 'b', 'c', UC_SENTINEL, CODE_CONTEXT, 'd', 'e', 'f', 0 };
EXPECT_EQ(CERR_ContextAndIndexInvalidInMatchNomatch, ValidateMatchNomatchOutput((PKMX_WCHAR)context));
const KMX_WCHAR contextex[] = { 'a', 'b', 'c', UC_SENTINEL, CODE_CONTEXTEX, 'd', 'e', 'f' };
const KMX_WCHAR contextex[] = { 'a', 'b', 'c', UC_SENTINEL, CODE_CONTEXTEX, 'd', 'e', 'f', 0 };
EXPECT_EQ(CERR_ContextAndIndexInvalidInMatchNomatch, ValidateMatchNomatchOutput((PKMX_WCHAR)contextex));
const KMX_WCHAR index[] = { 'a', 'b', 'c', UC_SENTINEL, CODE_INDEX, 'd', 'e', 'f' };
const KMX_WCHAR index[] = { 'a', 'b', 'c', UC_SENTINEL, CODE_INDEX, 'd', 'e', 'f', 0 };
EXPECT_EQ(CERR_ContextAndIndexInvalidInMatchNomatch, ValidateMatchNomatchOutput((PKMX_WCHAR)index));
const KMX_WCHAR sentinel[] = { 'a', 'b', 'c', UC_SENTINEL, 'd', 'e', 'f' };
const KMX_WCHAR sentinel[] = { 'a', 'b', 'c', UC_SENTINEL, 'd', 'e', 'f', 0 };
EXPECT_EQ(CERR_None, ValidateMatchNomatchOutput((PKMX_WCHAR)sentinel));
};

19
developer/src/kmcmplib/tests/gtest-compmsg-test.cpp Normal file
View file

@ -0,0 +1,19 @@
#include <gtest/gtest.h>
#include "../../common/include/kmn_compiler_errors.h"
#include "../../../../common/include/km_types.h"
const KMX_CHAR *GetCompilerErrorString(KMX_DWORD code);
class CompMsgTest : public testing::Test {
protected:
CompMsgTest() {}
~CompMsgTest() override {}
void SetUp() override {}
void TearDown() override {}
};
TEST_F(CompMsgTest, GetCompilerErrorString) {
EXPECT_EQ(nullptr, GetCompilerErrorString(CERR_None));
EXPECT_EQ(nullptr, GetCompilerErrorString(0x00004FFF)); // top of range ERROR
EXPECT_EQ("Invalid 'layout' command", GetCompilerErrorString(CERR_InvalidLayoutLine));
};

10
developer/src/kmcmplib/tests/kmcompxtest.cpp
View file

@ -19,6 +19,7 @@
#ifdef _MSC_VER
#else
#include <unistd.h>
#include <cstring>
#endif
using namespace std;
@ -85,10 +86,11 @@ int main(int argc, char *argv[])
fseek(fp2, 0, SEEK_END);
auto sz2 = ftell(fp2);
fseek(fp2, 0, SEEK_SET);
if (result.kmxSize != sz2) return __LINE__; // exit code: size of kmx-file in build differs from size of kmx-file in source folder
if (result.kmxSize != (size_t)sz2) return __LINE__; // exit code: size of kmx-file in build differs from size of kmx-file in source folder
char* buf2 = new char[result.kmxSize];
fread(buf2, 1, result.kmxSize, fp2);
auto sz3 = fread(buf2, 1, result.kmxSize, fp2);
if (result.kmxSize != sz3) return __LINE__; // exit code: when not able to read the build into the buffer
return memcmp(result.kmx, buf2, result.kmxSize) ? __LINE__ : 0; // exit code: when contents of kmx-file in build differs from contents of kmx-file in source folder
// success: when contents of kmx-file in build and source folder are the same
}
@ -102,7 +104,7 @@ int main(int argc, char *argv[])
std::istringstream(ErrNr) >> std::hex >> error_val;
// check if error_val is in Array of Errors; if it is found return 0 (it's not an error)
for (int i = 0; i < error_vec.size() ; i++) {
for (size_t i = 0; i < error_vec.size() ; i++) {
if (error_vec[i] == error_val) {
return 0; // success: CERR_ in Name + Error (specified in CERR_Name) IS found
}
@ -156,7 +158,7 @@ bool isDesktopKeyboard(FILE* fp) {
PCOMP_STORE s = pfs;
for(int i = 0; i < fk.cxStoreArray; i++, s++) {
for(KMX_DWORD i = 0; i < fk.cxStoreArray; i++, s++) {
if(s->dwSystemID != TSS_TARGETS) {
continue;
}

11
developer/src/kmcmplib/tests/meson.build
View file

@ -167,3 +167,14 @@ gtestcompilertest = executable('gtest-compiler-test', 'gtest-compiler-test.cpp',
)
test('gtest-compiler-test', gtestcompilertest)
gtestcompmsgtest = executable('gtest-compmsg-test', 'gtest-compmsg-test.cpp',
cpp_args: defns + flags,
include_directories: inc,
name_suffix: name_suffix,
link_args: links + tests_links,
objects: lib.extract_all_objects(),
dependencies: [ icuuc_dep, gtest_dep, gmock_dep ],
)
test('gtest-compmsg-test', gtestcompmsgtest)

6
developer/src/kmcmplib/tests/util_callbacks.cpp
View file

@ -16,7 +16,7 @@ int msgproc(int line, uint32_t dwMsgCode, const char* szText, void* context) {
case CERR_ERROR: t=" error"; break;
case CERR_FATAL: t=" fatal"; break;
}
printf("line %d %s %04.4x: %s\n", line, t, (unsigned int)dwMsgCode, szText);
printf("line %d %s %4.4x: %s\n", line, t, (unsigned int)dwMsgCode, szText);
return 1;
}
@ -49,11 +49,11 @@ bool loadfileProc(const char* filename, const char* baseFilename, void* data, in
}
} else {
// return data
if(fread(data, 1, *size, fp) != *size) {
if(fread(data, 1, *size, fp) != (size_t)(*size)) {
fclose(fp);
return false;
}
}
fclose(fp);
return true;
}
}

465
developer/src/tike/xml/help/contexthelp.xml
View file

@ -102,6 +102,18 @@
======================================================================== -->
<Form Name="context/keyboard-editor">
<Control Name="pages" Title="Keyboard Editor">
<p>Keyboard Editor page consists of:
<ul>
<li>Details</li>
<li>Layout</li>
<li>Icon</li>
<li>On-Screen</li>
<li>Touch Layout</li>
<li>Build</li>
</ul>
</p>
</Control>
<!-- Details tab -->
<Control Name="editName" Title="Keyboard Name">
@ -139,6 +151,24 @@
<p>This corresponds to the following source line:</p>
<div class="code">store(&amp;message) 'Here is a message about a keyboard'</div>
</Control>
<Control Name="editKeyboardVersion" Title="Keyboard Version">
<p>The Keyboard Version documents the version of the keyboard.</p>
<p>A keyboard version should be updated whenever there are changes to a keyboard. The good principles to follow are:
<ul>
<li> Increment the major version number for a
keyboard that has significant new functionality.</li>
<li> Increment the minor version number for changes that impact functionality but not
in a significant manner.</li>
<li> Optionally, use a third number for bug fixes.</li>
</ul>
</p>
<p>This corresponds to the following source line:</p>
<div class="code">store(&amp;keyboardversion) '1.1.2'</div>
<p><b>Note:</b> there is a difference between &amp;keyboardversion, which documents the keyboard version, and &amp;version,
which determines which version of Keyman a keyboard will run with. </p>
</Control>
<Control Name="memoComments" Title="Comments">
<p>In this field, enter information about the keyboard for your own reference. These comments will only be visible in the
source file, and not to users of your keyboard.</p>
@ -183,6 +213,41 @@
<p>Tests your KeymanWeb keyboard in an Internet Explorer embedded window</p>
</Control>
<Control Name="gridFeatures" Title="Features">
<p>The Features grid controls which additional file components are included in
the keyboard. Each of the features relates to a system store. Here are the file
components:
<ul>
<li>Embedded JavaScript</li>
<li>Embedded CSS</li>
<li>Web Help</li>
<li>Include Codes</li>
<li>Desktop On-Screen Keyboard (auto-included if Targets is any)</li>
<li>Touch-Optimised Keyboard (auto-included if Targets is any)</li>
</ul>
Icon will be automatically included when a new keyboard project is created.
</p>
</Control>
<Control Name="cmdAddFeature" Title="Add feature">
<p>This will open a selection dialog allowing you to choose a feature to add to
the keyboard project. Adding a feature will add an extra tab to the editor,
and add the corresponding store to the keyboard source</p>
</Control>
<Control Name="cmdEditFeature" Title="Edit feature">
<p>Depending on what is included in the Feature Grid, you can select a feature from
the grid then click on Edit... This will take you to the corresponding tab and
let you make changes.
</p>
</Control>
<Control Name="cmdRemoveFeature" Title="Remove feature">
<p>Removing a feature will not delete the component file,
but will just remove the store from the keyboard source.
</p>
</Control>
<!-- Layout tab -->
<!-- Icon tab -->
<Control Name="cmdbitmapChange" Title="Change" TopicName="context/keyboard-editor#toc-icon-tab">
@ -197,8 +262,40 @@
<Control Name="panEdit" Title="Icon Editor" TopicName="context/keyboard-editor#toc-icon-tab">
<p>The toolbox allows for the changing of the colours, addition of text and shapes. It also allows for the moving of the icon around the canvas and also a preview of the icon is displayed.</p>
</Control>
<Control Name="palColours" Title="Colours">
<p>Click on a colour from the box to apply the colour onto the Keyboard's
icon. You can see the Foreground Color displays the colour you chose. To deselect
the colour, click on the X mark on the corner left of the colour box, or choose
another colour.
</p>
</Control>
<!-- On-Screen tab: see context/keyboard-editor#toc-on-screen-tab -->
<Control Name="pages" Title="Pages">
<p>This tab allows you to edit the visual representation of your keyboard
layout. The content on this tab is stored in the .kvks file associated with
your keyboard. The visual representation is used only in desktop and desktop
web; however if no touch layout is defined, this layout will be synthesized
into a touch layout automatically.
</p>
<p>An On-Screen keyboard is optional but in most keyboards is recommended.
The On-Screen keyboard may not always match the actual layout identically,
because you may choose to hide some of the details of encoding from the
interface presented to the user.
</p>
<p>
This keyboard layout can also be printed or included in HTML or other documentation.
The editor allows you to export the file to HTML, PNG or BMP formats.
</p>
</Control>
<Control Name="chkFillUnderlyingLayout" Title="Auto-fill underlying layout">
<p>If this option is checked, when the Fill from layout button is clicked,
then keys without corresponding rules in the Layout will be filled with the
base layout character.
</p>
</Control>
<!-- Touch Layout tab: see context/keyboard-editor#toc-touch-layout-tab -->
<!-- Source tab -->
<!-- Compile tab -->
@ -331,33 +428,185 @@
<p>The debugger can be used without the debug information by clicking on Test without debugger.</p>
</Control>
</Form>
<!-- New file details -->
<Form Name="context/new-file-details">
<Control Name="*" Title="New File">
<p>Enter the name for your new keyboard. Click on the Browse button to change the location of the new keyboard.</p>
</Control>
</Form>
<!-- Package Editor -->
<Form Name="context/package-editor">
<Control Name="pages" Title="Keyboard Package">
<p>A keyboard package is most likely to have 7 tabs:
<ul>
<li>Files</li>
<li>Keyboards</li>
<li>Lexical Models</li>
<li>Details</li>
<li>Shortcuts</li>
<li>Source</li>
<li>Build</li>
</ul>
</p>
</Control>
<!-- Keyboard Layouts Tab -->
<Control Name="lbKeyboards" Title="Keyboard name">
<p>Usually, there is only one keyboard listed in the box, and by clicking on it
will show the keyboard information.</p>
</Control>
<Control Name="cbKeyboardOSKFont" Title="Keyboard font">
<p>When font files are added to the package, this dropdown tells the Keyman
apps which font to use when rendering the On Screen Keyboard touch keyboard.
</p>
</Control>
<Control Name="cbKeyboardDisplayFont" Title="Display font">
<p>When font files are added to the package, this dropdown tells the Keyman
apps for iOS and Android which font to use in edit fields. It only applies
within the Keyman app and apps that support this functionality.</p>
</Control>
<Control Name="gridKeyboardLanguages" Title="Keyboard languages">
<p>Each language listed here is a BCP 47 language tag and every keyboard must have a
minimum of one language. When Keyman installs the keyboard package, it will associate the
keyboard with the language(s) you select.</p>
</Control>
<Control Name="cmdKeyboardAddLanguage" Title="Add a language">
<p>This button will open up the BCP 47 Tag window to add the keyboard's language tag, script tag,
region tag, and language name.</p>
</Control>
<Control Name="cmdKeyboardRemoveLanguage" Title="Remove a language">
<p>Select on a language then click on Remove to delete the language tag.</p>
</Control>
<Control Name="cmdKeyboardEditLanguage" Title="Edit a language">
<p>Click on Edit... then the BCP 47 Tag window will pop up once again with the language information.</p>
</Control>
<!-- Lexical Models Tab -->
<Control Name="lbLexicalModels" Title="Lexical Models">
<p>This is the location of the Lexical Model (Predictive text) for the keyboard in the keyboard
package.</p>
</Control>
<Control Name="cmdLexicalModelLanguageAdd" Title="Add a language to the Lexical Model">
<p>Add button brings up the “Select BCP 47 Tag”
window, thus allowing the input of a language for the Lexical Model (Predictive text).</p>
</Control>
<Control Name="chkLexicalModelRTL" Title="Right-to-left Predictive text">
<p>By ticking this, it will set the text direction of the Lexical Model (Predictive text) to Right-to-left.</p>
</Control>
<Control Name="editLexicalModelDescription" Title="Lexical Model Description">
<p>A short or long text related to the Lexical Model (Predictive text) is encouraged to be added here but it is optional.</p>
</Control>
<!-- Details Tab -->
<Control Name="cbWelcomeFile" Title="Welcome file">
<p>Choose from the list to specifies which welcome file is suitable to display when they install the keyboard package.
However, this is optional, most keyboard package uses the default option which is (none).</p>
</Control>
<Control Name="cbLicense" Title="License file">
<p>Choose from the list if there is a License file to specified to the keyboard package.
However, this is optional, most keyboard package uses the default option which is (none) and we
only accept an open-license keyboard package.</p>
</Control>
<Control Name="editInfoAuthor" Title="Author's">
<p>Enter the name of the author or authors of the keyboard package.</p>
</Control>
<Control Name="editInfoCopyright" Title="Copyright">
<p>Enter the copyright details of the keyboard package This information will be displayed with the version, author and message information when the package is installed.</p>
</Control>
<Control Name="editInfoEmail" Title="Email address">
<p>Enter the contact email address for the keyboard package.</p>
</Control>
<Control Name="editInfoName" Title="Package Name">
<p>Package Name is the name that will be displayed when the package is installed. It should be a descriptive name, in any language, but remember that some applications may use a font that does not include the language you are writing the keyboard name in. Don't include a version number, help information or hotkey in the name.</p>
</Control>
<Control Name="editInfoVersion" Title="Package Version">
<p>The version number allows the user to check whether they have the latest version of the keyboard. The format should be 'major.minor[.subversion]'. Each number should be an integer, and you should avoid non-integer version strings. See help for more details.</p>
</Control>
<Control Name="editInfoWebSite" Title="Website">
<p>The website details for the keyboard package if available.</p>
</Control>
<Control Name="editLexicalModelVersion" Title="Lexical Model version">
<p>Specifies an independant version for the Lexical Model.</p>
</Control>
<Control Name="chkFollowKeyboardVersion" Title="Package and Keyboard version">
<p>By ticking this, the Lexical Model (Predictive text) version will receive a version bump alongside
the keyboard, even when the Model does not receive an update.</p>
</Control>
<Control Name="memoInfoDescription" Title="Description">
<p>A keyboard package's description about the language, wonderful community, script, or any related information
that would help users once they install the package.</p>
</Control>
<Control Name="gridRelatedPackages" Title="Related Packages">
<p>If a keyboard package is intended to replace an existing keyboard, or if there are related packages,
then the identifiers for these packages should be listed here.</p>
</Control>
<Control Name="cmdAddRelatedPackage" Title="Add a related package">
<p>Add the Package ID and specify Deprecated or Non-deprecated if a keyboard package is intended to
replace an existing keyboard, or if there are related packages.</p>
</Control>
<Control Name="cmdEditRelatedPackage" Title="Edit a related package">
<p>Selects the package and click Edit... to add any changes to the current information.</p>
</Control>
<Control Name="cmdRemoveRelatedPackage" Title="Remove a related package">
<p>A keyboard package's description about the language, community, script, or any related information
that would help users when they see once installing the package.</p>
</Control>
<!-- Shortcuts Tab -->
<Control Name="editStartMenuPath" Title="Start Menu Path">
<p>Enter the path of the start menu to be displayed when the keyboard package is installed.</p>
</Control>
<Control Name="lbStartMenuEntries" Title="Start menu entries">
<p>This is a list of Start menu entries for the keyboard package.</p>
</Control>
<Control Name="chkStartMenuUninstall" Title="Uninstall">
<p>The uninstall shortcut will be added automatically to the shortcut menu list when the package is installed.</p>
</Control>
<Control Name="chkCreateStartMenu" Title="Start menu">
<p>This option will allow you to create a folder on the Start menu when the keyboard package is installed.</p>
</Control>
<Control Name="cmdNewStartMenuEntry" Title="New">
<p>To add a new menu item to the list of shortcuts to be displayed in the Start menu folder created when the keyboard package is installed.</p>
</Control>
<Control Name="cmdDeleteStartMenuEntry" Title="Delete">
<p>To delete the selected shortcut menu item from the Start menu folder.</p>
</Control>
<Control Name="editStartMenuDescription" Title="Start">
<p>The text to be displayed for the selected file as a menu item in the Start up folder that is created when the package is installed.</p>
</Control>
<Control Name="editStartMenuParameters" Title="Shortcut Parameters" />
<Control Name="cbStartMenuProgram" Title="Shortcut Program" />
<!-- Compile Tab -->
<Control Name="cmdStartTestOnline" Title="Test package on web">
<p>Starts the Keyman Developer Web Server for the keyboard package.
This will list the various IP addresses and hostnames that Keyman Developer is listening on.</p>
</Control>
<Control Name="lbDebugHosts" Title="Web addresses">
<p>A list of available servers to test the keyboard package.</p>
</Control>
<Control Name="cmdOpenDebugHost" Title="Open in browser">
<p>Starts your default browser with the selected address to allow testing of the keyboard package
directly.</p>
</Control>
<Control Name="editBootstrapMSI" Title="Keyman MSI">
<p>Specifies the location of the Keyman MSI file. As of Keyman Developer 17, bundled executable package
installers for Keyman for Windows can be created using kmc, but cannot be created within the IDE.</p>
</Control>
<Control Name="cbImageFile" Title="Image File">
<p>You can include an image file that will be displayed to the left of the install details when the package is installed. This image should be 140 pixels wide and 250 pixels high.</p>
</Control>
<Control Name="cbReadmeFile" Title="Readme file">
<p>The readme file can be displayed after the installation of the keyboard package, but can also be accessed from the keyboard folder at a later time. The file must be loaded under the Files tab, add option before being able to be selected from the drop down list.</p>
</Control>
<Control Name="cbStartMenuProgram" Title="Shortcut Program" />
<Control Name="chkCreateStartMenu" Title="Start menu">
<p>This option will allow you to create a folder on the Start menu when the keyboard package is installed.</p>
</Control>
<Control Name="chkStartMenuUninstall" Title="Uninstall">
<p>The uninstall shortcut will be added automatically to the shortcut menu list when the package is installed.</p>
</Control>
<Control Name="cmdAddFile" Title="Add Files">
<p>This option allows the user to add new files to the package.</p>
</Control>
<Control Name="cmdDeleteStartMenuEntry" Title="Delete">
<p>To delete the selected shortcut menu item from the Start menu folder.</p>
</Control>
<Control Name="cmdInsertCopyright" Title="Insert Copyright">
<p>This allows for the insertion of a copyright symbol if needed.</p>
</Control>
@ -375,8 +624,6 @@
installed correctly. If you can, try installing your package on several different machines.</p>
</Control>
<!-- Compile Page -->
<Control Name="editOutPath" Title="Compile">
<p>Displays the output path and filename of the file when the package is compiled.</p>
</Control>
@ -404,9 +651,6 @@
<p>This option will install the package on the computer. A message will be displayed as to the success of the install.</p>
</Control>
<Control Name="cmdNewStartMenuEntry" Title="New">
<p>To add a new menu item to the list of shortcuts to be displayed in the Start menu folder created when the keyboard package is installed.</p>
</Control>
<Control Name="cmdOpenContainingFolder" Title="Open Containing Folder">
<p>Opens the source folder of the selected file.</p>
</Control>
@ -432,33 +676,6 @@
<p>File Type</p>
<p>Enter the details of the file type being added to the keyboard package.</p>
</Control>
<Control Name="editInfoAuthor" Title="Author's">
<p>Enter the name of the author or authors of the keyboard package.</p>
</Control>
<Control Name="editInfoCopyright" Title="Copyright">
<p>Enter the copyright details of the keyboard package This information will be displayed with the version, author and message information when the package is installed.</p>
</Control>
<Control Name="editInfoEmail" Title="Email address">
<p>Enter the contact email address for the keyboard package.</p>
</Control>
<Control Name="editInfoName" Title="Package Name">
<p>Package Name is the name that will be displayed when the package is installed. It should be a descriptive name, in any language, but remember that some applications may use a font that does not include the language you are writing the keyboard name in. Don't include a version number, help information or hotkey in the name.</p>
</Control>
<Control Name="editInfoVersion" Title="Package Version">
<p>The version number allows the user to check whether they have the latest version of the keyboard. The format should be 'major.minor[.subversion]'. Each number should be an integer, and you should avoid non-integer version strings. See help for more details.</p>
</Control>
<Control Name="editInfoWebSite" Title="Website">
<p>The website details for the keyboard package if available.</p>
</Control>
<Control Name="editStartMenuDescription" Title="Start">
<p>The text to be displayed for the selected file as a menu item in the Start up folder that is created when the package is installed.</p>
</Control>
<Control Name="editStartMenuParameters" Title="Shortcut Parameters" />
<Control Name="editStartMenuPath" Title="">
<p>Start Menu Path</p>
<p>Enter the path of the start menu to be displayed when the keyboard package is installed.</p>
</Control>
<Control Name="lbFiles" Title="Package Files">
<p>This will display all the files that have been added to the keyboard package. It allows for the addition and removal of files, the entering of file details and the editing of any of the files listed if the appropriate editor is available.</p>
</Control>
@ -467,12 +684,14 @@
</Control>
</Form>
<!-- Project -->
<Form Name="context/project">
<Control Name="*" Title="Project Manager">
<p>The Project Manager allows you to manage all the files related to a keyboard layout in a single location.</p>
</Control>
</Form>
<!-- New Lexical Model Project Parameters -->
<Form Name="context/new-model-project-parameters">
<Control Name="editAuthor" Title="Author Name">
<p>The name of the developer of the keyboard. This is either your full name or
@ -560,6 +779,7 @@
</Control>
</Form>
<!-- Wordlist Editor -->
<Form Name="context/wordlist-editor">
<Control Name="pages" Title="Wordlist tabs">
<p>Wordlist tabs have two views: Design, and Code. Changes to one view are reflected
@ -584,6 +804,7 @@
</Control>
</Form>
<!-- Editor -->
<Form Name="context/editor">
<Control Name="cefwp" Title="Editor Window">
<p>Editor windows in Keyman Developer supports standard Windows editing keystrokes.
@ -593,6 +814,7 @@
</Control>
</Form>
<!-- Character Identifier -->
<Form Name="context/character-identifier">
<Control Name="gridFonts" Title="Character Identifier">
<p>Attempt to identify the fonts on your system that will support the
@ -605,6 +827,7 @@
</Control>
</Form>
<!-- Messages -->
<Form Name="context/messages">
<Control Name="memoMessage" Title="Message Window">
<p>The message window appears at the bottom of the screen, or floating in a toolbar
@ -613,6 +836,7 @@
</Control>
</Form>
<!-- Debug -->
<Form Name="context/debug">
<Control Name="memo" Title="Debugger input window">
<p>The debugger input window is used for typing input to test the keyboard.
@ -689,6 +913,7 @@
</Control>
</Form>
<!-- About tike -->
<Form Name="context/about-tike">
<Control Name="cmdOK" Title="About Dialog">
<p>The About dialog displays copyright and registration information for Keyman Developer,
@ -696,6 +921,7 @@
</Control>
</Form>
<!-- Key test -->
<Form Name="context/key-test">
<Control Name="cmdInsert" Title="Virtual Key Identifier">
<p>This dialog lets you check the virtual key code for any key combination (except Window reserved key combinations such as Alt + Tab). You can then insert the virtual key code into the last active edit window at the current cursor position.</p><br></br>
@ -710,4 +936,155 @@
<p>To close the dialog, click the Close button or press Shift + Esc.</p>
</Control>
</Form>
<!-- Select BCP 47 Language -->
<Form Name="context/select-bcp47-language">
<Control Name="cbLanguageTag" Title="Language tag">
<p>The only required option is the Language tag, which is an ISO 639-1 or ISO 639-3 code.
ISO 639-1 tags are a two-letter code. ISO 639-3 tags are a three-letter code. First, try to find your
language on the list of two-letter ISO 639-1 codes.</p>
<p>If you can't find a two-letter code, you'll need to find the closest three-letter code. You can use Glottolog
to search for your language, and it will give you an appropriate code.</p>
<p>The Language tag is conventionally written in lower case.</p>
</Control>
<Control Name="cbScriptTag" Title="Script tag">
<p>The Script tag allows you to specify the writing system used in your language model or keyboard.
If your language only uses one writing system, omit the Script subtag. he Script subtag is conventionally
written in title case - first letter capitalized.</p>
</Control>
<Control Name="cbRegionTag" Title="Region tag">
<p>The Region tag allows you to specify the region your language or
dialect is spoken in. If your language is only spoken in one region, omit the Region subtag.
Alphabetic region will show up in upper case once the region tag is selected.</p>
</Control>
<Control Name="editBCP47Code" Title="BCP 47 Code">
<p>This is the BCP 47 Code forms from the language, script, and region tag which were previously
selected.</p>
</Control>
<Control Name="editLanguageName" Title="Language name">
<p>If lefts as default, a language name will be assigned automatically. Feel free to change the name
if you must.
</p>
</Control>
<Control Name="cmdResetLanguageName" Title="Reset">
<p>Reset the Language name to what it was assigned to.</p>
</Control>
<Control Name="cmdOK" Title="OK">
<p>Once everything is set, click OK to confirm.</p>
</Control>
<Control Name="cmdCancel" Title="Cancel">
<p>Exits out of the "Select BCP 47 Tag".</p>
</Control>
<Control Name="lblLinkToW3C" Title="Learn about BCP 47 Tag">
<p>Learn more about the BCP 47 Tag over at w3.org.</p>
</Control>
</Form>
<!-- New Project Parameters-->
<Form Name="context/new-project-parameters">
<Control Name="editKeyboardName" Title="Keyboard name">
<p>The descriptive name of the keyboard. This will be set in the <u>&amp;Name</u> store in the keyboard,
in the package name, and where appropriate in documentation and metadata.</p>
</Control>
<Control Name="memoDescription" Title="Description">
<p>A brief or long description about the keyboard's functionality, background, script, language, community,
or any information related to showcase to users.</p>
</Control>
<Control Name="editAuthor" Title="Author">
<p>The name of the developer of the keyboard. This is either your full name or
the organization you're creating the keyboard for.</p>
</Control>
<Control Name="editCopyright" Title="Copyright">
<p>This field should contain the word "Copyright", the copyright symbol "©", and the full name of the rights
owner. Do not put the year of copyright in this field (see Full Copyright). Typically,
you can use the automatically generated default value: "Copyright © Your Full Name or
Your Organization".</p>
<p>A copyright string for the keyboard. This will be set in the <u>&amp;Copyright</u> store in the keyboard,
in the package metadata, and where appropriate in documentation and metadata.</p>
</Control>
<Control Name="editFullCopyright" Title="Full Copyright">
<p>Who owns the rights to this keyboard? This field should
contain the word "Copyright", the copyright symbol "©", the first year of copyright,
and the full name of the rights owner. Do include the year of copyright in this field.
Typically, you can use the automatically generated default value:
"Copyright © Current-Year Your-Full Name or Your Organization".</p>
</Control>
<Control Name="editVersion" Title="Version">
<p>If this is the first time you've created a keyboard for the language, you should
leave the version as 1.0. Otherwise, your version number must conform to the following
rules: A version string made of major revision number.minor revision number.</p>
</Control>
<Control Name="clbTargets" Title="Targets">
<p>
Specifies the default deployment targets for the keyboard, set in the <u>&amp;Targets</u> store in the keyboard,
and controls the files added to the package initially. This also is reflected in documentation and metadata.
</p>
</Control>
<Control Name="editPath" Title="Edit Path">
<p>
Specifies the base path where the project folder will be created. The project folder name will be the keyboard ID.
If the folder already exists, then you will be prompted before Keyman Developer overwrites files inside it.
</p>
</Control>
<Control Name="cmdBrowse" Title="Browse">
<p>
Specifies a different path to store the project folder.
</p>
</Control>
<Control Name="editKeyboardID" Title="Keyboard ID">
<p>
The base filename of the keyboard, project and package. This must conform to the Keyman
keyboard identifier rules, using the characters a-z, 0-9 and _ (underscore) only.
</p>
</Control>
<Control Name="editProjectFilename" Title="Keyboard ID">
<p>
A Keyboard Project (.kpj) file is store inside the base path, this cannot be changed to anywhere else.
</p>
</Control>
<Control Name="cmdKeyboardAddLanguage" Title="Add a Language">
<p>This button will open up the BCP 47 Tag window to add the keyboard's language tag, script tag,
region tag, and language name.</p>
</Control>
<Control Name="cmdKeyboardEditLanguage" Title="Edit a Language">
<p>
Click on Edit... to open up the BCP 47 Tag window with the language information.
</p>
</Control>
<Control Name="cmdKeyboardRemoveLanguage" Title="Remove a Language">
<p>
Delete the selected language.
</p>
</Control>
<Control Name="cmdOK" Title="OK">
<p>
Once everything is ready, click OK and enjoy developing the keyboard.
</p>
</Control>
<Control Name="cmdCancel" Title="Cancel">
<p>Click Cancel to close the dialog and reset the process entirely.</p>
</Control>
</Form>
<!-- New Project -->
<Form Name="context/new-project">
<Control Name="lvItems" Title="New Project">
<p>
Creates a new Keyman Keyboard, LDML Keyboard, or a Wordlist Lexical Model,
or by importing from another source. Details are written on individual icons; clicking on
them shows a detailed explanation of each keyboard project.
</p>
</Control>
<Control Name="cmdOK" Title="OK with the Selected Project">
<p>
Once decided on a project to create, click OK will take you to the next step.
</p>
</Control>
<Control Name="cmdCancel" Title="Cancel Project Creation">
<p>
Cancels creating a project.
</p>
</Control>
</Form>
</ContextHelp>

4
developer/src/tike/xml/layoutbuilder/builder.xsl
View file

@ -271,6 +271,10 @@
</form>
</div>
<div id='addPlatformDialogNoPlatformsToAdd' title='Add platform'>
<p>All available platforms have already been added.</p>
</div>
<div id='addLayerDialog' title='Add layer'>
<form>
<fieldset>

24
developer/src/tike/xml/layoutbuilder/platform-controls.js
View file

@ -5,14 +5,20 @@ $(function() {
for (var platform in KVKL) {
platforms[platform] = 0;
}
let nPlatforms = 0;
for (platform in platforms) {
if (platforms[platform]) {
var opt = document.createElement('option');
$(opt).text(platform);
$('#selAddPlatform').append(opt);
nPlatforms++;
}
}
$('#addPlatformDialog').dialog('open')
if(nPlatforms == 0) {
$('#addPlatformDialogNoPlatformsToAdd').dialog('open')
} else {
$('#addPlatformDialog').dialog('open')
}
});
$('#btnDelPlatform').click(function () {
@ -69,6 +75,22 @@ $(function() {
}
});
//
// Platform dialog -- no platforms to add
//
$('#addPlatformDialogNoPlatformsToAdd').dialog({
autoOpen: false,
height: 150,
width: 350,
modal: true,
buttons: {
"OK": function () {
$(this).dialog('close');
}
}
});
//
// Platform Properties Dialog
//

4
docs/build/macos.md vendored
View file

@ -77,10 +77,10 @@ PATH="$HOMEBREW_PREFIX/opt/coreutils/libexec/gnubin:$PATH"
## KeymanWeb Dependencies
* node.js 18+, emscripten 3.1.46 or later, openjdk 8
* node.js 22+, emscripten 3.1.46 or later
```shell
brew install node emscripten openjdk@8
brew install node emscripten
```
Note: if you install emscripten with brew on macOS, only emscripten binaries are

7
linux/debian/changelog
View file

@ -1,3 +1,10 @@
keyman (17.0.326-1) unstable; urgency=medium
* New upstream release
* Re-release to Debian
-- Eberhard Beilharz <eb1@sil.org> Mon, 03 Jun 2024 21:58:16 +0200
keyman (17.0.295-1) unstable; urgency=medium
* Remove ibus-keyman.post{inst,rm} (closes: #1034040)

2
linux/ibus-keyman/tests/scripts/run-tests.sh
View file

@ -63,7 +63,7 @@ function run_tests() {
G_TEST_BUILDDIR="$(dirname "$0")/../../../build/$(arch)/${CONFIG}/tests"
setup "$DISPLAY_SERVER" "$ENV_FILE" "$CLEANUP_FILE" "$PID_FILE"
setup "$DISPLAY_SERVER" "$ENV_FILE" "$CLEANUP_FILE" "$PID_FILE" --standalone
echo "# NOTE: When the tests fail check /tmp/ibus-engine-keyman.log and /tmp/ibus-daemon.log!"
echo ""

28
linux/ibus-keyman/tests/scripts/test-helper.inc.sh
View file

@ -116,7 +116,7 @@ function _setup_init() {
echo > "$CLEANUP_FILE"
echo > "$PID_FILE"
TEMP_DATA_DIR=$(mktemp --directory)
echo "rm -rf ${TEMP_DATA_DIR} || true" >> "$CLEANUP_FILE"
echo "rm -rf \"${TEMP_DATA_DIR}\" || true # TEMP_DATA_DIR" >> "${CLEANUP_FILE}"
COMMON_ARCH_DIR=
[ -d "${TOP_SRCDIR}"/../../core/build/arch ] && COMMON_ARCH_DIR=${TOP_SRCDIR}/../../core/build/arch
@ -170,7 +170,7 @@ function _setup_display_server() {
# mutter-Message: 18:56:15.422: Using Wayland display name 'wayland-1'
mutter --wayland --headless --no-x11 --virtual-monitor 1024x768 &> "$TMPFILE" &
PID=$!
echo "kill -9 ${PID} || true" >> "$CLEANUP_FILE"
echo "kill -9 ${PID} || true # mutter" >> "$CLEANUP_FILE"
echo "${PID} mutter" >> "${PID_FILE}"
sleep 1s
export WAYLAND_DISPLAY
@ -189,7 +189,7 @@ function _setup_display_server() {
break
fi
done
echo "kill -9 ${PID} || true" >> "$CLEANUP_FILE"
echo "kill -9 ${PID} || true # Xvfb" >> "$CLEANUP_FILE"
echo "${PID} Xvfb" >> "${PID_FILE}"
while true; do
echo "Starting Xephyr..."
@ -201,12 +201,12 @@ function _setup_display_server() {
break
fi
done
echo "kill -9 ${PID} || true" >> "$CLEANUP_FILE"
echo "kill -9 ${PID} || true # Xephyr" >> "$CLEANUP_FILE"
echo "${PID} Xephyr" >> "${PID_FILE}"
echo "Starting metacity"
metacity --display=:${DISP_XEPHYR} &> /dev/null &
PID=$!
echo "kill -9 ${PID} || true" >> "$CLEANUP_FILE"
echo "kill -9 ${PID} || true # metacity" >> "$CLEANUP_FILE"
echo "${PID} metacity" >> "${PID_FILE}"
export DISPLAY=:${DISP_XEPHYR}
@ -241,16 +241,23 @@ function _setup_schema_and_gsettings() {
}
function _setup_ibus() {
local ENV_FILE CLEANUP_FILE PID_FILE PID
local ENV_FILE CLEANUP_FILE PID_FILE PID STANDALONE
ENV_FILE=$1
CLEANUP_FILE=$2
PID_FILE=$3
STANDALONE=${4:-}
echo "Starting ibus-daemon..."
#shellcheck disable=SC2086
ibus-daemon ${ARG_VERBOSE-} --daemonize --panel=disable --address=unix:abstract="${TEMP_DATA_DIR}/test-ibus" ${IBUS_CONFIG-} &> /tmp/ibus-daemon.log
PID=$(pgrep -f "${TEMP_DATA_DIR}/test-ibus")
echo "kill -9 ${PID} || true" >> "$CLEANUP_FILE"
if [[ "${STANDALONE}" == "--standalone" ]]; then
# manual test run
echo "if kill -9 ${PID}; then ibus restart || ibus start; fi # ibus-daemon" >> "${CLEANUP_FILE}"
else
# test run as part of the build
echo "kill -9 ${PID} || true" >> "${CLEANUP_FILE}"
fi
echo "${PID} ibus-daemon" >> "${PID_FILE}"
sleep 1s
@ -263,17 +270,18 @@ function _setup_ibus() {
#shellcheck disable=SC2086
"${TOP_BINDIR}/src/ibus-engine-keyman" --testing ${ARG_VERBOSE-} &> /tmp/ibus-engine-keyman.log &
PID=$!
echo "kill -9 ${PID} || true" >> "$CLEANUP_FILE"
echo "kill -9 ${PID} || true # ibus-engine-keyman" >> "${CLEANUP_FILE}"
echo "${PID} ibus-engine-keyman" >> "${PID_FILE}"
sleep 1s
}
function setup() {
local DISPLAY_SERVER ENV_FILE CLEANUP_FILE PID_FILE TESTBASEDIR TESTDIR
local DISPLAY_SERVER ENV_FILE CLEANUP_FILE PID_FILE TESTBASEDIR TESTDIR STANDALONE
DISPLAY_SERVER=$1
ENV_FILE=$2
CLEANUP_FILE=$3
PID_FILE=$4
STANDALONE=${5:-}
_setup_init "${ENV_FILE}" "${CLEANUP_FILE}" "${PID_FILE}"
@ -287,7 +295,7 @@ function setup() {
_setup_test_dbus_server "${ENV_FILE}" "${CLEANUP_FILE}"
_setup_display_server "${ENV_FILE}" "${CLEANUP_FILE}" "${PID_FILE}" "${DISPLAY_SERVER}"
_setup_schema_and_gsettings "${ENV_FILE}"
_setup_ibus "${ENV_FILE}" "${CLEANUP_FILE}" "${PID_FILE}"
_setup_ibus "${ENV_FILE}" "${CLEANUP_FILE}" "${PID_FILE}" "${STANDALONE}"
}
function setup_display_server_only() {

4
web/src/engine/main/src/keymanEngine.ts
View file

@ -349,6 +349,10 @@ export default class KeymanEngine<
this.core.keyboardProcessor.layerStore.handler = this.osk.layerChangeHandler;
}
this._osk = value;
// As the `new context` ruleset is designed to facilitate OSK layer-change updates
// based on the context being entered, we want the keyboard processor's current
// contextDevice to match that of the active OSK. See #11740.
this.core.keyboardProcessor.contextDevice = value.targetDevice ?? this.config.softDevice;
if(value) {
// Don't build an OSK if no keyboard is available yet; avoid the extra flash.
if(this.contextManager.activeKeyboard) {

3
web/src/engine/osk/src/input/gestures/browser/oskSubKey.ts
View file

@ -16,8 +16,7 @@ export default class OSKSubKey extends OSKKey {
}
getId(): string {
// Create (temporarily) unique ID by prefixing 'popup-' to actual key ID
return 'popup-'+this.layer+'-'+this.spec['id'];
return 'popup-'+this.spec.elementID;
}
construct(osk: VisualKeyboard, baseKey: KeyElement, width: number, topMargin: boolean): HTMLDivElement {

13
web/src/engine/osk/src/views/floatingOskView.ts
View file

@ -51,6 +51,7 @@ export default class FloatingOSKView extends OSKView {
// Add header element to OSK only for desktop browsers
this.titleBar = new TitleBar(this.titleDragHandler);
//
this.titleBar.on('help', () => {
this.legacyEvents.callEvent('helpclick', {});
});
@ -91,6 +92,12 @@ export default class FloatingOSKView extends OSKView {
listenerSpy.on('listenerremoved', onListenedEvent);
}
if(this.activeKeyboard) {
// If the keyboard was loaded during OSK init, we may need to set the
// title in place now, as it wasn't possible at the standard time.
this.postKeyboardAdjustments();
}
this.loadPersistedLayout();
}
@ -119,6 +126,12 @@ export default class FloatingOSKView extends OSKView {
}
protected postKeyboardAdjustments() {
// It is possible for this to be called during OSK initialization,
// when `this.titleBar` has not yet been initialized.
if(!this.titleBar) {
return;
}
// Add header element to OSK only for desktop browsers
this.enableMoveResizeHandlers();
if(this.activeKeyboard) {

1
web/src/test/manual/web/index.html
View file

@ -75,6 +75,7 @@
<h2><a href="./text_selection_tests_9073/index.html">Test text selection (#9073)</a></h2>
<h2><a href="./pr10506/index.html">Test key-cap scaling / font load interactions (#10506)</a></h2>
<h2><a href="./init-race-10743/index.html">Test page interaction + engine-initialization race condition handling (#10743)</a></h2>
<h2><a href="./issue11785/index.html">Test OSK loading with early add-keyboard calls (#11785)</a></h2>
<h1>Other</h1>
<h2><a href="./regression-tests/index.html">Keystroke processing regression test engine.</a></h2>
<hr>

109
web/src/test/manual/web/issue11785/index.html Normal file
View file

@ -0,0 +1,109 @@
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="content-type" content="text/html; charset=utf-8" />
<!-- Set the viewport width to match phone and tablet device widths -->
<meta name="viewport" content="width=device-width,user-scalable=no" />
<!-- Allow KeymanWeb to be saved to the iPhone home screen -->
<meta name="apple-mobile-web-app-capable" content="yes" />
<!-- Enable IE9 Standards mode -->
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<title>KeymanWeb Test Page - Keyboard Quick-Load</title>
<!-- Your page CSS -->
<style type='text/css'>
body {font-family: Tahoma,helvetica;}
h3 {font-size: 1em;font-weight:normal;color: darkred; margin-bottom: 4px}
.test {font-size: 24px; width:80%; min-height:30px; border: 1px solid gray;}
#KeymanWebControl {width:50%;min-width:600px;}
</style>
<!-- Insert uncompiled KeymanWeb source scripts -->
<script src="../../../../../build/publish/debug/keymanweb.js" type="application/javascript"></script>
<!--
For desktop browsers, a script for the user interface must be inserted here.
Standard UIs are toggle, button, float and toolbar.
The toolbar UI is best for any page designed to support keyboards for
a large number of languages.
-->
<script src="../../../../../build/publish/debug/kmwuitoggle.js"></script>
<!-- Add keyboard management script for local selection of keyboards to use -->
<script src="../commonHeader.js"></script>
<!-- Initialization: set paths to keyboards, resources and fonts as required -->
<script>
var kmw=window.keyman;
kmw.init({
attachType:'auto',
});
// Note: explicitly NOT deferred or waiting on the `init` Promise. This is
// the trigger for this page's test!
loadKeyboards(1);
</script>
</head>
<!-- Sample page HTML -->
<body>
<h2>KeymanWeb Test Page - Keyboard Quick-Load</h2>
<div>
<!--
The following DIV is used to position the Button or Toolbar User Interfaces on the page.
If omitted, those User Interfaces will appear at the top of the document body.
(It is ignored by other User Interfaces.)
-->
<div id='KeymanWebControl'></div>
<h3>Type in your language in this text area:</h3>
<textarea id='ta1' class='test' placeholder='Type here'></textarea>
<h3>or in this input field:</h3>
<input class='test' value='' placeholder='or here'/>
<!-- The following elements show how the language menu can be dynamically extended at any time -->
<h3>Add a keyboard by keyboard name:</h3>
<input type='input' id='kbd_id1' class='kmw-disabled' onkeypress="clickOnEnter(event,1);"/>
<input type='button' id='btn1' onclick='addKeyboard(1);' value='Add' />
<h3>Add a keyboard by BCP-47 language code:</h3>
<input type='input' id='kbd_id2' class='kmw-disabled' onkeypress="clickOnEnter(event,2);"/>
<input type='button' id='btn2' onclick='addKeyboard(2);' value='Add' />
<h3>Add a keyboard by language name(s):</h3>
<input type='input' id='kbd_id3' class='kmw-disabled' onkeypress="clickOnEnter(event,3);"/>
<input type='button' id='btn3' onclick='addKeyboard(3);' value='Add' />
<h3><a href="../.">Return to testing home page</a></h3>
</div>
<!-- include a blank div to enable scrolling -->
<div style="height:1000px"></div>
<p>--End of Document--</p>
</body>
<!--
*** DEVELOPER NOTE -- FIREFOX CONFIGURATION FOR TESTING ***
*
* If the URL bar starts with <b>file://</b>, Firefox may not load the font used
* to display the special characters used in the On-Screen Keyboard.
*
* To work around this Firefox bug, navigate to <b>about:config</b>
* and set <b>security.fileuri.strict_origin_policy</b> to <b>false</b>
* while testing.
*
* Firefox resolves website-based CSS URI references correctly without needing
* any configuration change, so this change should only be made for file-based testing.
*
***
-->
</html>