erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-11-03 01:27:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

feat(core): generator in core/src for util_normalize_table.h

Browse source 
- temporary header file generated by wasm during build
- built using icu
- test_unicode verifies the contents and synchronization with running ICU.
- this is used by util_normalize to provide normalization properties under wasm without needing to include ICU.

Fixes: #9467
This commit is contained in:
Steven R. Loomis 2024-06-05 15:47:20 -05:00
parent 0665d82d3f
commit f156a7264d
10 changed files with 145 additions and 329 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
core/build.sh
View file

@ -68,7 +68,6 @@ Libraries will be built in 'build/<target>/<configuration>/src'.
"uninstall uninstall libraries from current system" \
"${archtargets[@]}" \
"--no-tests do not configure tests (used by other projects)" \
"--update-unicode rebuild tables if the ICU4C Unicode version changes" \
"--test,-t=opt_tests test[s] to run (space separated)"
builder_parse "$@"
@ -178,12 +177,6 @@ if builder_start_action test:mac; then
builder_finish_action success test:mac
fi
if builder_has_option --update-unicode; then
# TODO: only works under mac. What's the right way to get the arch here?
meson compile -C ${KEYMAN_ROOT}/core/build/mac-x86_64/$BUILDER_CONFIGURATION tools/norm_data && mv -v ${KEYMAN_ROOT}/core/build/mac-x86_64/$BUILDER_CONFIGURATION/tools/nfd_table.h ${KEYMAN_ROOT}/resources/standards-data/unicode-character-database/
fi
# -------------------------------------------------------------------------------
do_action install

1
core/meson.build
View file

@ -44,4 +44,3 @@ subdir('doc')
subdir('include')
subdir('src')
subdir('tests')
subdir('tools')

24
core/src/meson.build
View file

@ -40,6 +40,29 @@ if icu_uc.found()
defns += '-DHAVE_ICU4C'
endif
# On wasm, generate util_normalize_table.h automatically from ICU
generated_headers = []
if cpp_compiler.get_id() == 'emscripten'
util_normalize_table_generator = executable('util_normalize_table_generator',
['util_normalize_table_generator.cpp'],
cpp_args: defns + warns,
include_directories: [inc],
link_args: links,
dependencies: [icu_uc, icu_i18n],
)
util_normalize_table_h = custom_target('util_normalize_table.h',
output: 'util_normalize_table.h',
command: [util_normalize_table_generator],
capture:true)
generated_headers += util_normalize_table_h
endif
kmx_files = files(
'actions_normalize.cpp',
@ -111,6 +134,7 @@ lib = library('keymancore',
kmx_files,
mock_files,
version_res,
generated_headers,
cpp_args: defns + warns + flags,
link_args: links,
version: lib_version,

2
core/src/util_normalize.cpp
View file

@ -31,7 +31,7 @@ EM_JS(char*, NormalizeNFC, (const char* input), {
});
// pull in the generated table
#include "../../resources/standards-data/unicode-character-database/nfd_table.h"
#include "util_normalize_table.h"
#endif

108
core/src/util_normalize_table_generator.cpp Normal file
View file

@ -0,0 +1,108 @@
/*
Copyright: © SIL International.
Description: Generator for util_normalize_table.h
Create Date: 5 Jun 2024
Authors: Steven R. Loomis
util_normalize_table.h is used under wasm by utilities in util_normalize.cpp to implement
normalization functions without needing ICU4C linked.
This generator is invoked automatically by meson as part of the build.
*/
#include "kmx/kmx_plus.h"
#include "kmx/kmx_xstring.h"
#define KMN_NO_ICU 0 // we will need ICU..
#include "core_icu.h"
#include <fstream>
#include <iostream>
#include <string>
#include <vector>
#include <assert.h>
#include <unicode/uchar.h>
int
write_nfd_table() {
#ifndef __EMSCRIPTEN__
std::cerr << "Note: This is unusual - this generator is usually only run under emscripten!" << std::endl;
#endif
// We write to stdout instead of to a file to avoid dealing with the filesystem under emscripten.
std::cerr << "Writing to stdout." << std::endl;
// write preamble
std::cout << "// GENERATED FILE: DO NOT EDIT" << std::endl;
std::cout << "//" << std::endl;
std::cout << "// util_normalize_table.h is generated by util_normalize_table_generator.cpp" << std::endl;
std::cout << "// and used by util_normalize.cpp" << std::endl;
std::cout << std::endl;
std::cout << "#pragma once" << std::endl;
std::cout << "#define KM_HASBOUNDARYBEFORE_UNICODE_VERSION \"" << U_UNICODE_VERSION << "\"" << std::endl;
std::cout << "#define KM_HASBOUNDARYBEFORE_ICU_VERSION \"" << U_ICU_VERSION << "\"" << std::endl;
std::cout << std::endl;
// we're going to need an NFD normalizer
UErrorCode status = U_ZERO_ERROR;
const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(status);
assert(U_SUCCESS(status));
// collect the raw list of chars that do NOT have a boundary before them.
std::vector<km_core_usv> noBoundary;
for (km_core_usv ch = 0; ch < 0x10FFFF; ch++) {
bool bb = nfd->hasBoundaryBefore(ch);
assert(!(ch == 0 && !bb)); // assert that we can use U+0000 as a terminator
if (bb) continue; //only emit nonboundary
noBoundary.push_back(ch);
}
// now, compress these into runs
std::vector<std::pair<km_core_usv,std::size_t>> runs; // start,len
km_core_usv first = 0;
km_core_usv last = 0;
for(auto i = noBoundary.begin(); i <= noBoundary.end(); i++) {
if (first == 0) {
first = last = *i;
} else {
last++;
if(i == noBoundary.end() || *i != last) {
// end of a run
runs.emplace_back(first, last - first);
if (i != noBoundary.end()) {
// setup for next
first = last = *i;
}
}
}
}
// finally, write out metadata and the runs themselves.
std::cout << "#define km_noBoundaryBefore_entries " << runs.size() << "\n";
std::cout << "static char32_t km_noBoundaryBefore[km_noBoundaryBefore_entries * 2 ] = {" << std::endl;
std::cout << "/* start codepoint, count (inclusive), ...range end */" << std::endl;
for (auto i = runs.begin(); i < runs.end(); i++) {
std::cout << "\t0x" << std::hex << i->first << std::dec << ",\t " << i->second << ", // ...0x" << std::hex << (i->first+i->second-1) << std::endl;
}
// termination
std::cout << "};" << std::endl;
std::cout << "// end" << std::endl;
std::cerr << "Wrote " << runs.size() << " runs representing " << noBoundary.size() << " entries." << std::endl;
return 0;
}
int
main(int /*argc*/, const char * /*argv*/[]) {
write_nfd_table();
return 0;
}

2
core/tests/unit/ldml/meson.build
View file

@ -132,7 +132,7 @@ test('test_context_normalization', test_context_normalization, suite: 'ldml')
# Build and run additional test_unicode test
test_unicode = executable('test_unicode', 'test_unicode.cpp',
['test_unicode.cpp', common_test_files],
['test_unicode.cpp', common_test_files, generated_headers],
cpp_args: defns + warns,
include_directories: [inc, libsrc, '../../../../developer/src/ext/json'],
link_args: links + tests_flags,

16
core/tests/unit/ldml/test_unicode.cpp
View file

@ -47,7 +47,7 @@
#ifdef __EMSCRIPTEN__
// Pull this in to verify versions
#include "../../../../resources/standards-data/unicode-character-database/nfd_table.h"
#include "util_normalize_table.h"
#endif
//-------------------------------------------------------------------------------------
@ -189,9 +189,15 @@ inline const char *boolstr(bool b) {
void test_has_boundary_before() {
std::cout << "= " << __FUNCTION__ << std::endl;
std::cout << "(this test only runs under emscripten. congratulations.)" << std::endl;
// static_assert(U_UNICODE_VERSION == KM_HASBOUNDARYBEFORE_UNICODE_VERSION, "nfd_table.h Unicode version does not match ICU's - see nfd_table.h");
std::cout << U_UNICODE_VERSION << "≈≈" << KM_HASBOUNDARYBEFORE_UNICODE_VERSION << std::endl;
std::cout << "I see we are on Emscripten / wasm! Now we will do some additional tests." << std::endl;
std::string icu4c_unicode(U_UNICODE_VERSION), header_unicode(KM_HASBOUNDARYBEFORE_UNICODE_VERSION),
icu4c_icu(U_ICU_VERSION), header_icu(KM_HASBOUNDARYBEFORE_ICU_VERSION);
std::cout << "Unicode: " << U_UNICODE_VERSION << ", and from the table file: " << KM_HASBOUNDARYBEFORE_UNICODE_VERSION << std::endl;
std::cout << "It would be very strange for these versions to be out of sync. Some sort of build or tool problem." << std::endl;
assert_basic_equal(icu4c_unicode, header_unicode);
assert_basic_equal(icu4c_icu, header_icu);
std::cout << std::endl << "Now, let's make sure has_nfd_boundary_before() matches ICU." << std::endl;
UErrorCode status = U_ZERO_ERROR;
const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(status);
@ -203,7 +209,7 @@ void test_has_boundary_before() {
auto icu_hbb = nfd->hasBoundaryBefore(cp);
if (km_hbb != icu_hbb) {
std::cerr << "Error: nfd_table.h said " << boolstr(km_hbb) << " but ICU said " << boolstr(icu_hbb) << " for "
std::cerr << "Error: util_normalize_table.h said " << boolstr(km_hbb) << " but ICU said " << boolstr(icu_hbb) << " for "
<< "has_nfd_boundary_before(0x" << std::hex << cp << std::dec << ")" << std::endl;
}
assert(km_hbb == icu_hbb);

33
core/tools/meson.build
View file

@ -1,33 +0,0 @@
# Copyright: © 2024 SIL International.
# Description: Cross platform build script to compile tool(s).
# Create Date: 31 May 2024
# Authors: Steven R. Loomis (SRL)
#
# TODO -- why are these differing from the standard.meson.build flags?
if cpp_compiler.get_id() == 'gcc' or cpp_compiler.get_id() == 'clang' or cpp_compiler.get_id() == 'emscripten'
warns = [
'-Wno-missing-field-initializers',
'-Wno-unused-parameter'
]
else
warns = []
endif
norm_unicode_update = executable('norm_unicode_update',
['norm_unicode_update.cpp'],
cpp_args: defns + warns,
include_directories: [inc, libsrc, '../../developer/src/ext/json'],
link_args: links,
dependencies: [icu_uc, icu_i18n],
# link_with: [lib],
objects: lib.extract_all_objects(recursive: false),
)
# ../../resources/standards-data/unicode-character-database/
norm_data = custom_target('norm_data', output: 'nfd_table.h', command: [norm_unicode_update, '@OUTPUT@'])
# TODO: execute it

84
core/tools/norm_unicode_update.cpp
View file

@ -1,84 +0,0 @@
#include "kmx/kmx_plus.h"
#include "kmx/kmx_xstring.h"
#include "core_icu.h"
#include <fstream>
#include <iostream>
#include <string>
#include <vector>
#include <assert.h>
#include <unicode/uchar.h>
#ifndef __EMSCRIPTEN__
int
write_nfd_table(const char *NFD_FILE) {
std::cout << " writing: " << NFD_FILE << std::endl;
auto f = std::ofstream(NFD_FILE);
assert(f.good());
// write preamble
f << "//NFD hasBoundaryBefore" << std::endl;
f << "#pragma once" << std::endl;
f << "#define KM_HASBOUNDARYBEFORE_UNICODE_VERSION \"" << U_UNICODE_VERSION << "\"" << std::endl;
f << "#define KM_HASBOUNDARYBEFORE_ICU_VERSION \"" << U_ICU_VERSION << "\"" << std::endl;
// we're going to need an NFD normalizer
UErrorCode status = U_ZERO_ERROR;
const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(status);
assert(U_SUCCESS(status));
std::vector<km_core_usv> noBoundary;
for (km_core_usv ch = 0; ch < 0x10FFFF; ch++) {
bool bb = nfd->hasBoundaryBefore(ch);
assert(!(ch == 0 && !bb)); // assert that we can use U+0000 as a terminator
if (bb) continue; //only emit nonboundary
noBoundary.push_back(ch);
}
std::vector<std::pair<km_core_usv,std::size_t>> runs; // start,len
km_core_usv first = 0;
km_core_usv last = 0;
for(auto i = noBoundary.begin(); i <= noBoundary.end(); i++) {
if (first == 0) {
first = last = *i;
} else {
last++;
if(i == noBoundary.end() || *i != last) {
// end of a run
runs.emplace_back(first, last - first);
if (i != noBoundary.end()) {
// setup for next
first = last = *i;
}
}
}
}
f << "#define km_noBoundaryBefore_entries " << runs.size() << "\n";
f << "static char32_t km_noBoundaryBefore[km_noBoundaryBefore_entries * 2 ] = {" << std::endl;
for (auto i = runs.begin(); i < runs.end(); i++) {
f << "\t0x" << std::hex << i->first << std::dec << ",\t " << i->second << ", // ...0x" << std::hex << (i->first+i->second-1) << std::endl;
}
// termination
f << "};" << std::endl;
return 0;
}
int
main(int argc, const char *argv[]) {
assert(argc == 2); // call with one param: @OUTPUT@
write_nfd_table(argv[1]);
return 0;
}
#else
int main(int argc, const char *argv[]) {
std::cerr << "Can't run this under Emscripten - run under another platform." << std::endl;
return 1;
}
#endif

197
resources/standards-data/unicode-character-database/nfd_table.h
View file

@ -1,197 +0,0 @@
//NFD hasBoundaryBefore
#pragma once
#define KM_HASBOUNDARYBEFORE_UNICODE_VERSION "15.0"
#define KM_HASBOUNDARYBEFORE_ICU_VERSION "73.1"
#define km_noBoundaryBefore_entries 190
static char32_t km_noBoundaryBefore[km_noBoundaryBefore_entries * 2 ] = {
0x300, 79, // ...0x34e
0x350, 32, // ...0x36f
0x483, 5, // ...0x487
0x591, 45, // ...0x5bd
0x5bf, 1, // ...0x5bf
0x5c1, 2, // ...0x5c2
0x5c4, 2, // ...0x5c5
0x5c7, 1, // ...0x5c7
0x610, 11, // ...0x61a
0x64b, 21, // ...0x65f
0x670, 1, // ...0x670
0x6d6, 7, // ...0x6dc
0x6df, 6, // ...0x6e4
0x6e7, 2, // ...0x6e8
0x6ea, 4, // ...0x6ed
0x711, 1, // ...0x711
0x730, 27, // ...0x74a
0x7eb, 9, // ...0x7f3
0x7fd, 1, // ...0x7fd
0x816, 4, // ...0x819
0x81b, 9, // ...0x823
0x825, 3, // ...0x827
0x829, 5, // ...0x82d
0x859, 3, // ...0x85b
0x898, 8, // ...0x89f
0x8ca, 24, // ...0x8e1
0x8e3, 29, // ...0x8ff
0x93c, 1, // ...0x93c
0x94d, 1, // ...0x94d
0x951, 4, // ...0x954
0x9bc, 1, // ...0x9bc
0x9cd, 1, // ...0x9cd
0x9fe, 1, // ...0x9fe
0xa3c, 1, // ...0xa3c
0xa4d, 1, // ...0xa4d
0xabc, 1, // ...0xabc
0xacd, 1, // ...0xacd
0xb3c, 1, // ...0xb3c
0xb4d, 1, // ...0xb4d
0xbcd, 1, // ...0xbcd
0xc3c, 1, // ...0xc3c
0xc4d, 1, // ...0xc4d
0xc55, 2, // ...0xc56
0xcbc, 1, // ...0xcbc
0xccd, 1, // ...0xccd
0xd3b, 2, // ...0xd3c
0xd4d, 1, // ...0xd4d
0xdca, 1, // ...0xdca
0xe38, 3, // ...0xe3a
0xe48, 4, // ...0xe4b
0xeb8, 3, // ...0xeba
0xec8, 4, // ...0xecb
0xf18, 2, // ...0xf19
0xf35, 1, // ...0xf35
0xf37, 1, // ...0xf37
0xf39, 1, // ...0xf39
0xf71, 5, // ...0xf75
0xf7a, 4, // ...0xf7d
0xf80, 5, // ...0xf84
0xf86, 2, // ...0xf87
0xfc6, 1, // ...0xfc6
0x1037, 1, // ...0x1037
0x1039, 2, // ...0x103a
0x108d, 1, // ...0x108d
0x135d, 3, // ...0x135f
0x1714, 2, // ...0x1715
0x1734, 1, // ...0x1734
0x17d2, 1, // ...0x17d2
0x17dd, 1, // ...0x17dd
0x18a9, 1, // ...0x18a9
0x1939, 3, // ...0x193b
0x1a17, 2, // ...0x1a18
0x1a60, 1, // ...0x1a60
0x1a75, 8, // ...0x1a7c
0x1a7f, 1, // ...0x1a7f
0x1ab0, 14, // ...0x1abd
0x1abf, 16, // ...0x1ace
0x1b34, 1, // ...0x1b34
0x1b44, 1, // ...0x1b44
0x1b6b, 9, // ...0x1b73
0x1baa, 2, // ...0x1bab
0x1be6, 1, // ...0x1be6
0x1bf2, 2, // ...0x1bf3
0x1c37, 1, // ...0x1c37
0x1cd0, 3, // ...0x1cd2
0x1cd4, 13, // ...0x1ce0
0x1ce2, 7, // ...0x1ce8
0x1ced, 1, // ...0x1ced
0x1cf4, 1, // ...0x1cf4
0x1cf8, 2, // ...0x1cf9
0x1dc0, 64, // ...0x1dff
0x20d0, 13, // ...0x20dc
0x20e1, 1, // ...0x20e1
0x20e5, 12, // ...0x20f0
0x2cef, 3, // ...0x2cf1
0x2d7f, 1, // ...0x2d7f
0x2de0, 32, // ...0x2dff
0x302a, 6, // ...0x302f
0x3099, 2, // ...0x309a
0xa66f, 1, // ...0xa66f
0xa674, 10, // ...0xa67d
0xa69e, 2, // ...0xa69f
0xa6f0, 2, // ...0xa6f1
0xa806, 1, // ...0xa806
0xa82c, 1, // ...0xa82c
0xa8c4, 1, // ...0xa8c4
0xa8e0, 18, // ...0xa8f1
0xa92b, 3, // ...0xa92d
0xa953, 1, // ...0xa953
0xa9b3, 1, // ...0xa9b3
0xa9c0, 1, // ...0xa9c0
0xaab0, 1, // ...0xaab0
0xaab2, 3, // ...0xaab4
0xaab7, 2, // ...0xaab8
0xaabe, 2, // ...0xaabf
0xaac1, 1, // ...0xaac1
0xaaf6, 1, // ...0xaaf6
0xabed, 1, // ...0xabed
0xfb1e, 1, // ...0xfb1e
0xfe20, 16, // ...0xfe2f
0x101fd, 1, // ...0x101fd
0x102e0, 1, // ...0x102e0
0x10376, 5, // ...0x1037a
0x10a0d, 1, // ...0x10a0d
0x10a0f, 1, // ...0x10a0f
0x10a38, 3, // ...0x10a3a
0x10a3f, 1, // ...0x10a3f
0x10ae5, 2, // ...0x10ae6
0x10d24, 4, // ...0x10d27
0x10eab, 2, // ...0x10eac
0x10efd, 3, // ...0x10eff
0x10f46, 11, // ...0x10f50
0x10f82, 4, // ...0x10f85
0x11046, 1, // ...0x11046
0x11070, 1, // ...0x11070
0x1107f, 1, // ...0x1107f
0x110b9, 2, // ...0x110ba
0x11100, 3, // ...0x11102
0x11133, 2, // ...0x11134
0x11173, 1, // ...0x11173
0x111c0, 1, // ...0x111c0
0x111ca, 1, // ...0x111ca
0x11235, 2, // ...0x11236
0x112e9, 2, // ...0x112ea
0x1133b, 2, // ...0x1133c
0x1134d, 1, // ...0x1134d
0x11366, 7, // ...0x1136c
0x11370, 5, // ...0x11374
0x11442, 1, // ...0x11442
0x11446, 1, // ...0x11446
0x1145e, 1, // ...0x1145e
0x114c2, 2, // ...0x114c3
0x115bf, 2, // ...0x115c0
0x1163f, 1, // ...0x1163f
0x116b6, 2, // ...0x116b7
0x1172b, 1, // ...0x1172b
0x11839, 2, // ...0x1183a
0x1193d, 2, // ...0x1193e
0x11943, 1, // ...0x11943
0x119e0, 1, // ...0x119e0
0x11a34, 1, // ...0x11a34
0x11a47, 1, // ...0x11a47
0x11a99, 1, // ...0x11a99
0x11c3f, 1, // ...0x11c3f
0x11d42, 1, // ...0x11d42
0x11d44, 2, // ...0x11d45
0x11d97, 1, // ...0x11d97
0x11f41, 2, // ...0x11f42
0x16af0, 5, // ...0x16af4
0x16b30, 7, // ...0x16b36
0x16ff0, 2, // ...0x16ff1
0x1bc9e, 1, // ...0x1bc9e
0x1d165, 5, // ...0x1d169
0x1d16d, 6, // ...0x1d172
0x1d17b, 8, // ...0x1d182
0x1d185, 7, // ...0x1d18b
0x1d1aa, 4, // ...0x1d1ad
0x1d242, 3, // ...0x1d244
0x1e000, 7, // ...0x1e006
0x1e008, 17, // ...0x1e018
0x1e01b, 7, // ...0x1e021
0x1e023, 2, // ...0x1e024
0x1e026, 5, // ...0x1e02a
0x1e08f, 1, // ...0x1e08f
0x1e130, 7, // ...0x1e136
0x1e2ae, 1, // ...0x1e2ae
0x1e2ec, 4, // ...0x1e2ef
0x1e4ec, 4, // ...0x1e4ef
0x1e8d0, 7, // ...0x1e8d6
0x1e944, 7, // ...0x1e94a
};