From 95b4469ea8c74cd4a3658d61a14eb18c7bcc3415 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Wed, 17 Aug 2022 15:53:53 -0500 Subject: [PATCH 1/8] feat(core): build out ldml structures - checkin spec to repo - new header: kmx_plus.h with data access structs and functions - static asserts to validate header sizes - early validation of data - internal dump of kmx+ data --- core/include/ldml/keyboardprocessor_ldml.h | 9 + core/include/ldml/keyboardprocessor_ldml.ts | 111 +++++++----- .../ldml/keyboardproessor_ldml_sects.h | 1 + core/src/kmx/kmx_plus.cpp | 60 +++++++ core/src/kmx/kmx_plus.h | 157 +++++++++++++++++ core/src/ldml/C7043_ldml.md | 165 ++++++++++++++++++ core/src/ldml/ldml_processor.cpp | 4 + core/src/meson.build | 1 + core/tests/unit/ldml/001_tiny.kmx | Bin 768 -> 768 bytes core/tests/unit/ldml/meson.build | 9 + core/tests/unit/ldml/test_kmx_plus.cpp | 10 ++ 11 files changed, 479 insertions(+), 48 deletions(-) create mode 100644 core/include/ldml/keyboardproessor_ldml_sects.h create mode 100644 core/src/kmx/kmx_plus.cpp create mode 100644 core/src/kmx/kmx_plus.h create mode 100644 core/src/ldml/C7043_ldml.md create mode 100644 core/tests/unit/ldml/test_kmx_plus.cpp diff --git a/core/include/ldml/keyboardprocessor_ldml.h b/core/include/ldml/keyboardprocessor_ldml.h index fe10cf1f39..996f03494a 100644 --- a/core/include/ldml/keyboardprocessor_ldml.h +++ b/core/include/ldml/keyboardprocessor_ldml.h @@ -6,6 +6,13 @@ // #pragma once +#define LDML_LENGTH_HEADER 8 +#define LDML_LENGTH_KEYS 16 +#define LDML_LENGTH_LOCA 12 +#define LDML_LENGTH_META 36 +#define LDML_LENGTH_SECT 16 +#define LDML_LENGTH_STRS 16 +#define LDML_LENGTH_VKEY 12 #define LDML_META_SETTINGS_FALLBACK_OMIT 1 #define LDML_META_SETTINGS_TRANSFORMFAILURE_OMIT 2 #define LDML_META_SETTINGS_TRANSFORMPARTIAL_HIDE 4 @@ -19,5 +26,7 @@ #define LDML_SECTION_SECT ((uint32_t)'sect') // Section ID #define LDML_SECTION_STRS ((uint32_t)'strs') +// Section ID +#define LDML_SECTION_VKEY ((uint32_t)'vkey') #define LDML_STRS_FLAGS_EXTEND 1 #define LDML_VERSION "1.0" diff --git a/core/include/ldml/keyboardprocessor_ldml.ts b/core/include/ldml/keyboardprocessor_ldml.ts index bf1eb2c9b5..ba15c74a17 100644 --- a/core/include/ldml/keyboardprocessor_ldml.ts +++ b/core/include/ldml/keyboardprocessor_ldml.ts @@ -10,57 +10,72 @@ // It is not updated automatically. // TODO-LDML: namespace com.keyman.core.ldml { +/** + * Constants for the KMXPlus data format + * These are shared between the data access layer and the compiler + */ +export const constants = { /** - * Constants for the KMXPlus data format - * These are shared between the data access layer and the compiler + * The version of the LDML processor */ - export const constants = { - /** - * The version of the LDML processor - */ - version: '1.0', + version: '1.0', - /** - * Section ID for the keybag - */ - section_keys: 'keys', - /** - * Section ID for the locale list - */ - section_loca: 'loca', - /** - * Section ID for the metadata - */ - section_meta: 'meta', - /** - * Section ID for the section header - */ - section_sect: 'sect', - /** - * Section ID for the string table - */ - section_strs: 'strs', + /** + * Length of a raw section header, in bytes + */ + length_header: 8, + /** + * Section ID for the keybag + */ + section_keys: 'keys', + length_keys: 16, + /** + * Section ID for the locale list + */ + section_loca: 'loca', + length_loca: 12, + /** + * Section ID for the metadata + */ + section_meta: 'meta', + length_meta: 36, + /** + * bitwise or value for fallback=omit in meta.settings + */ + meta_settings_fallback_omit: 1, + /** + * bitwise or value for transformFailure=omit in meta.settings + */ + meta_settings_transformFailure_omit: 2, + /** + * bitwise or value for transformPartial=hide in meta.settings + */ + meta_settings_transformPartial_hide: 4, + /** + * Section ID for the section header + */ + section_sect: 'sect', + /** + * Minimum length of the 'sect' section, not including entries + */ + length_sect: 16, + /** + * Section ID for the string table + */ + section_strs: 'strs', - /** - * bitwise or value for fallback=omit in meta.settings - */ - meta_settings_fallback_omit: 1, - /** - * bitwise or value for transformFailure=omit in meta.settings - */ - meta_settings_transformFailure_omit: 2, - /** - * bitwise or value for transformPartial=hide in meta.settings - */ - meta_settings_transformPartial_hide: 4, + length_strs: 16, + /** + * bitwise or value for extend in strs[key].flags. + * If bit is 1, then 'to' is a string. + * If bit is 0, then 'to' is an offset. + * + * `extend = flags & strs_flags_extend` + */ + strs_flags_extend: 1, - /** - * bitwise or value for extend in strs[key].flags. - * If bit is 1, then 'to' is a string. - * If bit is 0, then 'to' is an offset. - * - * `extend = flags & strs_flags_extend` - */ - strs_flags_extend: 1, - }; + section_vkey: 'vkey', + length_vkey: 12, + +}; // } diff --git a/core/include/ldml/keyboardproessor_ldml_sects.h b/core/include/ldml/keyboardproessor_ldml_sects.h new file mode 100644 index 0000000000..aa78802ff9 --- /dev/null +++ b/core/include/ldml/keyboardproessor_ldml_sects.h @@ -0,0 +1 @@ +keyboardproessor_ldml_structs.h diff --git a/core/src/kmx/kmx_plus.cpp b/core/src/kmx/kmx_plus.cpp new file mode 100644 index 0000000000..28d58dd27c --- /dev/null +++ b/core/src/kmx/kmx_plus.cpp @@ -0,0 +1,60 @@ +#include +#include + +namespace km { +namespace kbp { +namespace kmx { + +static void +dump_section_name(KMX_DWORD ident) { + for (int i = 0; i < 4; i++) { + putchar((ident & 0xFF000000) >> 24); + ident <<= 8; + } +} + +static void +dump_kmxplus_header(const COMP_KMXPLUS_HEADER* hdr) { + dump_section_name(hdr->ident); + printf(": (%X) size 0x%X\n", hdr->ident, hdr->size); +} + +static void +dump_kmxplus_sect(const uint8_t* /*data*/, const COMP_KMXPLUS_SECT* sect) { + dump_kmxplus_header((const COMP_KMXPLUS_HEADER*)sect); + printf("sect: total 0x%X\n", sect->total); + printf("sect: count 0x%X\n", sect->count); + + for (KMX_DWORD i = 0; i < sect->count; i++) { + dump_section_name(sect->entries[i].sect); + printf(" sect#%d: %X @ %X\n", i, sect->entries[i].sect, sect->entries[i].offset); + } +} + +void +dump_kmxplus_data(const uint8_t* data) { + const COMP_KMXPLUS_SECT* sect = as_kmxplus_sect((void*)data); + if (sect == NULL) { + printf("Err: 'sect' null from %p\n", data); + return; + } + dump_kmxplus_sect(data, sect); +} + +void +dump_kmxplus_data(kmx::PCOMP_KEYBOARD keyboard) { + printf("dump_kmxplus_data(): Got a PCOMP_KEYBOARD at %p\n", keyboard); + if (!(keyboard->dwFlags & KF_KMXPLUS)) { + printf("Err: flags KF_KMXPLUS not set\n"); + return; + } + const COMP_KEYBOARD_EX* ex = reinterpret_cast(keyboard); + + printf("KMXPlus offset 0x%X, KMXPlus size 0x%X\n", ex->kmxplus.dpKMXPlus, ex->kmxplus.dwKMXPlusSize); + const uint8_t* rawdata = reinterpret_cast(keyboard); + dump_kmxplus_data(rawdata + ex->kmxplus.dpKMXPlus); +} + +} // namespace kmx +} // namespace kbp +} // namespace km diff --git a/core/src/kmx/kmx_plus.h b/core/src/kmx/kmx_plus.h new file mode 100644 index 0000000000..7da4750e69 --- /dev/null +++ b/core/src/kmx/kmx_plus.h @@ -0,0 +1,157 @@ +/* + Copyright: Copyright (C) 2022 SIL International. + Authors: srl295 + This file defines the structure of a kmx_plus file, starting at COMP_KEYBOARD_KMXPLUSINFO.dpKMXPlus +*/ + +#pragma once + +#include +#include +#include + +namespace km { +namespace kbp { +namespace kmx { + +struct COMP_KMXPLUS_HEADER { + KMX_DWORD ident; // 0000 Section name + KMX_DWORD size; // 0004 Section length +}; + +// Assert that the length matches the declared length +static_assert(sizeof(struct COMP_KMXPLUS_HEADER) == LDML_LENGTH_HEADER, "mismatched size of section header"); + +struct COMP_KMXPLUS_SECT_ENTRY { + KMX_DWORD sect; // 0010+ Section identity + KMX_DWORD offset; // 0014+ Section offset relative to dpKMXPlus of section +}; + +struct COMP_KMXPLUS_SECT { + COMP_KMXPLUS_HEADER header; + KMX_DWORD total; // 0008 KMXPlus entire length + KMX_DWORD count; // 000B number of section headers + COMP_KMXPLUS_SECT_ENTRY entries[0]; // 0010 section entries +}; + +// Assert that the length matches the declared length +static_assert(sizeof(struct COMP_KMXPLUS_SECT) == LDML_LENGTH_SECT, "mismatched size of section sect"); + +struct COMP_KMXPLUS_STRS_ENTRY { + KMX_DWORD offset; // 0010+ offset from this blob + KMX_DWORD length; // 0014+ str length (UTF-16LE units) +}; + +struct COMP_KMXPLUS_STRS { + COMP_KMXPLUS_HEADER header; + KMX_DWORD count; // 0008 count of str entries + KMX_DWORD reserved; // 000C padding + COMP_KMXPLUS_STRS_ENTRY entries[0]; // 0010+ entries +}; + +static_assert(sizeof(struct COMP_KMXPLUS_STRS) == LDML_LENGTH_STRS, "mismatched size of section strs"); + +struct COMP_KMXPLUS_META { + COMP_KMXPLUS_HEADER header; + KMX_DWORD name; + KMX_DWORD author; + KMX_DWORD conform; + KMX_DWORD layout; + KMX_DWORD normalization; + KMX_DWORD indicator; + KMX_DWORD settings; +}; + +static_assert(sizeof(struct COMP_KMXPLUS_META) == LDML_LENGTH_META, "mismatched size of section meta"); + +struct COMP_KMXPLUS_LOCA_ENTRY { + KMX_DWORD locale; // 000C+ locale string entry +}; + +struct COMP_KMXPLUS_LOCA { + COMP_KMXPLUS_HEADER header; + KMX_DWORD count; // 0008 number of locales + COMP_KMXPLUS_LOCA_ENTRY entries[0]; +}; + +static_assert(sizeof(struct COMP_KMXPLUS_LOCA) == LDML_LENGTH_LOCA, "mismatched size of section loca"); + +struct COMP_KMXPLUS_KEYS_ENTRY { + KMX_DWORD vkey; + KMX_DWORD mod; + KMX_DWORD to; + KMX_DWORD flags; +}; + +struct COMP_KMXPLUS_KEYS { + COMP_KMXPLUS_HEADER header; + KMX_DWORD count; // number of keys + KMX_DWORD reserved; // padding + COMP_KMXPLUS_KEYS_ENTRY entries[0]; +}; + +static_assert(sizeof(struct COMP_KMXPLUS_KEYS) == LDML_LENGTH_KEYS, "mismatched size of section keys"); + +struct COMP_KMXPLUS_VKEY_ENTRY { + KMX_DWORD vkey; + KMX_DWORD target; +}; + +struct COMP_KMXPLUS_VKEY { + COMP_KMXPLUS_HEADER header; + KMX_DWORD count; + COMP_KMXPLUS_VKEY_ENTRY entries[0]; +}; + +static_assert(sizeof(struct COMP_KMXPLUS_VKEY) == LDML_LENGTH_VKEY, "mismatched size of section vkey"); + +/** + * @brief Validate that this data is the named section. + * + * @param data raw data + * @param ident 4-byte section type + * @return COMP_KMXPLUS_ALLDATA* or null + */ +static inline const COMP_KMXPLUS_HEADER * +validate_as_section(void *data, uint32_t ident) { + if (!data) { + return NULL; + } + const COMP_KMXPLUS_HEADER *all = reinterpret_cast(data); + // TODO-LDML these fail on 000null .. + // assert(all->size >= LDML_LENGTH_HEADER); + // assert(ident == all->ident); + if (ident != all->ident || (all->size < LDML_LENGTH_HEADER)) { + return NULL; // invalid header or wrong section + } + return all; +} + +/** + * convert raw data to section + * @return section data or null on error + */ +static inline const COMP_KMXPLUS_SECT * +as_kmxplus_sect(void *data) { + const COMP_KMXPLUS_HEADER *all = validate_as_section(data, LDML_SECTION_SECT); + return reinterpret_cast(all); +} + +// TODO-LDML: add as_kmxplus_keys, etc. + +/** + * @brief Temporary function to dump raw data + * + */ +void dump_kmxplus_data(void *kmxplusdata); + +/** + * @brief Temporary functino to dump raw data + * + * @param keyboard + */ +void dump_kmxplus_data(kmx::PCOMP_KEYBOARD keyboard); + +} // namespace kmx +} // namespace kbp +} // namespace km diff --git a/core/src/ldml/C7043_ldml.md b/core/src/ldml/C7043_ldml.md new file mode 100644 index 0000000000..934e50c9a2 --- /dev/null +++ b/core/src/ldml/C7043_ldml.md @@ -0,0 +1,165 @@ +# KMX Plus Binary Format #7043 + +- copied from + +- Authors: MD SL + +## C7043.0 Introduction + +This document discusses the binary format for KMXPlus, which contains keyboards converted from source LDML data. + +Draft spec PR: + +## C7043.1 Principles + +- The data described here is located at byte offset `dpKMXPlus`. +- All integer values are unsigned 32-bit little-endian unless otherwise specified. +- All strings are UTF-16LE unless otherwise specified. (See the `strs` section.) String data items are identified with `str:`, indicating a 32 bit index into the `strs` table. +- All offsets are 32-bit little-endian values. For all sections except for the `'sect'` section (which see), offsets are relative to the beginning of each section. + +## C7043.2 Sections + +- Data is divided into several sections. The very first section is the `'sect'` section which is the table of contents. +- All sections, including the first, begin on a 128-bit boundary with zero padding as needed. +- All sections begin with a 32-bit (four character) section identifier, and a 32-bit section size. +- Other than the `sect` table itself, the rest of the sections follow in binary order in the file. In other words, the binary ordering of the section identifiers determines the order of the file layout. + +### C7043.2.1 `sect`—Section Table of contents + +The very first section is a table of contents listing the rest of the sections. The table of contents does not list itself. + +This is the only section where all byte offsets are relative to the value of `dpKMXPlus`. + +| ∆ | Bits | Name | Description | +|---|------|---------|------------------------------------------| +| 0 | 32 | ident | `sect` | +| 4 | 32 | size | int: Length of section | +| 8 | 32 | total | int: KMXPlus entire length | +|12 | 32 | count | int: Number of following section headers | + +Then for `count` repetitions: + +| ∆ | Bits | Name | Description | +|---|------|---------|---------------------------------------------------| +|16+| 32 | sect | Section identity, e.g. `loca` | +|20+| 32 | offset | off: offset relative to `dpKMXPlus` of section | + +This list is in sorted order based on the `sect` identifier. + +### C7043.2.2 `strs`—Strings + +All strings are stored in the Strings section. + +| ∆ | Bits | Name | Description | +|---|------|---------------|-------------------------------------| +| 0 | 32 | ident | `strs` | +| 4 | 32 | size | int: Length of section | +| 8 | 32 | count | int: Number of strings | +|12 | 32 | reserved | Padding | + +Then for each string: + +| ∆ | Bits | Name | Description | +|---|------|---------------|-----------------------------------------------| +|16+| 32 | offset | off: Offset to string | +|20+| 32 | length | int: Length of string in UTF-16LE code units | + +After the string offset table comes the actual UTF-16LE data. There is a null (\u0000) after each string, which is _not_ included in the string length. + +The string offset table, and then strings themselves, are sorted according to a binary codepoint sort, not including the null. + +### C7043.2.3 `meta`—Metadata + +| ∆ | Bits | Name | Description | +|---|------|---------------|-------------------------------------| +| 0 | 32 | ident | `meta` | +| 4 | 32 | size | int: Length of section | +| 8 | 32 | name | str: Keyboard name | +|12 | 32 | author | str: Keyboard author | +|16 | 32 | conform | str: CLDR 'conformsTo' version | +|20 | 32 | layout | str: layout type | +|24 | 32 | normalization | str: normalization mode | +|28 | 32 | indicator | str: indicator | +|32 | 32 | settings | int: keyboard settings | + +The `settings` is a 32-bit bitfield as below: + +| Bit position | Meaning | Description | +|--------------|----------|---------------------------------------------| +| 0 | fallback | fallback=omit | +| 1 | transformFailure | transformFailure=omit | +| 2 | transformPartial | transformPartial=hide | + + +### C7043.2.4 `loca`—Locales + + +| ∆ | Bits | Name | Description | +|---|------|---------|------------------------------------------| +| 0 | 32 | ident | `loca` | +| 4 | 32 | size | int: Length of section | +| 8 | 32 | count | int: Number of locales | + +`count` is always ≥1, because a keyboard always has a primary locale identifier. + +For each locale ID in `count` + +| ∆ | Bits | Name | Description | +|---|------|---------|------------------------------------------| +|12+| 32 | locale | str: Locale ID in BCP47 format | + +The first locale ID is always the primary locale identifier. The rest of the locale IDs (starting at offset 16) are in sorted binary order. + +### C7043.2.5 `keys`—Keybag + +| ∆ | Bits | Name | Description | +|---|------|---------|------------------------------------------| +| 0 | 32 | ident | `keys` | +| 4 | 32 | size | int: Length of section | +| 8 | 32 | count | int: Number of keys | +|12 | 32 | reserved | reserved | + +The keys are sorted in binary order based on the `vkey` and `mod` fields. + +For each key: + +| ∆ | Bits | Name | Description | +|---|------|---------|------------------------------------------| +|16+| 32 | vkey | int: vkey ID | +|20+| 32 | mod | int: modifier key flags | +|24+| 32 | to | str: output string OR UTF-32LE codepoint | +|28+| 32 | flags | int: per-key flags | + +- `vkey`: If this is 0-255, it is the resolved standard/predefined vkey (K_A, etc.). It is resolved because the `vkeyMap` from LDML has already been applied. If this is 256 or above, it is a custom touch layout vkey generated by the compiler. +- `mod`: TODO define this. 0 for no modifiers. +- `flags`: Flags is a 32-bit bitfield defined as below: + +| Bit position | Meaning | Description | +|--------------|----------|---------------------------------------------| +| 0 | extend | 0: `to` is a char, 1: `to` is a string | + +- `to`: If `extend` is 0, `to` is a UTF-32LE codepoint. If `extend` is 1, `to` is a 32 bit index into the `strs` table. + +### C7043.2.6 `vkey`—VKey Map + +| ∆ | Bits | Name | Description | +|---|------|---------|------------------------------------------| +| 0 | 32 | ident | `vkey` | +| 4 | 32 | size | int: Length of section | +| 8 | 32 | count | int: Number of vkeys | + +The keys are sorted in binary order based on the `vkey` field. + +For each key: + +| ∆ | Bits | Name | Description | +|---|------|---------|------------------------------------------| +|12+| 32 | vkey | int: source vkey ID (0…255) | +|16+| 32 | target | int: target vkey ID (0…255) | + +- `vkey`: Is the standard vkey, 0-255 +- `target`: Is the target (resolved) vkey, 0-255. + +### C7043.2.7 Transforms and friends + +> TODO: transforms diff --git a/core/src/ldml/ldml_processor.cpp b/core/src/ldml/ldml_processor.cpp index 05f679fa37..7ed673e69f 100644 --- a/core/src/ldml/ldml_processor.cpp +++ b/core/src/ldml/ldml_processor.cpp @@ -9,6 +9,7 @@ #include "ldml/ldml_processor.hpp" #include "state.hpp" #include "../kmx/kmx_file.h" +#include "../kmx/kmx_plus.h" #include "ldml/keyboardprocessor_ldml.h" namespace { @@ -67,6 +68,9 @@ bool ldml_processor::is_kmxplus_file(path const & kb_path, std::vector& return false; } + // Dump data + dump_kmxplus_data(comp_keyboard); + // A KMXPlus file is in the buffer (although more validation is required) return true; } diff --git a/core/src/meson.build b/core/src/meson.build index 83ef77697e..0ab24eb7f1 100644 --- a/core/src/meson.build +++ b/core/src/meson.build @@ -83,6 +83,7 @@ lib = library('kmnkbp0', 'kmx/kmx_file.cpp', 'kmx/kmx_modifiers.cpp', 'kmx/kmx_options.cpp', + 'kmx/kmx_plus.cpp', 'kmx/kmx_processor.cpp', 'kmx/kmx_xstring.cpp', 'utfcodec.cpp', diff --git a/core/tests/unit/ldml/001_tiny.kmx b/core/tests/unit/ldml/001_tiny.kmx index 3c160a87064d3213a2fd7fda31e1e0825a3f6579..f4c093aefb5189a7aefc7b27ad5830a9f22497af 100644 GIT binary patch delta 38 qcmZo*Yhc?T#3)%(ynum$A+aPi*MSj8C+FuF0O{h&)XmzAR~Z4q*a|KH delta 38 qcmZo*Yhc?T#3;#7ynum$A+aPi*MSj8C+FuF0O{h?%FWu0R~Z4dED5>* diff --git a/core/tests/unit/ldml/meson.build b/core/tests/unit/ldml/meson.build index a2c0bfa2ad..f2167e32cc 100644 --- a/core/tests/unit/ldml/meson.build +++ b/core/tests/unit/ldml/meson.build @@ -166,3 +166,12 @@ else test(kbd, ldml, args: [kbd_src, kbd_obj]) endforeach endif + + +e = executable('test_kmx_plus', 'test_kmx_plus.cpp', + cpp_args: defns + warns, + include_directories: [inc, libsrc], + link_args: links + tests_flags, + objects: lib.extract_all_objects()) +test('test_kmx_plus', e) + diff --git a/core/tests/unit/ldml/test_kmx_plus.cpp b/core/tests/unit/ldml/test_kmx_plus.cpp new file mode 100644 index 0000000000..1d32184a57 --- /dev/null +++ b/core/tests/unit/ldml/test_kmx_plus.cpp @@ -0,0 +1,10 @@ +#include +#include +#include + +using namespace km::kbp::kmx; + +int main(int argc, const char *argv[]) { + printf("OKAY\n"); + return 0; +} From 590ac8f8f61fcec2488aae8eb4083d6ab91b5fa6 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Wed, 17 Aug 2022 16:25:41 -0500 Subject: [PATCH 2/8] feat(core): build out ldml structures - beginning of str handling --- core/src/kmx/kmx_plus.cpp | 39 ++++++++++++++++++++++++-- core/src/kmx/kmx_plus.h | 59 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 94 insertions(+), 4 deletions(-) diff --git a/core/src/kmx/kmx_plus.cpp b/core/src/kmx/kmx_plus.cpp index 28d58dd27c..6ad45af3c0 100644 --- a/core/src/kmx/kmx_plus.cpp +++ b/core/src/kmx/kmx_plus.cpp @@ -20,14 +20,29 @@ dump_kmxplus_header(const COMP_KMXPLUS_HEADER* hdr) { } static void -dump_kmxplus_sect(const uint8_t* /*data*/, const COMP_KMXPLUS_SECT* sect) { +dump_kmxplus_strs(const uint8_t* /*data*/, const COMP_KMXPLUS_STRS* strs) { + dump_kmxplus_header((const COMP_KMXPLUS_HEADER*)strs); + printf("strs: count 0x%X\n", strs->count); +} + +static void +dump_kmxplus_sect(const uint8_t* data, const COMP_KMXPLUS_SECT* sect) { dump_kmxplus_header((const COMP_KMXPLUS_HEADER*)sect); printf("sect: total 0x%X\n", sect->total); printf("sect: count 0x%X\n", sect->count); for (KMX_DWORD i = 0; i < sect->count; i++) { - dump_section_name(sect->entries[i].sect); - printf(" sect#%d: %X @ %X\n", i, sect->entries[i].sect, sect->entries[i].offset); + const COMP_KMXPLUS_SECT_ENTRY& entry = sect->entries[i]; + dump_section_name(entry.sect); + printf(" sect#%d: %X @ %X\n", i, entry.sect, entry.offset); + + switch(entry.sect) { + case LDML_SECTION_STRS: + dump_kmxplus_strs(data, as_kmxplus_strs((void*)(data+entry.offset))); + break; + default: + ; + } } } @@ -55,6 +70,24 @@ dump_kmxplus_data(kmx::PCOMP_KEYBOARD keyboard) { dump_kmxplus_data(rawdata + ex->kmxplus.dpKMXPlus); } + PKMX_WCHAR +COMP_KMXPLUS_STRS::get(KMX_DWORD entry, PKMX_WCHAR buf, KMX_DWORD bufsiz) { + assert(entry < count); + if (entry >= count) { + return NULL; + } + KMX_DWORD offset = entries[entry].offset; + KMX_DWORD length = entries[entry].length; + assert(bufsiz > (length+1)); // assert bufsiz big enough + assert(offset+((length+1)*2) <= header.size); // assert not out of bounds + const uint8_t* thisptr = reinterpret_cast(this); + const KMX_WCHAR* start = reinterpret_cast(thisptr+offset); + for(KMX_DWORD i=0;i<=length;i++) { + buf[i] = start[i]; + } + return buf; +} + } // namespace kmx } // namespace kbp } // namespace km diff --git a/core/src/kmx/kmx_plus.h b/core/src/kmx/kmx_plus.h index 7da4750e69..2f4636f43a 100644 --- a/core/src/kmx/kmx_plus.h +++ b/core/src/kmx/kmx_plus.h @@ -47,6 +47,16 @@ struct COMP_KMXPLUS_STRS { KMX_DWORD count; // 0008 count of str entries KMX_DWORD reserved; // 000C padding COMP_KMXPLUS_STRS_ENTRY entries[0]; // 0010+ entries + + /** + * @brief Get a string entry + * + * @param entry entry number + * @param buf output buffer + * @param bufsiz buffer size in bytes + * @return NULL or a pointer to the output buffer + */ + PKMX_WCHAR get(KMX_DWORD entry, PKMX_WCHAR buf, KMX_DWORD bufsiz); }; static_assert(sizeof(struct COMP_KMXPLUS_STRS) == LDML_LENGTH_STRS, "mismatched size of section strs"); @@ -137,7 +147,54 @@ as_kmxplus_sect(void *data) { return reinterpret_cast(all); } -// TODO-LDML: add as_kmxplus_keys, etc. +/** + * convert raw data to section + * @return section data or null on error + */ +static inline const COMP_KMXPLUS_STRS * +as_kmxplus_strs(void *data) { + const COMP_KMXPLUS_HEADER *all = validate_as_section(data, LDML_SECTION_STRS); + return reinterpret_cast(all); +} + +/** + * convert raw data to section + * @return section data or null on error + */ +static inline const COMP_KMXPLUS_KEYS * +as_kmxplus_keys(void *data) { + const COMP_KMXPLUS_HEADER *all = validate_as_section(data, LDML_SECTION_KEYS); + return reinterpret_cast(all); +} +/** + * convert raw data to section + * @return section data or null on error + */ +static inline const COMP_KMXPLUS_LOCA * +as_kmxplus_loca(void *data) { + const COMP_KMXPLUS_HEADER *all = validate_as_section(data, LDML_SECTION_LOCA); + return reinterpret_cast(all); +} +/** + * convert raw data to section + * @return section data or null on error + */ +static inline const COMP_KMXPLUS_META * +as_kmxplus_meta(void *data) { + const COMP_KMXPLUS_HEADER *all = validate_as_section(data, LDML_SECTION_META); + return reinterpret_cast(all); +} +/** + * convert raw data to section + * @return section data or null on error + */ +static inline const COMP_KMXPLUS_VKEY * +as_kmxplus_vkey(void *data) { + const COMP_KMXPLUS_HEADER *all = validate_as_section(data, LDML_SECTION_VKEY); + return reinterpret_cast(all); +} + + /** * @brief Temporary function to dump raw data From 5926dfe5fe9306050d0b2a67eeae04c8a35e1016 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Wed, 17 Aug 2022 17:43:14 -0500 Subject: [PATCH 3/8] feat(core): more kmx_plus_dump functions - also update sample data --- core/src/kmx/kmx_plus.cpp | 87 ++++++++++++++++++++++++++++-- core/src/kmx/kmx_plus.h | 18 +++---- core/tests/unit/ldml/001_tiny.kmx | Bin 768 -> 768 bytes 3 files changed, 90 insertions(+), 15 deletions(-) diff --git a/core/src/kmx/kmx_plus.cpp b/core/src/kmx/kmx_plus.cpp index 6ad45af3c0..31b19bc775 100644 --- a/core/src/kmx/kmx_plus.cpp +++ b/core/src/kmx/kmx_plus.cpp @@ -15,14 +15,76 @@ dump_section_name(KMX_DWORD ident) { static void dump_kmxplus_header(const COMP_KMXPLUS_HEADER* hdr) { + if (hdr == NULL) { + printf("! dump_kmxplus_header: NULL header\n"); + } dump_section_name(hdr->ident); printf(": (%X) size 0x%X\n", hdr->ident, hdr->size); } +static void +dump_kmxplus_keys(const uint8_t* /*data*/, const COMP_KMXPLUS_KEYS* keys) { + if(keys == NULL) { + printf("! could not load 'keys' section\n"); + return; + } + dump_kmxplus_header((const COMP_KMXPLUS_HEADER*)keys); + // TODO-LDML +} + +static void +dump_kmxplus_loca(const uint8_t* /*data*/, const COMP_KMXPLUS_LOCA* loca) { + if(loca == NULL) { + printf("! could not load 'loca' section\n"); + return; + } + dump_kmxplus_header((const COMP_KMXPLUS_HEADER*)loca); + // TODO-LDML +} + +static void +dump_kmxplus_meta(const uint8_t* /*data*/, const COMP_KMXPLUS_META* meta) { + if(meta == NULL) { + printf("! could not load 'meta' section\n"); + return; + } + dump_kmxplus_header((const COMP_KMXPLUS_HEADER*)meta); + // TODO-LDML +} + +static void +dump_kmxplus_vkey(const uint8_t* /*data*/, const COMP_KMXPLUS_VKEY* vkey) { + if(vkey == NULL) { + printf("! could not load 'vkey' section\n"); + return; + } + dump_kmxplus_header((const COMP_KMXPLUS_HEADER*)vkey); + // TODO-LDML +} + + + static void dump_kmxplus_strs(const uint8_t* /*data*/, const COMP_KMXPLUS_STRS* strs) { dump_kmxplus_header((const COMP_KMXPLUS_HEADER*)strs); printf("strs: count 0x%X\n", strs->count); + for (KMX_DWORD i=0; icount; i++) { + KMX_WCHAR buf[BUFSIZ]; + printf("#0x%X: ", i); + PKMX_WCHAR str = strs->get(i, buf, BUFSIZ); + if (!str) { + printf("NULL/ERR\n"); + continue; + } + for(int j=0; str[j] && j<0x30; j++) { + if (str[j] < 0x7F && str[j] != 0x0020 && str[j] > 0x20) { + putchar(str[j]); + } else { + printf("U+%04X ", str[j]); + } + } + printf("\n"); + } } static void @@ -35,20 +97,35 @@ dump_kmxplus_sect(const uint8_t* data, const COMP_KMXPLUS_SECT* sect) { const COMP_KMXPLUS_SECT_ENTRY& entry = sect->entries[i]; dump_section_name(entry.sect); printf(" sect#%d: %X @ %X\n", i, entry.sect, entry.offset); - + const uint8_t* entrydata = (data+entry.offset); switch(entry.sect) { + case LDML_SECTION_KEYS: + dump_kmxplus_keys(data, as_kmxplus_keys(entrydata)); + break; + case LDML_SECTION_LOCA: + dump_kmxplus_loca(data, as_kmxplus_loca(entrydata)); + break; + case LDML_SECTION_META: + dump_kmxplus_meta(data, as_kmxplus_meta(entrydata)); + break; + case LDML_SECTION_SECT: + printf("! Cowardly refusing to dump nested 'sect' section.\n"); + break; case LDML_SECTION_STRS: - dump_kmxplus_strs(data, as_kmxplus_strs((void*)(data+entry.offset))); + dump_kmxplus_strs(data, as_kmxplus_strs(entrydata)); + break; + case LDML_SECTION_VKEY: + dump_kmxplus_vkey(data, as_kmxplus_vkey(entrydata)); break; default: - ; + printf("Unknown section %X", entry.sect); } } } void dump_kmxplus_data(const uint8_t* data) { - const COMP_KMXPLUS_SECT* sect = as_kmxplus_sect((void*)data); + const COMP_KMXPLUS_SECT* sect = as_kmxplus_sect(data); if (sect == NULL) { printf("Err: 'sect' null from %p\n", data); return; @@ -71,7 +148,7 @@ dump_kmxplus_data(kmx::PCOMP_KEYBOARD keyboard) { } PKMX_WCHAR -COMP_KMXPLUS_STRS::get(KMX_DWORD entry, PKMX_WCHAR buf, KMX_DWORD bufsiz) { +COMP_KMXPLUS_STRS::get(KMX_DWORD entry, PKMX_WCHAR buf, KMX_DWORD bufsiz) const { assert(entry < count); if (entry >= count) { return NULL; diff --git a/core/src/kmx/kmx_plus.h b/core/src/kmx/kmx_plus.h index 2f4636f43a..8aff84b00f 100644 --- a/core/src/kmx/kmx_plus.h +++ b/core/src/kmx/kmx_plus.h @@ -56,7 +56,7 @@ struct COMP_KMXPLUS_STRS { * @param bufsiz buffer size in bytes * @return NULL or a pointer to the output buffer */ - PKMX_WCHAR get(KMX_DWORD entry, PKMX_WCHAR buf, KMX_DWORD bufsiz); + PKMX_WCHAR get(KMX_DWORD entry, PKMX_WCHAR buf, KMX_DWORD bufsiz) const; }; static_assert(sizeof(struct COMP_KMXPLUS_STRS) == LDML_LENGTH_STRS, "mismatched size of section strs"); @@ -123,7 +123,7 @@ static_assert(sizeof(struct COMP_KMXPLUS_VKEY) == LDML_LENGTH_VKEY, "mismatched * @return COMP_KMXPLUS_ALLDATA* or null */ static inline const COMP_KMXPLUS_HEADER * -validate_as_section(void *data, uint32_t ident) { +validate_as_section(const uint8_t *data, uint32_t ident) { if (!data) { return NULL; } @@ -142,7 +142,7 @@ validate_as_section(void *data, uint32_t ident) { * @return section data or null on error */ static inline const COMP_KMXPLUS_SECT * -as_kmxplus_sect(void *data) { +as_kmxplus_sect(const uint8_t *data) { const COMP_KMXPLUS_HEADER *all = validate_as_section(data, LDML_SECTION_SECT); return reinterpret_cast(all); } @@ -152,7 +152,7 @@ as_kmxplus_sect(void *data) { * @return section data or null on error */ static inline const COMP_KMXPLUS_STRS * -as_kmxplus_strs(void *data) { +as_kmxplus_strs(const uint8_t *data) { const COMP_KMXPLUS_HEADER *all = validate_as_section(data, LDML_SECTION_STRS); return reinterpret_cast(all); } @@ -162,7 +162,7 @@ as_kmxplus_strs(void *data) { * @return section data or null on error */ static inline const COMP_KMXPLUS_KEYS * -as_kmxplus_keys(void *data) { +as_kmxplus_keys(const uint8_t *data) { const COMP_KMXPLUS_HEADER *all = validate_as_section(data, LDML_SECTION_KEYS); return reinterpret_cast(all); } @@ -171,7 +171,7 @@ as_kmxplus_keys(void *data) { * @return section data or null on error */ static inline const COMP_KMXPLUS_LOCA * -as_kmxplus_loca(void *data) { +as_kmxplus_loca(const uint8_t *data) { const COMP_KMXPLUS_HEADER *all = validate_as_section(data, LDML_SECTION_LOCA); return reinterpret_cast(all); } @@ -180,7 +180,7 @@ as_kmxplus_loca(void *data) { * @return section data or null on error */ static inline const COMP_KMXPLUS_META * -as_kmxplus_meta(void *data) { +as_kmxplus_meta(const uint8_t *data) { const COMP_KMXPLUS_HEADER *all = validate_as_section(data, LDML_SECTION_META); return reinterpret_cast(all); } @@ -189,13 +189,11 @@ as_kmxplus_meta(void *data) { * @return section data or null on error */ static inline const COMP_KMXPLUS_VKEY * -as_kmxplus_vkey(void *data) { +as_kmxplus_vkey(const uint8_t *data) { const COMP_KMXPLUS_HEADER *all = validate_as_section(data, LDML_SECTION_VKEY); return reinterpret_cast(all); } - - /** * @brief Temporary function to dump raw data * diff --git a/core/tests/unit/ldml/001_tiny.kmx b/core/tests/unit/ldml/001_tiny.kmx index f4c093aefb5189a7aefc7b27ad5830a9f22497af..80671fde1937fe8a0f9a895d40c5e7aa8697a611 100644 GIT binary patch delta 114 zcmZo*Yhc@8#>mJx*_^R3h=TzF3V;+75H|oZI}lF*Vm2WD0K_~%yaCET0Hm3L_yUyw d0Lllc1L;1jw3fz?3t29#ayK2>>Vr4%Ywx delta 113 zcmZo*Yhc@8#>n`8vN>a65ElakBmgNUAT9u6b|7v5Vm2UVU}Rw60a6Qq7{uQI#LPf^ l0Ej{S3qTCwKLBEo{tpb3e=(Xc6% Date: Wed, 17 Aug 2022 18:13:17 -0500 Subject: [PATCH 4/8] feat(core): more kmx_plus_dump functions - also update sample data again --- core/include/ldml/keyboardprocessor_ldml.h | 2 +- core/include/ldml/keyboardprocessor_ldml.ts | 8 +++---- core/src/kmx/kmx_plus.cpp | 25 ++++++++++++++++++-- core/src/kmx/kmx_plus.h | 4 ++-- core/tests/unit/ldml/001_tiny.kmx | Bin 768 -> 768 bytes 5 files changed, 30 insertions(+), 9 deletions(-) diff --git a/core/include/ldml/keyboardprocessor_ldml.h b/core/include/ldml/keyboardprocessor_ldml.h index 996f03494a..0d56d7de47 100644 --- a/core/include/ldml/keyboardprocessor_ldml.h +++ b/core/include/ldml/keyboardprocessor_ldml.h @@ -6,6 +6,7 @@ // #pragma once +#define LDML_KEYS_FLAGS_EXTEND 1 #define LDML_LENGTH_HEADER 8 #define LDML_LENGTH_KEYS 16 #define LDML_LENGTH_LOCA 12 @@ -28,5 +29,4 @@ #define LDML_SECTION_STRS ((uint32_t)'strs') // Section ID #define LDML_SECTION_VKEY ((uint32_t)'vkey') -#define LDML_STRS_FLAGS_EXTEND 1 #define LDML_VERSION "1.0" diff --git a/core/include/ldml/keyboardprocessor_ldml.ts b/core/include/ldml/keyboardprocessor_ldml.ts index ba15c74a17..afa7e9511c 100644 --- a/core/include/ldml/keyboardprocessor_ldml.ts +++ b/core/include/ldml/keyboardprocessor_ldml.ts @@ -66,13 +66,13 @@ export const constants = { length_strs: 16, /** - * bitwise or value for extend in strs[key].flags. + * bitwise or value for extend in keys[key].flags. * If bit is 1, then 'to' is a string. - * If bit is 0, then 'to' is an offset. + * If bit is 0, then 'to' is a UTF-32LE codepoint * - * `extend = flags & strs_flags_extend` + * `extend = flags & keys_flags_extend` */ - strs_flags_extend: 1, + keys_flags_extend: 1, section_vkey: 'vkey', length_vkey: 12, diff --git a/core/src/kmx/kmx_plus.cpp b/core/src/kmx/kmx_plus.cpp index 31b19bc775..d6600d25d8 100644 --- a/core/src/kmx/kmx_plus.cpp +++ b/core/src/kmx/kmx_plus.cpp @@ -29,7 +29,19 @@ dump_kmxplus_keys(const uint8_t* /*data*/, const COMP_KMXPLUS_KEYS* keys) { return; } dump_kmxplus_header((const COMP_KMXPLUS_HEADER*)keys); - // TODO-LDML + printf(" count: #0x%X\n", keys->count); + for (KMX_DWORD i = 0; icount; i++) { + printf(" #0x%d\n", i); + const COMP_KMXPLUS_KEYS_ENTRY& entry = keys->entries[i]; + printf(" vkey\t0x%X\n", entry.vkey); + printf(" mod\t0x%X\n", entry.mod); + printf(" flags\t0x%X\n", entry.flags); + if (entry.flags & LDML_KEYS_FLAGS_EXTEND) { + printf(" \t Extend: String #0x%X\n", entry.to); + } else { + printf(" \t UTF-32:U+%04X\n", entry.to); + } + } } static void @@ -40,6 +52,9 @@ dump_kmxplus_loca(const uint8_t* /*data*/, const COMP_KMXPLUS_LOCA* loca) { } dump_kmxplus_header((const COMP_KMXPLUS_HEADER*)loca); // TODO-LDML + for(KMX_DWORD i=0; icount; i++) { + printf(" Locale #%d: #0x%X\n", i, loca->entries[i].locale); + } } static void @@ -49,7 +64,13 @@ dump_kmxplus_meta(const uint8_t* /*data*/, const COMP_KMXPLUS_META* meta) { return; } dump_kmxplus_header((const COMP_KMXPLUS_HEADER*)meta); - // TODO-LDML + printf(" name:\t#0x%X\n", meta->name); + printf(" author:\t#0x%X\n", meta->author); + printf(" conform:\t#0x%X\n", meta->conform); + printf(" layout:\t#0x%X\n", meta->layout); + printf(" normalization:\t#0x%X\n", meta->normalization); + printf(" indicator:\t#0x%X\n", meta->indicator); + printf(" settings:\t0x%X\n", meta->settings); } static void diff --git a/core/src/kmx/kmx_plus.h b/core/src/kmx/kmx_plus.h index 8aff84b00f..4dd76c4d4e 100644 --- a/core/src/kmx/kmx_plus.h +++ b/core/src/kmx/kmx_plus.h @@ -129,8 +129,8 @@ validate_as_section(const uint8_t *data, uint32_t ident) { } const COMP_KMXPLUS_HEADER *all = reinterpret_cast(data); // TODO-LDML these fail on 000null .. - // assert(all->size >= LDML_LENGTH_HEADER); - // assert(ident == all->ident); + assert(all->size >= LDML_LENGTH_HEADER); + assert(ident == all->ident); if (ident != all->ident || (all->size < LDML_LENGTH_HEADER)) { return NULL; // invalid header or wrong section } diff --git a/core/tests/unit/ldml/001_tiny.kmx b/core/tests/unit/ldml/001_tiny.kmx index 80671fde1937fe8a0f9a895d40c5e7aa8697a611..b25c094fedfdf2764b7fff759a757467e083ca6b 100644 GIT binary patch delta 79 zcmZo*Yhc?T%P7dez>t`npJM=|iz`#J3m7KrF*@r2Ss)Q6AO?Xz2*sez2w{Xn`RqWN Kar0cpWJUlgCJP_{ delta 79 zcmZo*Yhc?T%P7dmz>t`npJM=|iz`#J3m7NsF*@s{R%RP8FfcFyF$e@gC@`rGR2dHC O1DOo$o98knGXelA7Yic* From a333eade22041728ded1c3379abba452bf2fa9d6 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 18 Aug 2022 14:27:17 -0500 Subject: [PATCH 5/8] feat(core): ldml: tiny passes tests - removed null from build, invalid --- core/src/kmx/kmx_plus.cpp | 23 ++++++- core/src/kmx/kmx_plus.h | 8 +++ core/src/ldml/ldml_processor.cpp | 111 +++++++++++++++++++++++++++++-- core/src/ldml/ldml_processor.hpp | 4 +- core/tests/unit/ldml/meson.build | 2 +- 5 files changed, 135 insertions(+), 13 deletions(-) diff --git a/core/src/kmx/kmx_plus.cpp b/core/src/kmx/kmx_plus.cpp index d6600d25d8..a349928628 100644 --- a/core/src/kmx/kmx_plus.cpp +++ b/core/src/kmx/kmx_plus.cpp @@ -83,8 +83,6 @@ dump_kmxplus_vkey(const uint8_t* /*data*/, const COMP_KMXPLUS_VKEY* vkey) { // TODO-LDML } - - static void dump_kmxplus_strs(const uint8_t* /*data*/, const COMP_KMXPLUS_STRS* strs) { dump_kmxplus_header((const COMP_KMXPLUS_HEADER*)strs); @@ -108,6 +106,15 @@ dump_kmxplus_strs(const uint8_t* /*data*/, const COMP_KMXPLUS_STRS* strs) { } } +KMX_DWORD COMP_KMXPLUS_SECT::find(KMX_DWORD ident) const { + for (KMX_DWORD i = 0; i < count; i++) { + if (ident == entries[i].sect) { + return entries[i].offset; + } + } + return 0; +} + static void dump_kmxplus_sect(const uint8_t* data, const COMP_KMXPLUS_SECT* sect) { dump_kmxplus_header((const COMP_KMXPLUS_HEADER*)sect); @@ -168,7 +175,17 @@ dump_kmxplus_data(kmx::PCOMP_KEYBOARD keyboard) { dump_kmxplus_data(rawdata + ex->kmxplus.dpKMXPlus); } - PKMX_WCHAR +const COMP_KMXPLUS_KEYS_ENTRY *COMP_KMXPLUS_KEYS::find(KMX_DWORD vkey, KMX_DWORD mod) const { + // TODO-LDML: eventually, assume sorted order & binary search + for (KMX_DWORD i=0; i= count) { diff --git a/core/src/kmx/kmx_plus.h b/core/src/kmx/kmx_plus.h index 4dd76c4d4e..5d125fd90d 100644 --- a/core/src/kmx/kmx_plus.h +++ b/core/src/kmx/kmx_plus.h @@ -32,6 +32,13 @@ struct COMP_KMXPLUS_SECT { KMX_DWORD total; // 0008 KMXPlus entire length KMX_DWORD count; // 000B number of section headers COMP_KMXPLUS_SECT_ENTRY entries[0]; // 0010 section entries + /** + * @brief Get the offset of a section, or 0 + * + * @param ident section id such as 'strs'. Never 'sect' + * @return KMX_DWORD offset from beginning of kmxplus + */ + KMX_DWORD find(KMX_DWORD ident) const; }; // Assert that the length matches the declared length @@ -98,6 +105,7 @@ struct COMP_KMXPLUS_KEYS { KMX_DWORD count; // number of keys KMX_DWORD reserved; // padding COMP_KMXPLUS_KEYS_ENTRY entries[0]; + const COMP_KMXPLUS_KEYS_ENTRY *find(KMX_DWORD vkey, KMX_DWORD mod) const; }; static_assert(sizeof(struct COMP_KMXPLUS_KEYS) == LDML_LENGTH_KEYS, "mismatched size of section keys"); diff --git a/core/src/ldml/ldml_processor.cpp b/core/src/ldml/ldml_processor.cpp index 7ed673e69f..9d9fc83db3 100644 --- a/core/src/ldml/ldml_processor.cpp +++ b/core/src/ldml/ldml_processor.cpp @@ -12,6 +12,38 @@ #include "../kmx/kmx_plus.h" #include "ldml/keyboardprocessor_ldml.h" +// extern "C" { +// #include "../../../common/windows/cpp/src/ConvertUTF.c" +// } + +// HACK +enum ConversionResult { + conversionOK, + conversionNotOk +}; +enum ConversionFlags { + strictConversion +}; + +typedef KMX_WCHAR UTF16; +typedef KMX_DWORD UTF32; + +ConversionResult ConvertUTF32toUTF16( + UTF32** sourceStart, const UTF32* sourceEnd, + UTF16** targetStart, const UTF16* targetEnd, const ConversionFlags /*flags*/) { + if(sourceEnd > (*sourceStart+1)) return conversionNotOk; // Don't support >1 char + if(**sourceStart & 0xFFFF0000) { + // Don't support supplemental chars yet + return conversionNotOk; + } + *((*targetStart)++) = (UTF16)((*(*sourceStart)++) & 0xFFFF); // BMP + + assert(*sourceStart==sourceEnd); + assert(*targetStart <= targetEnd); + return(conversionOK); +} + + namespace { constexpr km_kbp_attr const engine_attrs = { 256, @@ -26,13 +58,22 @@ namespace { namespace km { namespace kbp { -ldml_processor::ldml_processor(path const & kb_path, const std::vector _kmn_unused(data)) +ldml_processor::ldml_processor(path const & kb_path, const std::vector data) : abstract_processor( keyboard_attributes(kb_path.stem(), KM_KBP_LMDL_PROCESSOR_VERSION, kb_path.parent(), {}) - ) + ), rawdata(data) // TODO-LDML: load instead of clone { // TODO-LDML: load the file from the buffer (KMXPlus format) // Note: kb_path is essentially debug metadata here + + if (data.size() == 0) { + std::ifstream file(static_cast(kb_path), std::ios::binary | std::ios::ate); + const std::streamsize size = file.tellg(); + file.seekg(0, std::ios::beg); + rawdata.reserve((size_t)size); + file.read((char *) rawdata.data(), size); + file.close(); + } } bool ldml_processor::is_kmxplus_file(path const & kb_path, std::vector& data) { @@ -68,7 +109,7 @@ bool ldml_processor::is_kmxplus_file(path const & kb_path, std::vector& return false; } - // Dump data + // Dump data. This also does some validation. dump_kmxplus_data(comp_keyboard); // A KMXPlus file is in the buffer (although more validation is required) @@ -102,7 +143,7 @@ km_kbp_status ldml_processor::process_event( km_kbp_state *state, km_kbp_virtual_key vk, - uint16_t _kmn_unused(modifier_state), + uint16_t modifier_state, uint8_t is_key_down, uint16_t /*event_flags*/ // TODO-LDML: unused... for now... ) { @@ -127,9 +168,65 @@ ldml_processor::process_event( state->actions().push_backspace(KM_KBP_BT_UNKNOWN); // Assuming we don't know the character break; default: - /* We're going to push an 'a' for a passing unit test */ - state->context().push_character('a'); - state->actions().push_character('a'); + // TODO-LDML: temporary code here + // Don't want to do this work each time + const kmx::PCOMP_KEYBOARD comp_keyboard = (kmx::PCOMP_KEYBOARD)rawdata.data(); + assert(comp_keyboard->dwFlags & KF_KMXPLUS); + const kmx::COMP_KEYBOARD_EX* ex = reinterpret_cast(comp_keyboard); + + // printf("KMXPlus offset 0x%X, KMXPlus size 0x%X\n", ex->kmxplus.dpKMXPlus, ex->kmxplus.dwKMXPlusSize); + const uint8_t* kmxplusdata = rawdata.data() + ex->kmxplus.dpKMXPlus; + // Get out the SECT header + const kmx::COMP_KMXPLUS_SECT *sect = kmx::as_kmxplus_sect(kmxplusdata); + assert(sect != NULL); + assert(sect->header.ident == LDML_SECTION_SECT); + KMX_DWORD offset; + // Fill out the other sections we need. + offset = sect->find(LDML_SECTION_STRS); + assert(offset != 0); // or else section not found + const kmx::COMP_KMXPLUS_STRS *strs = kmx::as_kmxplus_strs(kmxplusdata+offset); + assert(strs->header.ident == LDML_SECTION_STRS); + offset = sect->find(LDML_SECTION_KEYS); + assert(offset != 0); // or else section not found + const kmx::COMP_KMXPLUS_KEYS *keys = kmx::as_kmxplus_keys(kmxplusdata+offset); + assert(keys->header.ident == LDML_SECTION_KEYS); + // Look up the key + const kmx::COMP_KMXPLUS_KEYS_ENTRY *key = keys->find(vk, modifier_state); + assert(key != NULL); + if (!key) { + return KM_KBP_STATUS_KEY_ERROR; + } + // Prepare output chars + KMX_DWORD len = 0; + KMX_WCHAR out[BUFSIZ]; + if (key->flags && LDML_KEYS_FLAGS_EXTEND) { + // It's a string. + assert(NULL != strs->get(key->to, out, BUFSIZ)); + // u_strlen() + for(len=0; lento; // UTF-32 + buf32[1] = 0; // to avoid UMR warning + UTF32 *sourceStart = &buf32[0]; + const UTF32 *sourceEnd = &buf32[1]; // Reference off the end. NULL to avoid UMR. + UTF16 *targetStart = (UTF16*)out; + const UTF16 *targetEnd = (UTF16*)out+BUFSIZ-1; + ConversionResult result = ::ConvertUTF32toUTF16(&sourceStart, sourceEnd, &targetStart, targetEnd, strictConversion); + assert(result == conversionOK); + *targetStart = 0; + len = 1; // TODO=LDML calculate + // len = (targetStart - out); + assert(len>=1 && len <= 2); + } + assert(len>=0); + assert(lencontext().push_character(out[i]); + state->actions().push_character(out[i]); + } } state->actions().commit(); diff --git a/core/src/ldml/ldml_processor.hpp b/core/src/ldml/ldml_processor.hpp index a5490ff07c..28d055ecf9 100644 --- a/core/src/ldml/ldml_processor.hpp +++ b/core/src/ldml/ldml_processor.hpp @@ -11,7 +11,6 @@ #include #include #include - #include "processor.hpp" #include "option.hpp" @@ -21,7 +20,8 @@ namespace kbp { #define KM_KBP_LMDL_PROCESSOR_VERSION u"1.0" class ldml_processor : public abstract_processor { - + private: + std::vector rawdata; // TODO-LDML: should be 'unpacked' format instead. public: ldml_processor( path const & kb_path, diff --git a/core/tests/unit/ldml/meson.build b/core/tests/unit/ldml/meson.build index f2167e32cc..53fd7bf014 100644 --- a/core/tests/unit/ldml/meson.build +++ b/core/tests/unit/ldml/meson.build @@ -33,7 +33,7 @@ ldml = executable('ldml', objects: lib.extract_all_objects()) tests = [ - '000_null_keyboard', + # '000_null_keyboard', '001_tiny' ] From 747dfe8c0dad061fb871e2ead9d9cef86c880852 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 18 Aug 2022 14:33:53 -0500 Subject: [PATCH 6/8] feat(core): ldml: header cleanup --- core/include/ldml/keyboardprocessor_ldml.ts | 26 ++++++++++++++++--- .../ldml/keyboardproessor_ldml_sects.h | 1 - 2 files changed, 22 insertions(+), 5 deletions(-) delete mode 100644 core/include/ldml/keyboardproessor_ldml_sects.h diff --git a/core/include/ldml/keyboardprocessor_ldml.ts b/core/include/ldml/keyboardprocessor_ldml.ts index afa7e9511c..9010e3f8f9 100644 --- a/core/include/ldml/keyboardprocessor_ldml.ts +++ b/core/include/ldml/keyboardprocessor_ldml.ts @@ -19,7 +19,6 @@ export const constants = { * The version of the LDML processor */ version: '1.0', - /** * Length of a raw section header, in bytes */ @@ -28,16 +27,27 @@ export const constants = { * Section ID for the keybag */ section_keys: 'keys', + /** + * Minimum length of the 'keys' section + * not including variable parts + */ length_keys: 16, /** * Section ID for the locale list */ section_loca: 'loca', + /** + * Minimum length of the 'loca' section + * not including variable parts + */ length_loca: 12, /** * Section ID for the metadata */ section_meta: 'meta', + /** + * length of the 'meta' section + */ length_meta: 36, /** * bitwise or value for fallback=omit in meta.settings @@ -63,7 +73,10 @@ export const constants = { * Section ID for the string table */ section_strs: 'strs', - + /** + * Minimum length of the 'strs' section + * not including variable parts + */ length_strs: 16, /** * bitwise or value for extend in keys[key].flags. @@ -73,9 +86,14 @@ export const constants = { * `extend = flags & keys_flags_extend` */ keys_flags_extend: 1, - + /** + * Section ID for the vkeys map + */ section_vkey: 'vkey', + /** + * Minimum length of the 'vkey' section + * not including variable parts + */ length_vkey: 12, - }; // } diff --git a/core/include/ldml/keyboardproessor_ldml_sects.h b/core/include/ldml/keyboardproessor_ldml_sects.h deleted file mode 100644 index aa78802ff9..0000000000 --- a/core/include/ldml/keyboardproessor_ldml_sects.h +++ /dev/null @@ -1 +0,0 @@ -keyboardproessor_ldml_structs.h From a4080d2b7ab5ef38a643006a47c75b7a1bb1c818 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 19 Aug 2022 17:34:18 -0500 Subject: [PATCH 7/8] feat(core): ldml: cleanup of header files - quell debug messages unless KMXPLUS_DEBUG=1 - don't read the file twice #5015 --- core/src/kmx/kmx_plus.cpp | 34 +++++++++++++++++++++++--------- core/src/kmx/kmx_plus.h | 17 ++++++++-------- core/src/ldml/ldml_processor.cpp | 16 ++++----------- core/src/ldml/ldml_processor.hpp | 2 +- 4 files changed, 39 insertions(+), 30 deletions(-) diff --git a/core/src/kmx/kmx_plus.cpp b/core/src/kmx/kmx_plus.cpp index a349928628..b860a1fb37 100644 --- a/core/src/kmx/kmx_plus.cpp +++ b/core/src/kmx/kmx_plus.cpp @@ -1,10 +1,21 @@ #include + +/** + * @def KMXPLUS_DEBUG Set to 1 to enable debug output + */ +#define KMXPLUS_DEBUG 1 + +#if KMXPLUS_DEBUG #include +#endif + +#include namespace km { namespace kbp { namespace kmx { +#if KMXPLUS_DEBUG static void dump_section_name(KMX_DWORD ident) { for (int i = 0; i < 4; i++) { @@ -106,15 +117,6 @@ dump_kmxplus_strs(const uint8_t* /*data*/, const COMP_KMXPLUS_STRS* strs) { } } -KMX_DWORD COMP_KMXPLUS_SECT::find(KMX_DWORD ident) const { - for (KMX_DWORD i = 0; i < count; i++) { - if (ident == entries[i].sect) { - return entries[i].offset; - } - } - return 0; -} - static void dump_kmxplus_sect(const uint8_t* data, const COMP_KMXPLUS_SECT* sect) { dump_kmxplus_header((const COMP_KMXPLUS_HEADER*)sect); @@ -150,19 +152,23 @@ dump_kmxplus_sect(const uint8_t* data, const COMP_KMXPLUS_SECT* sect) { } } } +#endif void dump_kmxplus_data(const uint8_t* data) { +#if KMXPLUS_DEBUG const COMP_KMXPLUS_SECT* sect = as_kmxplus_sect(data); if (sect == NULL) { printf("Err: 'sect' null from %p\n", data); return; } dump_kmxplus_sect(data, sect); +#endif } void dump_kmxplus_data(kmx::PCOMP_KEYBOARD keyboard) { +#if KMXPLUS_DEBUG printf("dump_kmxplus_data(): Got a PCOMP_KEYBOARD at %p\n", keyboard); if (!(keyboard->dwFlags & KF_KMXPLUS)) { printf("Err: flags KF_KMXPLUS not set\n"); @@ -173,6 +179,7 @@ dump_kmxplus_data(kmx::PCOMP_KEYBOARD keyboard) { printf("KMXPlus offset 0x%X, KMXPlus size 0x%X\n", ex->kmxplus.dpKMXPlus, ex->kmxplus.dwKMXPlusSize); const uint8_t* rawdata = reinterpret_cast(keyboard); dump_kmxplus_data(rawdata + ex->kmxplus.dpKMXPlus); +#endif } const COMP_KMXPLUS_KEYS_ENTRY *COMP_KMXPLUS_KEYS::find(KMX_DWORD vkey, KMX_DWORD mod) const { @@ -185,6 +192,15 @@ const COMP_KMXPLUS_KEYS_ENTRY *COMP_KMXPLUS_KEYS::find(KMX_DWORD vkey, KMX_DWORD return NULL; } +KMX_DWORD COMP_KMXPLUS_SECT::find(KMX_DWORD ident) const { + for (KMX_DWORD i = 0; i < count; i++) { + if (ident == entries[i].sect) { + return entries[i].offset; + } + } + return 0; +} + PKMX_WCHAR COMP_KMXPLUS_STRS::get(KMX_DWORD entry, PKMX_WCHAR buf, KMX_DWORD bufsiz) const { assert(entry < count); diff --git a/core/src/kmx/kmx_plus.h b/core/src/kmx/kmx_plus.h index 5d125fd90d..89c3421d7f 100644 --- a/core/src/kmx/kmx_plus.h +++ b/core/src/kmx/kmx_plus.h @@ -146,7 +146,7 @@ validate_as_section(const uint8_t *data, uint32_t ident) { } /** - * convert raw data to section + * cast raw data to section * @return section data or null on error */ static inline const COMP_KMXPLUS_SECT * @@ -156,7 +156,7 @@ as_kmxplus_sect(const uint8_t *data) { } /** - * convert raw data to section + * cast raw data to section * @return section data or null on error */ static inline const COMP_KMXPLUS_STRS * @@ -166,7 +166,7 @@ as_kmxplus_strs(const uint8_t *data) { } /** - * convert raw data to section + * cast raw data to section * @return section data or null on error */ static inline const COMP_KMXPLUS_KEYS * @@ -175,7 +175,7 @@ as_kmxplus_keys(const uint8_t *data) { return reinterpret_cast(all); } /** - * convert raw data to section + * cast raw data to section * @return section data or null on error */ static inline const COMP_KMXPLUS_LOCA * @@ -184,7 +184,7 @@ as_kmxplus_loca(const uint8_t *data) { return reinterpret_cast(all); } /** - * convert raw data to section + * cast raw data to section * @return section data or null on error */ static inline const COMP_KMXPLUS_META * @@ -204,14 +204,15 @@ as_kmxplus_vkey(const uint8_t *data) { /** * @brief Temporary function to dump raw data - * + * May have no output if KMXPLUS_DEBUG is 0 + * @param kmxplusdata data from the beginning of the KMXPlus section */ void dump_kmxplus_data(void *kmxplusdata); /** * @brief Temporary functino to dump raw data - * - * @param keyboard + * May have no output if KMXPLUS_DEBUG is 0 + * @param keyboard pointer to PCOMP_KEYBOARD with plus data following */ void dump_kmxplus_data(kmx::PCOMP_KEYBOARD keyboard); diff --git a/core/src/ldml/ldml_processor.cpp b/core/src/ldml/ldml_processor.cpp index 9d9fc83db3..d20caf44a3 100644 --- a/core/src/ldml/ldml_processor.cpp +++ b/core/src/ldml/ldml_processor.cpp @@ -58,22 +58,14 @@ namespace { namespace km { namespace kbp { -ldml_processor::ldml_processor(path const & kb_path, const std::vector data) +ldml_processor::ldml_processor(path const & kb_path, const std::vector &data) : abstract_processor( keyboard_attributes(kb_path.stem(), KM_KBP_LMDL_PROCESSOR_VERSION, kb_path.parent(), {}) - ), rawdata(data) // TODO-LDML: load instead of clone + ), rawdata(data) // TODO-LDML: parse the data, don't just copy it { // TODO-LDML: load the file from the buffer (KMXPlus format) // Note: kb_path is essentially debug metadata here - - if (data.size() == 0) { - std::ifstream file(static_cast(kb_path), std::ios::binary | std::ios::ate); - const std::streamsize size = file.tellg(); - file.seekg(0, std::ios::beg); - rawdata.reserve((size_t)size); - file.read((char *) rawdata.data(), size); - file.close(); - } + assert(data.size() != 0); } bool ldml_processor::is_kmxplus_file(path const & kb_path, std::vector& data) { @@ -92,7 +84,7 @@ bool ldml_processor::is_kmxplus_file(path const & kb_path, std::vector& file.seekg(0, std::ios::beg); - data.reserve((size_t)size); + data.resize((size_t)size); if(!file.read((char *) data.data(), size)) { return false; } diff --git a/core/src/ldml/ldml_processor.hpp b/core/src/ldml/ldml_processor.hpp index 28d055ecf9..6e7851c0ca 100644 --- a/core/src/ldml/ldml_processor.hpp +++ b/core/src/ldml/ldml_processor.hpp @@ -25,7 +25,7 @@ namespace kbp { public: ldml_processor( path const & kb_path, - const std::vector data + const std::vector & data ); // ~ldml_processor() override; From f5db1749b4b116dae159d52916ccbc1aa5664f3f Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 22 Aug 2022 17:13:45 -0500 Subject: [PATCH 8/8] style(core): ldml: review comments #5015 - add copyright notices --- core/include/ldml/keyboardprocessor_ldml.h | 8 ++++++++ core/include/ldml/keyboardprocessor_ldml.ts | 7 +++++++ core/include/ldml/ldml-const-builder.ts | 16 +++++++++++++++- core/src/kmx/kmx_plus.cpp | 6 ++++++ 4 files changed, 36 insertions(+), 1 deletion(-) diff --git a/core/include/ldml/keyboardprocessor_ldml.h b/core/include/ldml/keyboardprocessor_ldml.h index 0d56d7de47..a6e623debb 100644 --- a/core/include/ldml/keyboardprocessor_ldml.h +++ b/core/include/ldml/keyboardprocessor_ldml.h @@ -1,3 +1,11 @@ + +/* + Copyright: Copyright (C) 2022 SIL International. + Authors: srl295 + This file provides constants for the KMX Plus (LDML support) binary format, + to be shared between TypeScript and C++ via the generator (below) +*/ + // // Generated File - do not edit // diff --git a/core/include/ldml/keyboardprocessor_ldml.ts b/core/include/ldml/keyboardprocessor_ldml.ts index 9010e3f8f9..e4a5a78346 100644 --- a/core/include/ldml/keyboardprocessor_ldml.ts +++ b/core/include/ldml/keyboardprocessor_ldml.ts @@ -1,3 +1,10 @@ +/* + Copyright: Copyright (C) 2022 SIL International. + Authors: srl295 + This file provides constants for the KMX Plus (LDML support) binary format, + to be shared between TypeScript and C++ via the generator (below) +*/ + // Notice! // diff --git a/core/include/ldml/ldml-const-builder.ts b/core/include/ldml/ldml-const-builder.ts index 731e9e40b9..d8a398b72e 100644 --- a/core/include/ldml/ldml-const-builder.ts +++ b/core/include/ldml/ldml-const-builder.ts @@ -1,8 +1,22 @@ +/* + Copyright: Copyright (C) 2022 SIL International. + Authors: srl295 + This tool generates a .h version of the keyboardprocessor_ldml.ts file +*/ + import { constants } from './keyboardprocessor_ldml'; const keys = Object.keys(constants); keys.sort(); -console.log(`// +console.log(` +/* + Copyright: Copyright (C) 2022 SIL International. + Authors: srl295 + This file provides constants for the KMX Plus (LDML support) binary format, + to be shared between TypeScript and C++ via the generator (below) +*/ + +// // Generated File - do not edit // // This file is generated by core/tools/ldml-const-builder/build.sh diff --git a/core/src/kmx/kmx_plus.cpp b/core/src/kmx/kmx_plus.cpp index b860a1fb37..7949cb030f 100644 --- a/core/src/kmx/kmx_plus.cpp +++ b/core/src/kmx/kmx_plus.cpp @@ -1,3 +1,9 @@ +/* + Copyright: Copyright (C) 2022 SIL International. + Authors: srl295 + Implementation for the KMX Plus utilities +*/ + #include /**