diff --git a/core/include/ldml/keyboardprocessor_ldml.h b/core/include/ldml/keyboardprocessor_ldml.h index fe10cf1f39..a6e623debb 100644 --- a/core/include/ldml/keyboardprocessor_ldml.h +++ b/core/include/ldml/keyboardprocessor_ldml.h @@ -1,3 +1,11 @@ + +/* + Copyright: Copyright (C) 2022 SIL International. + Authors: srl295 + This file provides constants for the KMX Plus (LDML support) binary format, + to be shared between TypeScript and C++ via the generator (below) +*/ + // // Generated File - do not edit // @@ -6,6 +14,14 @@ // #pragma once +#define LDML_KEYS_FLAGS_EXTEND 1 +#define LDML_LENGTH_HEADER 8 +#define LDML_LENGTH_KEYS 16 +#define LDML_LENGTH_LOCA 12 +#define LDML_LENGTH_META 36 +#define LDML_LENGTH_SECT 16 +#define LDML_LENGTH_STRS 16 +#define LDML_LENGTH_VKEY 12 #define LDML_META_SETTINGS_FALLBACK_OMIT 1 #define LDML_META_SETTINGS_TRANSFORMFAILURE_OMIT 2 #define LDML_META_SETTINGS_TRANSFORMPARTIAL_HIDE 4 @@ -19,5 +35,6 @@ #define LDML_SECTION_SECT ((uint32_t)'sect') // Section ID #define LDML_SECTION_STRS ((uint32_t)'strs') -#define LDML_STRS_FLAGS_EXTEND 1 +// Section ID +#define LDML_SECTION_VKEY ((uint32_t)'vkey') #define LDML_VERSION "1.0" diff --git a/core/include/ldml/keyboardprocessor_ldml.ts b/core/include/ldml/keyboardprocessor_ldml.ts index bf1eb2c9b5..e4a5a78346 100644 --- a/core/include/ldml/keyboardprocessor_ldml.ts +++ b/core/include/ldml/keyboardprocessor_ldml.ts @@ -1,3 +1,10 @@ +/* + Copyright: Copyright (C) 2022 SIL International. + Authors: srl295 + This file provides constants for the KMX Plus (LDML support) binary format, + to be shared between TypeScript and C++ via the generator (below) +*/ + // Notice! // @@ -10,57 +17,90 @@ // It is not updated automatically. // TODO-LDML: namespace com.keyman.core.ldml { +/** + * Constants for the KMXPlus data format + * These are shared between the data access layer and the compiler + */ +export const constants = { /** - * Constants for the KMXPlus data format - * These are shared between the data access layer and the compiler + * The version of the LDML processor */ - export const constants = { - /** - * The version of the LDML processor - */ - version: '1.0', - - /** - * Section ID for the keybag - */ - section_keys: 'keys', - /** - * Section ID for the locale list - */ - section_loca: 'loca', - /** - * Section ID for the metadata - */ - section_meta: 'meta', - /** - * Section ID for the section header - */ - section_sect: 'sect', - /** - * Section ID for the string table - */ - section_strs: 'strs', - - /** - * bitwise or value for fallback=omit in meta.settings - */ - meta_settings_fallback_omit: 1, - /** - * bitwise or value for transformFailure=omit in meta.settings - */ - meta_settings_transformFailure_omit: 2, - /** - * bitwise or value for transformPartial=hide in meta.settings - */ - meta_settings_transformPartial_hide: 4, - - /** - * bitwise or value for extend in strs[key].flags. - * If bit is 1, then 'to' is a string. - * If bit is 0, then 'to' is an offset. - * - * `extend = flags & strs_flags_extend` - */ - strs_flags_extend: 1, - }; + version: '1.0', + /** + * Length of a raw section header, in bytes + */ + length_header: 8, + /** + * Section ID for the keybag + */ + section_keys: 'keys', + /** + * Minimum length of the 'keys' section + * not including variable parts + */ + length_keys: 16, + /** + * Section ID for the locale list + */ + section_loca: 'loca', + /** + * Minimum length of the 'loca' section + * not including variable parts + */ + length_loca: 12, + /** + * Section ID for the metadata + */ + section_meta: 'meta', + /** + * length of the 'meta' section + */ + length_meta: 36, + /** + * bitwise or value for fallback=omit in meta.settings + */ + meta_settings_fallback_omit: 1, + /** + * bitwise or value for transformFailure=omit in meta.settings + */ + meta_settings_transformFailure_omit: 2, + /** + * bitwise or value for transformPartial=hide in meta.settings + */ + meta_settings_transformPartial_hide: 4, + /** + * Section ID for the section header + */ + section_sect: 'sect', + /** + * Minimum length of the 'sect' section, not including entries + */ + length_sect: 16, + /** + * Section ID for the string table + */ + section_strs: 'strs', + /** + * Minimum length of the 'strs' section + * not including variable parts + */ + length_strs: 16, + /** + * bitwise or value for extend in keys[key].flags. + * If bit is 1, then 'to' is a string. + * If bit is 0, then 'to' is a UTF-32LE codepoint + * + * `extend = flags & keys_flags_extend` + */ + keys_flags_extend: 1, + /** + * Section ID for the vkeys map + */ + section_vkey: 'vkey', + /** + * Minimum length of the 'vkey' section + * not including variable parts + */ + length_vkey: 12, +}; // } diff --git a/core/include/ldml/ldml-const-builder.ts b/core/include/ldml/ldml-const-builder.ts index 731e9e40b9..d8a398b72e 100644 --- a/core/include/ldml/ldml-const-builder.ts +++ b/core/include/ldml/ldml-const-builder.ts @@ -1,8 +1,22 @@ +/* + Copyright: Copyright (C) 2022 SIL International. + Authors: srl295 + This tool generates a .h version of the keyboardprocessor_ldml.ts file +*/ + import { constants } from './keyboardprocessor_ldml'; const keys = Object.keys(constants); keys.sort(); -console.log(`// +console.log(` +/* + Copyright: Copyright (C) 2022 SIL International. + Authors: srl295 + This file provides constants for the KMX Plus (LDML support) binary format, + to be shared between TypeScript and C++ via the generator (below) +*/ + +// // Generated File - do not edit // // This file is generated by core/tools/ldml-const-builder/build.sh diff --git a/core/src/kmx/kmx_plus.cpp b/core/src/kmx/kmx_plus.cpp new file mode 100644 index 0000000000..7949cb030f --- /dev/null +++ b/core/src/kmx/kmx_plus.cpp @@ -0,0 +1,230 @@ +/* + Copyright: Copyright (C) 2022 SIL International. + Authors: srl295 + Implementation for the KMX Plus utilities +*/ + +#include + +/** + * @def KMXPLUS_DEBUG Set to 1 to enable debug output + */ +#define KMXPLUS_DEBUG 1 + +#if KMXPLUS_DEBUG +#include +#endif + +#include + +namespace km { +namespace kbp { +namespace kmx { + +#if KMXPLUS_DEBUG +static void +dump_section_name(KMX_DWORD ident) { + for (int i = 0; i < 4; i++) { + putchar((ident & 0xFF000000) >> 24); + ident <<= 8; + } +} + +static void +dump_kmxplus_header(const COMP_KMXPLUS_HEADER* hdr) { + if (hdr == NULL) { + printf("! dump_kmxplus_header: NULL header\n"); + } + dump_section_name(hdr->ident); + printf(": (%X) size 0x%X\n", hdr->ident, hdr->size); +} + +static void +dump_kmxplus_keys(const uint8_t* /*data*/, const COMP_KMXPLUS_KEYS* keys) { + if(keys == NULL) { + printf("! could not load 'keys' section\n"); + return; + } + dump_kmxplus_header((const COMP_KMXPLUS_HEADER*)keys); + printf(" count: #0x%X\n", keys->count); + for (KMX_DWORD i = 0; icount; i++) { + printf(" #0x%d\n", i); + const COMP_KMXPLUS_KEYS_ENTRY& entry = keys->entries[i]; + printf(" vkey\t0x%X\n", entry.vkey); + printf(" mod\t0x%X\n", entry.mod); + printf(" flags\t0x%X\n", entry.flags); + if (entry.flags & LDML_KEYS_FLAGS_EXTEND) { + printf(" \t Extend: String #0x%X\n", entry.to); + } else { + printf(" \t UTF-32:U+%04X\n", entry.to); + } + } +} + +static void +dump_kmxplus_loca(const uint8_t* /*data*/, const COMP_KMXPLUS_LOCA* loca) { + if(loca == NULL) { + printf("! could not load 'loca' section\n"); + return; + } + dump_kmxplus_header((const COMP_KMXPLUS_HEADER*)loca); + // TODO-LDML + for(KMX_DWORD i=0; icount; i++) { + printf(" Locale #%d: #0x%X\n", i, loca->entries[i].locale); + } +} + +static void +dump_kmxplus_meta(const uint8_t* /*data*/, const COMP_KMXPLUS_META* meta) { + if(meta == NULL) { + printf("! could not load 'meta' section\n"); + return; + } + dump_kmxplus_header((const COMP_KMXPLUS_HEADER*)meta); + printf(" name:\t#0x%X\n", meta->name); + printf(" author:\t#0x%X\n", meta->author); + printf(" conform:\t#0x%X\n", meta->conform); + printf(" layout:\t#0x%X\n", meta->layout); + printf(" normalization:\t#0x%X\n", meta->normalization); + printf(" indicator:\t#0x%X\n", meta->indicator); + printf(" settings:\t0x%X\n", meta->settings); +} + +static void +dump_kmxplus_vkey(const uint8_t* /*data*/, const COMP_KMXPLUS_VKEY* vkey) { + if(vkey == NULL) { + printf("! could not load 'vkey' section\n"); + return; + } + dump_kmxplus_header((const COMP_KMXPLUS_HEADER*)vkey); + // TODO-LDML +} + +static void +dump_kmxplus_strs(const uint8_t* /*data*/, const COMP_KMXPLUS_STRS* strs) { + dump_kmxplus_header((const COMP_KMXPLUS_HEADER*)strs); + printf("strs: count 0x%X\n", strs->count); + for (KMX_DWORD i=0; icount; i++) { + KMX_WCHAR buf[BUFSIZ]; + printf("#0x%X: ", i); + PKMX_WCHAR str = strs->get(i, buf, BUFSIZ); + if (!str) { + printf("NULL/ERR\n"); + continue; + } + for(int j=0; str[j] && j<0x30; j++) { + if (str[j] < 0x7F && str[j] != 0x0020 && str[j] > 0x20) { + putchar(str[j]); + } else { + printf("U+%04X ", str[j]); + } + } + printf("\n"); + } +} + +static void +dump_kmxplus_sect(const uint8_t* data, const COMP_KMXPLUS_SECT* sect) { + dump_kmxplus_header((const COMP_KMXPLUS_HEADER*)sect); + printf("sect: total 0x%X\n", sect->total); + printf("sect: count 0x%X\n", sect->count); + + for (KMX_DWORD i = 0; i < sect->count; i++) { + const COMP_KMXPLUS_SECT_ENTRY& entry = sect->entries[i]; + dump_section_name(entry.sect); + printf(" sect#%d: %X @ %X\n", i, entry.sect, entry.offset); + const uint8_t* entrydata = (data+entry.offset); + switch(entry.sect) { + case LDML_SECTION_KEYS: + dump_kmxplus_keys(data, as_kmxplus_keys(entrydata)); + break; + case LDML_SECTION_LOCA: + dump_kmxplus_loca(data, as_kmxplus_loca(entrydata)); + break; + case LDML_SECTION_META: + dump_kmxplus_meta(data, as_kmxplus_meta(entrydata)); + break; + case LDML_SECTION_SECT: + printf("! Cowardly refusing to dump nested 'sect' section.\n"); + break; + case LDML_SECTION_STRS: + dump_kmxplus_strs(data, as_kmxplus_strs(entrydata)); + break; + case LDML_SECTION_VKEY: + dump_kmxplus_vkey(data, as_kmxplus_vkey(entrydata)); + break; + default: + printf("Unknown section %X", entry.sect); + } + } +} +#endif + +void +dump_kmxplus_data(const uint8_t* data) { +#if KMXPLUS_DEBUG + const COMP_KMXPLUS_SECT* sect = as_kmxplus_sect(data); + if (sect == NULL) { + printf("Err: 'sect' null from %p\n", data); + return; + } + dump_kmxplus_sect(data, sect); +#endif +} + +void +dump_kmxplus_data(kmx::PCOMP_KEYBOARD keyboard) { +#if KMXPLUS_DEBUG + printf("dump_kmxplus_data(): Got a PCOMP_KEYBOARD at %p\n", keyboard); + if (!(keyboard->dwFlags & KF_KMXPLUS)) { + printf("Err: flags KF_KMXPLUS not set\n"); + return; + } + const COMP_KEYBOARD_EX* ex = reinterpret_cast(keyboard); + + printf("KMXPlus offset 0x%X, KMXPlus size 0x%X\n", ex->kmxplus.dpKMXPlus, ex->kmxplus.dwKMXPlusSize); + const uint8_t* rawdata = reinterpret_cast(keyboard); + dump_kmxplus_data(rawdata + ex->kmxplus.dpKMXPlus); +#endif +} + +const COMP_KMXPLUS_KEYS_ENTRY *COMP_KMXPLUS_KEYS::find(KMX_DWORD vkey, KMX_DWORD mod) const { + // TODO-LDML: eventually, assume sorted order & binary search + for (KMX_DWORD i=0; i= count) { + return NULL; + } + KMX_DWORD offset = entries[entry].offset; + KMX_DWORD length = entries[entry].length; + assert(bufsiz > (length+1)); // assert bufsiz big enough + assert(offset+((length+1)*2) <= header.size); // assert not out of bounds + const uint8_t* thisptr = reinterpret_cast(this); + const KMX_WCHAR* start = reinterpret_cast(thisptr+offset); + for(KMX_DWORD i=0;i<=length;i++) { + buf[i] = start[i]; + } + return buf; +} + +} // namespace kmx +} // namespace kbp +} // namespace km diff --git a/core/src/kmx/kmx_plus.h b/core/src/kmx/kmx_plus.h new file mode 100644 index 0000000000..89c3421d7f --- /dev/null +++ b/core/src/kmx/kmx_plus.h @@ -0,0 +1,221 @@ +/* + Copyright: Copyright (C) 2022 SIL International. + Authors: srl295 + This file defines the structure of a kmx_plus file, starting at COMP_KEYBOARD_KMXPLUSINFO.dpKMXPlus +*/ + +#pragma once + +#include +#include +#include + +namespace km { +namespace kbp { +namespace kmx { + +struct COMP_KMXPLUS_HEADER { + KMX_DWORD ident; // 0000 Section name + KMX_DWORD size; // 0004 Section length +}; + +// Assert that the length matches the declared length +static_assert(sizeof(struct COMP_KMXPLUS_HEADER) == LDML_LENGTH_HEADER, "mismatched size of section header"); + +struct COMP_KMXPLUS_SECT_ENTRY { + KMX_DWORD sect; // 0010+ Section identity + KMX_DWORD offset; // 0014+ Section offset relative to dpKMXPlus of section +}; + +struct COMP_KMXPLUS_SECT { + COMP_KMXPLUS_HEADER header; + KMX_DWORD total; // 0008 KMXPlus entire length + KMX_DWORD count; // 000B number of section headers + COMP_KMXPLUS_SECT_ENTRY entries[0]; // 0010 section entries + /** + * @brief Get the offset of a section, or 0 + * + * @param ident section id such as 'strs'. Never 'sect' + * @return KMX_DWORD offset from beginning of kmxplus + */ + KMX_DWORD find(KMX_DWORD ident) const; +}; + +// Assert that the length matches the declared length +static_assert(sizeof(struct COMP_KMXPLUS_SECT) == LDML_LENGTH_SECT, "mismatched size of section sect"); + +struct COMP_KMXPLUS_STRS_ENTRY { + KMX_DWORD offset; // 0010+ offset from this blob + KMX_DWORD length; // 0014+ str length (UTF-16LE units) +}; + +struct COMP_KMXPLUS_STRS { + COMP_KMXPLUS_HEADER header; + KMX_DWORD count; // 0008 count of str entries + KMX_DWORD reserved; // 000C padding + COMP_KMXPLUS_STRS_ENTRY entries[0]; // 0010+ entries + + /** + * @brief Get a string entry + * + * @param entry entry number + * @param buf output buffer + * @param bufsiz buffer size in bytes + * @return NULL or a pointer to the output buffer + */ + PKMX_WCHAR get(KMX_DWORD entry, PKMX_WCHAR buf, KMX_DWORD bufsiz) const; +}; + +static_assert(sizeof(struct COMP_KMXPLUS_STRS) == LDML_LENGTH_STRS, "mismatched size of section strs"); + +struct COMP_KMXPLUS_META { + COMP_KMXPLUS_HEADER header; + KMX_DWORD name; + KMX_DWORD author; + KMX_DWORD conform; + KMX_DWORD layout; + KMX_DWORD normalization; + KMX_DWORD indicator; + KMX_DWORD settings; +}; + +static_assert(sizeof(struct COMP_KMXPLUS_META) == LDML_LENGTH_META, "mismatched size of section meta"); + +struct COMP_KMXPLUS_LOCA_ENTRY { + KMX_DWORD locale; // 000C+ locale string entry +}; + +struct COMP_KMXPLUS_LOCA { + COMP_KMXPLUS_HEADER header; + KMX_DWORD count; // 0008 number of locales + COMP_KMXPLUS_LOCA_ENTRY entries[0]; +}; + +static_assert(sizeof(struct COMP_KMXPLUS_LOCA) == LDML_LENGTH_LOCA, "mismatched size of section loca"); + +struct COMP_KMXPLUS_KEYS_ENTRY { + KMX_DWORD vkey; + KMX_DWORD mod; + KMX_DWORD to; + KMX_DWORD flags; +}; + +struct COMP_KMXPLUS_KEYS { + COMP_KMXPLUS_HEADER header; + KMX_DWORD count; // number of keys + KMX_DWORD reserved; // padding + COMP_KMXPLUS_KEYS_ENTRY entries[0]; + const COMP_KMXPLUS_KEYS_ENTRY *find(KMX_DWORD vkey, KMX_DWORD mod) const; +}; + +static_assert(sizeof(struct COMP_KMXPLUS_KEYS) == LDML_LENGTH_KEYS, "mismatched size of section keys"); + +struct COMP_KMXPLUS_VKEY_ENTRY { + KMX_DWORD vkey; + KMX_DWORD target; +}; + +struct COMP_KMXPLUS_VKEY { + COMP_KMXPLUS_HEADER header; + KMX_DWORD count; + COMP_KMXPLUS_VKEY_ENTRY entries[0]; +}; + +static_assert(sizeof(struct COMP_KMXPLUS_VKEY) == LDML_LENGTH_VKEY, "mismatched size of section vkey"); + +/** + * @brief Validate that this data is the named section. + * + * @param data raw data + * @param ident 4-byte section type + * @return COMP_KMXPLUS_ALLDATA* or null + */ +static inline const COMP_KMXPLUS_HEADER * +validate_as_section(const uint8_t *data, uint32_t ident) { + if (!data) { + return NULL; + } + const COMP_KMXPLUS_HEADER *all = reinterpret_cast(data); + // TODO-LDML these fail on 000null .. + assert(all->size >= LDML_LENGTH_HEADER); + assert(ident == all->ident); + if (ident != all->ident || (all->size < LDML_LENGTH_HEADER)) { + return NULL; // invalid header or wrong section + } + return all; +} + +/** + * cast raw data to section + * @return section data or null on error + */ +static inline const COMP_KMXPLUS_SECT * +as_kmxplus_sect(const uint8_t *data) { + const COMP_KMXPLUS_HEADER *all = validate_as_section(data, LDML_SECTION_SECT); + return reinterpret_cast(all); +} + +/** + * cast raw data to section + * @return section data or null on error + */ +static inline const COMP_KMXPLUS_STRS * +as_kmxplus_strs(const uint8_t *data) { + const COMP_KMXPLUS_HEADER *all = validate_as_section(data, LDML_SECTION_STRS); + return reinterpret_cast(all); +} + +/** + * cast raw data to section + * @return section data or null on error + */ +static inline const COMP_KMXPLUS_KEYS * +as_kmxplus_keys(const uint8_t *data) { + const COMP_KMXPLUS_HEADER *all = validate_as_section(data, LDML_SECTION_KEYS); + return reinterpret_cast(all); +} +/** + * cast raw data to section + * @return section data or null on error + */ +static inline const COMP_KMXPLUS_LOCA * +as_kmxplus_loca(const uint8_t *data) { + const COMP_KMXPLUS_HEADER *all = validate_as_section(data, LDML_SECTION_LOCA); + return reinterpret_cast(all); +} +/** + * cast raw data to section + * @return section data or null on error + */ +static inline const COMP_KMXPLUS_META * +as_kmxplus_meta(const uint8_t *data) { + const COMP_KMXPLUS_HEADER *all = validate_as_section(data, LDML_SECTION_META); + return reinterpret_cast(all); +} +/** + * convert raw data to section + * @return section data or null on error + */ +static inline const COMP_KMXPLUS_VKEY * +as_kmxplus_vkey(const uint8_t *data) { + const COMP_KMXPLUS_HEADER *all = validate_as_section(data, LDML_SECTION_VKEY); + return reinterpret_cast(all); +} + +/** + * @brief Temporary function to dump raw data + * May have no output if KMXPLUS_DEBUG is 0 + * @param kmxplusdata data from the beginning of the KMXPlus section + */ +void dump_kmxplus_data(void *kmxplusdata); + +/** + * @brief Temporary functino to dump raw data + * May have no output if KMXPLUS_DEBUG is 0 + * @param keyboard pointer to PCOMP_KEYBOARD with plus data following + */ +void dump_kmxplus_data(kmx::PCOMP_KEYBOARD keyboard); + +} // namespace kmx +} // namespace kbp +} // namespace km diff --git a/core/src/ldml/C7043_ldml.md b/core/src/ldml/C7043_ldml.md new file mode 100644 index 0000000000..934e50c9a2 --- /dev/null +++ b/core/src/ldml/C7043_ldml.md @@ -0,0 +1,165 @@ +# KMX Plus Binary Format #7043 + +- copied from + +- Authors: MD SL + +## C7043.0 Introduction + +This document discusses the binary format for KMXPlus, which contains keyboards converted from source LDML data. + +Draft spec PR: + +## C7043.1 Principles + +- The data described here is located at byte offset `dpKMXPlus`. +- All integer values are unsigned 32-bit little-endian unless otherwise specified. +- All strings are UTF-16LE unless otherwise specified. (See the `strs` section.) String data items are identified with `str:`, indicating a 32 bit index into the `strs` table. +- All offsets are 32-bit little-endian values. For all sections except for the `'sect'` section (which see), offsets are relative to the beginning of each section. + +## C7043.2 Sections + +- Data is divided into several sections. The very first section is the `'sect'` section which is the table of contents. +- All sections, including the first, begin on a 128-bit boundary with zero padding as needed. +- All sections begin with a 32-bit (four character) section identifier, and a 32-bit section size. +- Other than the `sect` table itself, the rest of the sections follow in binary order in the file. In other words, the binary ordering of the section identifiers determines the order of the file layout. + +### C7043.2.1 `sect`—Section Table of contents + +The very first section is a table of contents listing the rest of the sections. The table of contents does not list itself. + +This is the only section where all byte offsets are relative to the value of `dpKMXPlus`. + +| ∆ | Bits | Name | Description | +|---|------|---------|------------------------------------------| +| 0 | 32 | ident | `sect` | +| 4 | 32 | size | int: Length of section | +| 8 | 32 | total | int: KMXPlus entire length | +|12 | 32 | count | int: Number of following section headers | + +Then for `count` repetitions: + +| ∆ | Bits | Name | Description | +|---|------|---------|---------------------------------------------------| +|16+| 32 | sect | Section identity, e.g. `loca` | +|20+| 32 | offset | off: offset relative to `dpKMXPlus` of section | + +This list is in sorted order based on the `sect` identifier. + +### C7043.2.2 `strs`—Strings + +All strings are stored in the Strings section. + +| ∆ | Bits | Name | Description | +|---|------|---------------|-------------------------------------| +| 0 | 32 | ident | `strs` | +| 4 | 32 | size | int: Length of section | +| 8 | 32 | count | int: Number of strings | +|12 | 32 | reserved | Padding | + +Then for each string: + +| ∆ | Bits | Name | Description | +|---|------|---------------|-----------------------------------------------| +|16+| 32 | offset | off: Offset to string | +|20+| 32 | length | int: Length of string in UTF-16LE code units | + +After the string offset table comes the actual UTF-16LE data. There is a null (\u0000) after each string, which is _not_ included in the string length. + +The string offset table, and then strings themselves, are sorted according to a binary codepoint sort, not including the null. + +### C7043.2.3 `meta`—Metadata + +| ∆ | Bits | Name | Description | +|---|------|---------------|-------------------------------------| +| 0 | 32 | ident | `meta` | +| 4 | 32 | size | int: Length of section | +| 8 | 32 | name | str: Keyboard name | +|12 | 32 | author | str: Keyboard author | +|16 | 32 | conform | str: CLDR 'conformsTo' version | +|20 | 32 | layout | str: layout type | +|24 | 32 | normalization | str: normalization mode | +|28 | 32 | indicator | str: indicator | +|32 | 32 | settings | int: keyboard settings | + +The `settings` is a 32-bit bitfield as below: + +| Bit position | Meaning | Description | +|--------------|----------|---------------------------------------------| +| 0 | fallback | fallback=omit | +| 1 | transformFailure | transformFailure=omit | +| 2 | transformPartial | transformPartial=hide | + + +### C7043.2.4 `loca`—Locales + + +| ∆ | Bits | Name | Description | +|---|------|---------|------------------------------------------| +| 0 | 32 | ident | `loca` | +| 4 | 32 | size | int: Length of section | +| 8 | 32 | count | int: Number of locales | + +`count` is always ≥1, because a keyboard always has a primary locale identifier. + +For each locale ID in `count` + +| ∆ | Bits | Name | Description | +|---|------|---------|------------------------------------------| +|12+| 32 | locale | str: Locale ID in BCP47 format | + +The first locale ID is always the primary locale identifier. The rest of the locale IDs (starting at offset 16) are in sorted binary order. + +### C7043.2.5 `keys`—Keybag + +| ∆ | Bits | Name | Description | +|---|------|---------|------------------------------------------| +| 0 | 32 | ident | `keys` | +| 4 | 32 | size | int: Length of section | +| 8 | 32 | count | int: Number of keys | +|12 | 32 | reserved | reserved | + +The keys are sorted in binary order based on the `vkey` and `mod` fields. + +For each key: + +| ∆ | Bits | Name | Description | +|---|------|---------|------------------------------------------| +|16+| 32 | vkey | int: vkey ID | +|20+| 32 | mod | int: modifier key flags | +|24+| 32 | to | str: output string OR UTF-32LE codepoint | +|28+| 32 | flags | int: per-key flags | + +- `vkey`: If this is 0-255, it is the resolved standard/predefined vkey (K_A, etc.). It is resolved because the `vkeyMap` from LDML has already been applied. If this is 256 or above, it is a custom touch layout vkey generated by the compiler. +- `mod`: TODO define this. 0 for no modifiers. +- `flags`: Flags is a 32-bit bitfield defined as below: + +| Bit position | Meaning | Description | +|--------------|----------|---------------------------------------------| +| 0 | extend | 0: `to` is a char, 1: `to` is a string | + +- `to`: If `extend` is 0, `to` is a UTF-32LE codepoint. If `extend` is 1, `to` is a 32 bit index into the `strs` table. + +### C7043.2.6 `vkey`—VKey Map + +| ∆ | Bits | Name | Description | +|---|------|---------|------------------------------------------| +| 0 | 32 | ident | `vkey` | +| 4 | 32 | size | int: Length of section | +| 8 | 32 | count | int: Number of vkeys | + +The keys are sorted in binary order based on the `vkey` field. + +For each key: + +| ∆ | Bits | Name | Description | +|---|------|---------|------------------------------------------| +|12+| 32 | vkey | int: source vkey ID (0…255) | +|16+| 32 | target | int: target vkey ID (0…255) | + +- `vkey`: Is the standard vkey, 0-255 +- `target`: Is the target (resolved) vkey, 0-255. + +### C7043.2.7 Transforms and friends + +> TODO: transforms diff --git a/core/src/ldml/ldml_processor.cpp b/core/src/ldml/ldml_processor.cpp index 05f679fa37..d20caf44a3 100644 --- a/core/src/ldml/ldml_processor.cpp +++ b/core/src/ldml/ldml_processor.cpp @@ -9,8 +9,41 @@ #include "ldml/ldml_processor.hpp" #include "state.hpp" #include "../kmx/kmx_file.h" +#include "../kmx/kmx_plus.h" #include "ldml/keyboardprocessor_ldml.h" +// extern "C" { +// #include "../../../common/windows/cpp/src/ConvertUTF.c" +// } + +// HACK +enum ConversionResult { + conversionOK, + conversionNotOk +}; +enum ConversionFlags { + strictConversion +}; + +typedef KMX_WCHAR UTF16; +typedef KMX_DWORD UTF32; + +ConversionResult ConvertUTF32toUTF16( + UTF32** sourceStart, const UTF32* sourceEnd, + UTF16** targetStart, const UTF16* targetEnd, const ConversionFlags /*flags*/) { + if(sourceEnd > (*sourceStart+1)) return conversionNotOk; // Don't support >1 char + if(**sourceStart & 0xFFFF0000) { + // Don't support supplemental chars yet + return conversionNotOk; + } + *((*targetStart)++) = (UTF16)((*(*sourceStart)++) & 0xFFFF); // BMP + + assert(*sourceStart==sourceEnd); + assert(*targetStart <= targetEnd); + return(conversionOK); +} + + namespace { constexpr km_kbp_attr const engine_attrs = { 256, @@ -25,13 +58,14 @@ namespace { namespace km { namespace kbp { -ldml_processor::ldml_processor(path const & kb_path, const std::vector _kmn_unused(data)) +ldml_processor::ldml_processor(path const & kb_path, const std::vector &data) : abstract_processor( keyboard_attributes(kb_path.stem(), KM_KBP_LMDL_PROCESSOR_VERSION, kb_path.parent(), {}) - ) + ), rawdata(data) // TODO-LDML: parse the data, don't just copy it { // TODO-LDML: load the file from the buffer (KMXPlus format) // Note: kb_path is essentially debug metadata here + assert(data.size() != 0); } bool ldml_processor::is_kmxplus_file(path const & kb_path, std::vector& data) { @@ -50,7 +84,7 @@ bool ldml_processor::is_kmxplus_file(path const & kb_path, std::vector& file.seekg(0, std::ios::beg); - data.reserve((size_t)size); + data.resize((size_t)size); if(!file.read((char *) data.data(), size)) { return false; } @@ -67,6 +101,9 @@ bool ldml_processor::is_kmxplus_file(path const & kb_path, std::vector& return false; } + // Dump data. This also does some validation. + dump_kmxplus_data(comp_keyboard); + // A KMXPlus file is in the buffer (although more validation is required) return true; } @@ -98,7 +135,7 @@ km_kbp_status ldml_processor::process_event( km_kbp_state *state, km_kbp_virtual_key vk, - uint16_t _kmn_unused(modifier_state), + uint16_t modifier_state, uint8_t is_key_down, uint16_t /*event_flags*/ // TODO-LDML: unused... for now... ) { @@ -123,9 +160,65 @@ ldml_processor::process_event( state->actions().push_backspace(KM_KBP_BT_UNKNOWN); // Assuming we don't know the character break; default: - /* We're going to push an 'a' for a passing unit test */ - state->context().push_character('a'); - state->actions().push_character('a'); + // TODO-LDML: temporary code here + // Don't want to do this work each time + const kmx::PCOMP_KEYBOARD comp_keyboard = (kmx::PCOMP_KEYBOARD)rawdata.data(); + assert(comp_keyboard->dwFlags & KF_KMXPLUS); + const kmx::COMP_KEYBOARD_EX* ex = reinterpret_cast(comp_keyboard); + + // printf("KMXPlus offset 0x%X, KMXPlus size 0x%X\n", ex->kmxplus.dpKMXPlus, ex->kmxplus.dwKMXPlusSize); + const uint8_t* kmxplusdata = rawdata.data() + ex->kmxplus.dpKMXPlus; + // Get out the SECT header + const kmx::COMP_KMXPLUS_SECT *sect = kmx::as_kmxplus_sect(kmxplusdata); + assert(sect != NULL); + assert(sect->header.ident == LDML_SECTION_SECT); + KMX_DWORD offset; + // Fill out the other sections we need. + offset = sect->find(LDML_SECTION_STRS); + assert(offset != 0); // or else section not found + const kmx::COMP_KMXPLUS_STRS *strs = kmx::as_kmxplus_strs(kmxplusdata+offset); + assert(strs->header.ident == LDML_SECTION_STRS); + offset = sect->find(LDML_SECTION_KEYS); + assert(offset != 0); // or else section not found + const kmx::COMP_KMXPLUS_KEYS *keys = kmx::as_kmxplus_keys(kmxplusdata+offset); + assert(keys->header.ident == LDML_SECTION_KEYS); + // Look up the key + const kmx::COMP_KMXPLUS_KEYS_ENTRY *key = keys->find(vk, modifier_state); + assert(key != NULL); + if (!key) { + return KM_KBP_STATUS_KEY_ERROR; + } + // Prepare output chars + KMX_DWORD len = 0; + KMX_WCHAR out[BUFSIZ]; + if (key->flags && LDML_KEYS_FLAGS_EXTEND) { + // It's a string. + assert(NULL != strs->get(key->to, out, BUFSIZ)); + // u_strlen() + for(len=0; lento; // UTF-32 + buf32[1] = 0; // to avoid UMR warning + UTF32 *sourceStart = &buf32[0]; + const UTF32 *sourceEnd = &buf32[1]; // Reference off the end. NULL to avoid UMR. + UTF16 *targetStart = (UTF16*)out; + const UTF16 *targetEnd = (UTF16*)out+BUFSIZ-1; + ConversionResult result = ::ConvertUTF32toUTF16(&sourceStart, sourceEnd, &targetStart, targetEnd, strictConversion); + assert(result == conversionOK); + *targetStart = 0; + len = 1; // TODO=LDML calculate + // len = (targetStart - out); + assert(len>=1 && len <= 2); + } + assert(len>=0); + assert(lencontext().push_character(out[i]); + state->actions().push_character(out[i]); + } } state->actions().commit(); diff --git a/core/src/ldml/ldml_processor.hpp b/core/src/ldml/ldml_processor.hpp index a5490ff07c..6e7851c0ca 100644 --- a/core/src/ldml/ldml_processor.hpp +++ b/core/src/ldml/ldml_processor.hpp @@ -11,7 +11,6 @@ #include #include #include - #include "processor.hpp" #include "option.hpp" @@ -21,11 +20,12 @@ namespace kbp { #define KM_KBP_LMDL_PROCESSOR_VERSION u"1.0" class ldml_processor : public abstract_processor { - + private: + std::vector rawdata; // TODO-LDML: should be 'unpacked' format instead. public: ldml_processor( path const & kb_path, - const std::vector data + const std::vector & data ); // ~ldml_processor() override; diff --git a/core/src/meson.build b/core/src/meson.build index 83ef77697e..0ab24eb7f1 100644 --- a/core/src/meson.build +++ b/core/src/meson.build @@ -83,6 +83,7 @@ lib = library('kmnkbp0', 'kmx/kmx_file.cpp', 'kmx/kmx_modifiers.cpp', 'kmx/kmx_options.cpp', + 'kmx/kmx_plus.cpp', 'kmx/kmx_processor.cpp', 'kmx/kmx_xstring.cpp', 'utfcodec.cpp', diff --git a/core/tests/unit/ldml/001_tiny.kmx b/core/tests/unit/ldml/001_tiny.kmx index 3c160a8706..b25c094fed 100644 Binary files a/core/tests/unit/ldml/001_tiny.kmx and b/core/tests/unit/ldml/001_tiny.kmx differ diff --git a/core/tests/unit/ldml/meson.build b/core/tests/unit/ldml/meson.build index a2c0bfa2ad..53fd7bf014 100644 --- a/core/tests/unit/ldml/meson.build +++ b/core/tests/unit/ldml/meson.build @@ -33,7 +33,7 @@ ldml = executable('ldml', objects: lib.extract_all_objects()) tests = [ - '000_null_keyboard', + # '000_null_keyboard', '001_tiny' ] @@ -166,3 +166,12 @@ else test(kbd, ldml, args: [kbd_src, kbd_obj]) endforeach endif + + +e = executable('test_kmx_plus', 'test_kmx_plus.cpp', + cpp_args: defns + warns, + include_directories: [inc, libsrc], + link_args: links + tests_flags, + objects: lib.extract_all_objects()) +test('test_kmx_plus', e) + diff --git a/core/tests/unit/ldml/test_kmx_plus.cpp b/core/tests/unit/ldml/test_kmx_plus.cpp new file mode 100644 index 0000000000..1d32184a57 --- /dev/null +++ b/core/tests/unit/ldml/test_kmx_plus.cpp @@ -0,0 +1,10 @@ +#include +#include +#include + +using namespace km::kbp::kmx; + +int main(int argc, const char *argv[]) { + printf("OKAY\n"); + return 0; +}