diff --git a/core/include/ldml/keyboardprocessor_ldml.h b/core/include/ldml/keyboardprocessor_ldml.h index 71e5d0dd1c..732bb278b2 100644 --- a/core/include/ldml/keyboardprocessor_ldml.h +++ b/core/include/ldml/keyboardprocessor_ldml.h @@ -15,7 +15,21 @@ #pragma once +#define LDML_ELEM_FLAGS_ORDER_BITSHIFT 0x10 +#define LDML_ELEM_FLAGS_ORDER_MASK 0xFF0000 +#define LDML_ELEM_FLAGS_PREBASE 0x4 +#define LDML_ELEM_FLAGS_TERTIARY_BASE 0x2 +#define LDML_ELEM_FLAGS_TERTIARY_BITSHIFT 0x18 +#define LDML_ELEM_FLAGS_TERTIARY_MASK 0xFF000000 +#define LDML_ELEM_FLAGS_UNICODE_SET 0x1 #define LDML_KEYS_FLAGS_EXTEND 0x1 +#define LDML_LENGTH_BKSP 0x10 +#define LDML_LENGTH_BKSP_ITEM 0x10 +#define LDML_LENGTH_ELEM 0x10 +#define LDML_LENGTH_ELEM_ITEM 0x8 +#define LDML_LENGTH_ELEM_ITEM_ELEMENT 0x8 +#define LDML_LENGTH_FINL 0x10 +#define LDML_LENGTH_FINL_ITEM 0x10 #define LDML_LENGTH_HEADER 0x8 #define LDML_LENGTH_KEYS 0x10 #define LDML_LENGTH_KEYS_ITEM 0x10 @@ -24,15 +38,25 @@ #define LDML_LENGTH_META 0x28 #define LDML_LENGTH_NAME 0x10 #define LDML_LENGTH_NAME_ITEM 0x4 +#define LDML_LENGTH_ORDR 0x10 +#define LDML_LENGTH_ORDR_ITEM 0x8 #define LDML_LENGTH_SECT 0x10 #define LDML_LENGTH_SECT_ITEM 0x8 #define LDML_LENGTH_STRS 0x10 #define LDML_LENGTH_STRS_ITEM 0x8 +#define LDML_LENGTH_TRAN 0x10 +#define LDML_LENGTH_TRAN_ITEM 0x10 #define LDML_LENGTH_VKEY 0x10 #define LDML_LENGTH_VKEY_ITEM 0x8 #define LDML_META_SETTINGS_FALLBACK_OMIT 0x1 #define LDML_META_SETTINGS_TRANSFORMFAILURE_OMIT 0x2 #define LDML_META_SETTINGS_TRANSFORMPARTIAL_HIDE 0x4 +#define LDML_SECTIONID_BKSP 0x70736B62 /* "bksp" */ +#define LDML_SECTIONNAME_BKSP "bksp" +#define LDML_SECTIONID_ELEM 0x6D656C65 /* "elem" */ +#define LDML_SECTIONNAME_ELEM "elem" +#define LDML_SECTIONID_FINL 0x6C6E6966 /* "finl" */ +#define LDML_SECTIONNAME_FINL "finl" #define LDML_SECTIONID_KEYS 0x7379656B /* "keys" */ #define LDML_SECTIONNAME_KEYS "keys" #define LDML_SECTIONID_LOCA 0x61636F6C /* "loca" */ @@ -41,10 +65,14 @@ #define LDML_SECTIONNAME_META "meta" #define LDML_SECTIONID_NAME 0x656D616E /* "name" */ #define LDML_SECTIONNAME_NAME "name" +#define LDML_SECTIONID_ORDR 0x7264726F /* "ordr" */ +#define LDML_SECTIONNAME_ORDR "ordr" #define LDML_SECTIONID_SECT 0x74636573 /* "sect" */ #define LDML_SECTIONNAME_SECT "sect" #define LDML_SECTIONID_STRS 0x73727473 /* "strs" */ #define LDML_SECTIONNAME_STRS "strs" +#define LDML_SECTIONID_TRAN 0x6E617274 /* "tran" */ +#define LDML_SECTIONNAME_TRAN "tran" #define LDML_SECTIONID_VKEY 0x79656B76 /* "vkey" */ #define LDML_SECTIONNAME_VKEY "vkey" #define LDML_VERSION "1.0" diff --git a/core/include/ldml/keyboardprocessor_ldml.ts b/core/include/ldml/keyboardprocessor_ldml.ts index ec6dc4c4fc..91c384c289 100644 --- a/core/include/ldml/keyboardprocessor_ldml.ts +++ b/core/include/ldml/keyboardprocessor_ldml.ts @@ -24,13 +24,19 @@ */ export type SectionIdent = 'sect' | + 'bksp' | + 'elem' | + 'finl' | 'keys' | 'loca' | 'meta' | 'name' | + 'ordr' | 'strs' | + 'tran' | 'vkey'; + type SectionMap = { [id in SectionIdent]: SectionIdent; } @@ -74,6 +80,116 @@ export const constants: Constants = { */ length_sect_item: 8, + /* ------------------------------------------------------------------ + * bksp section + ------------------------------------------------------------------ */ + + /** + * Minimum length of the 'bksp' section, not including entries + */ + length_bksp: 16, + /** + * Length of each item in the 'bksp' section variable part + */ + length_bksp_item: 16, + + /* ------------------------------------------------------------------ + * elem section + ------------------------------------------------------------------ */ + + /** + * Minimum length of the 'elem' section, not including entries + */ + length_elem: 16, + /** + * Length of each elem string in the 'elem' section variable part + */ + length_elem_item: 8, + /** + * Length of each element in an elem string + */ + length_elem_item_element: 8, + + /** + * bitwise or value for unicode_set in elem[elemstr][element].flags. + * If bit is 1, then 'element' is a UnicodeSet string. + * If bit is 0, then 'element' is a UTF-32LE codepoint + * + * `unicode_set = flags & elem_flags_unicode_set` + */ + elem_flags_unicode_set: 0x00000001, + + /** + * bitwise or value for tertiary_base in elem[elemstr][element].flags. + * If bit is 1, then tertiary_base is true. + * If bit is 0, then tertiary_base is false. + * + * Used only for `ordr`-type element strings. + * + * `tertiary_base = flags & elem_flags_tertiary_base` + */ + elem_flags_tertiary_base: 0x00000002, + + /** + * bitwise or value for tertiary_base in elem[elemstr][element].flags. + * If bit is 1, then prebase is true. + * If bit is 0, then prebase is false. + * + * Used only for `ordr`-type element strings. + * + * `prebase = flags & elem_flags_prebase` + */ + elem_flags_prebase: 0x00000004, + + /** + * bitwise mask for order in elem[elemstr][element].flags. + * + * Used only for `ordr`-type element strings. 1 byte signed integer. + * + * `order = (flags & elem_flags_order_mask) >> elem_flags_order_bitshift` + */ + elem_flags_order_mask: 0x00FF0000, + + /** + * bit shift for order in elem[elemstr][element].flags. + * + * Used only for `ordr`-type element strings. + * + * `order = (flags & elem_flags_order_mask) >> elem_flags_order_bitshift` + */ + elem_flags_order_bitshift: 16, + + /** + * bitwise mask for tertiary sort in elem[elemstr][element].flags. + * + * Used only for `ordr`-type element strings. 1 byte signed integer. + * + * `tertiary = (flags & elem_flags_tertiary_mask) >> elem_flags_tertiary_bitshift` + */ + elem_flags_tertiary_mask: 0xFF000000, + + /** + * bit shift for tertiary sort in elem[elemstr][element].flags. + * + * Used only for `ordr`-type element strings. 1 byte signed integer. + * + * `order = (flags & elem_flags_tertiary_mask) >> elem_flags_tertiary_bitshift` + */ + elem_flags_tertiary_bitshift: 24, + + /* ------------------------------------------------------------------ + * finl section + ------------------------------------------------------------------ */ + + /** + * Minimum length of the 'finl' section, not including entries + */ + length_finl: 16, + /** + * Length of each item in the 'finl' section variable part + */ + length_finl_item: 16, + /* ------------------------------------------------------------------ * keys section ------------------------------------------------------------------ */ @@ -93,7 +209,7 @@ export const constants: Constants = { * * `extend = flags & keys_flags_extend` */ - keys_flags_extend: 1, + keys_flags_extend: 1, /* ------------------------------------------------------------------ * loca section @@ -136,11 +252,24 @@ export const constants: Constants = { /** * Minimum length of the 'name' section not including variable parts */ - length_name: 16, - /** - * Length of each item in the 'name' section variable part - */ - length_name_item: 4, + length_name: 16, + /** + * Length of each item in the 'name' section variable part + */ + length_name_item: 4, + + /* ------------------------------------------------------------------ + * ordr section + ------------------------------------------------------------------ */ + + /** + * Minimum length of the 'ordr' section, not including entries + */ + length_ordr: 16, + /** + * Length of each item in the 'ordr' section variable part + */ + length_ordr_item: 8, /* ------------------------------------------------------------------ * strs section @@ -155,15 +284,30 @@ export const constants: Constants = { */ length_strs_item: 8, + /* ------------------------------------------------------------------ + * tran section + ------------------------------------------------------------------ */ + + /** + * Minimum length of the 'tran' section, not including entries + */ + length_tran: 16, + /** + * Length of each item in the 'tran' section variable part + */ + length_tran_item: 16, + /* ------------------------------------------------------------------ * vkey section ------------------------------------------------------------------ */ /** - * Minimum length of the 'vkey' section - * not including variable parts + * Minimum length of the 'vkey' section not including variable parts */ length_vkey: 16, + /** + * Length of each item in the 'vkey' section variable part + */ length_vkey_item: 8, /** @@ -171,11 +315,16 @@ export const constants: Constants = { */ section: { sect: 'sect', + bksp: 'bksp', + elem: 'elem', + finl: 'finl', keys: 'keys', loca: 'loca', meta: 'meta', name: 'name', + ordr: 'ordr', strs: 'strs', + tran: 'tran', vkey: 'vkey', }, diff --git a/core/src/kmx/kmx_plus.h b/core/src/kmx/kmx_plus.h index 27d0159415..8cab5ade90 100644 --- a/core/src/kmx/kmx_plus.h +++ b/core/src/kmx/kmx_plus.h @@ -35,6 +35,11 @@ namespace kmx { #endif typedef KMX_DWORD KMXPLUS_STR; +typedef KMX_DWORD KMXPLUS_ELEM; + +// forward declarations +struct COMP_KMXPLUS_TRAN_ENTRY; +struct COMP_KMXPLUS_TRAN; struct COMP_KMXPLUS_HEADER { KMX_DWORD ident; // 0000 Section name @@ -78,6 +83,53 @@ struct COMP_KMXPLUS_SECT { static_assert(sizeof(struct COMP_KMXPLUS_SECT) == LDML_LENGTH_SECT, "mismatched size of section sect"); static_assert(sizeof(struct COMP_KMXPLUS_SECT) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); +/* ------------------------------------------------------------------ + * bksp section + ------------------------------------------------------------------ */ + +typedef COMP_KMXPLUS_TRAN_ENTRY COMP_KMXPLUS_BKSP_ENTRY; +typedef COMP_KMXPLUS_TRAN COMP_KMXPLUS_BKSP; + +/* ------------------------------------------------------------------ + * elem section + ------------------------------------------------------------------ */ + +struct COMP_KMXPLUS_ELEM_ELEMENT { + KMX_DWORD element; // str: output string or UTF-32LE codepoint + KMX_DWORD flags; // flag and order values +}; + +struct COMP_KMXPLUS_ELEM_ENTRY { + KMX_DWORD offset; // 0010+ offset from this blob + KMX_DWORD length; // 0014+ str length (ELEMENT units) +}; + +struct COMP_KMXPLUS_ELEM { + static const KMX_DWORD IDENT = LDML_SECTIONID_ELEM; + COMP_KMXPLUS_HEADER header; + KMX_DWORD count; // 0008 count of str entries + KMX_DWORD reserved; // 000C padding + COMP_KMXPLUS_ELEM_ENTRY entries[]; // 0010+ entries + + /** + * @brief True if section is valid. + */ + bool valid(KMX_DWORD length) const; +}; + +static_assert(sizeof(struct COMP_KMXPLUS_ELEM) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); +static_assert(sizeof(struct COMP_KMXPLUS_ELEM) == LDML_LENGTH_ELEM, "mismatched size of section elem"); + +/* ------------------------------------------------------------------ + * finl section + ------------------------------------------------------------------ */ + +// TODO-LDML: IDENT +typedef COMP_KMXPLUS_TRAN_ENTRY COMP_KMXPLUS_FINL_ENTRY; + +// TODO-LDML: IDENT +typedef COMP_KMXPLUS_TRAN COMP_KMXPLUS_FINL; + /* ------------------------------------------------------------------ * keys section ------------------------------------------------------------------ */ @@ -173,6 +225,30 @@ struct COMP_KMXPLUS_NAME { static_assert(sizeof(struct COMP_KMXPLUS_NAME) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); static_assert(sizeof(struct COMP_KMXPLUS_NAME) == LDML_LENGTH_NAME, "mismatched size of section name"); +/* ------------------------------------------------------------------ + * ordr section + ------------------------------------------------------------------ */ + +struct COMP_KMXPLUS_ORDR_ENTRY { + KMXPLUS_ELEM elements; + KMXPLUS_ELEM before; +}; + +struct COMP_KMXPLUS_ORDR { + static const KMX_DWORD IDENT = LDML_SECTIONID_ORDR; + COMP_KMXPLUS_HEADER header; + KMX_DWORD count; + KMX_DWORD reserved; + COMP_KMXPLUS_ORDR_ENTRY entries[]; + /** + * @brief True if section is valid. + */ + bool valid(KMX_DWORD length) const; +}; + +static_assert(sizeof(struct COMP_KMXPLUS_ORDR) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); +static_assert(sizeof(struct COMP_KMXPLUS_ORDR) == LDML_LENGTH_ORDR, "mismatched size of section ordr"); + /* ------------------------------------------------------------------ * strs section ------------------------------------------------------------------ */ @@ -207,6 +283,33 @@ struct COMP_KMXPLUS_STRS { static_assert(sizeof(struct COMP_KMXPLUS_STRS) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); static_assert(sizeof(struct COMP_KMXPLUS_STRS) == LDML_LENGTH_STRS, "mismatched size of section strs"); +/* ------------------------------------------------------------------ + * tran section + ------------------------------------------------------------------ */ + +struct COMP_KMXPLUS_TRAN_ENTRY { + KMXPLUS_ELEM from; + KMXPLUS_STR to; + KMXPLUS_ELEM before; + KMX_DWORD flags; +}; + + +struct COMP_KMXPLUS_TRAN { + static const KMX_DWORD IDENT = LDML_SECTIONID_TRAN; + COMP_KMXPLUS_HEADER header; + KMX_DWORD count; + KMX_DWORD reserved; + COMP_KMXPLUS_TRAN_ENTRY entries[]; + /** + * @brief True if section is valid. + */ + bool valid(KMX_DWORD length) const; +}; + +static_assert(sizeof(struct COMP_KMXPLUS_TRAN) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); +static_assert(sizeof(struct COMP_KMXPLUS_TRAN) == LDML_LENGTH_TRAN, "mismatched size of section tran"); + /* ------------------------------------------------------------------ * vkey section ------------------------------------------------------------------ */ diff --git a/core/src/ldml/C7043_ldml.md b/core/src/ldml/C7043_ldml.md index 0e69de493c..cde45bf1c6 100644 --- a/core/src/ldml/C7043_ldml.md +++ b/core/src/ldml/C7043_ldml.md @@ -62,7 +62,57 @@ Then for `count` repetitions: This list is in sorted order based on the `sect` identifier. -### C7043.2.2 `keys`—Keybag +### C7043.2.2 `bksp`—Backspace transform + +See [C7043.2.11](#c7043211-tran-finl-and-bksptransforms). + +### C7043.2.3 `elem`—Transform and Reorder element strings + +| ∆ | Bits | Name | Description | +|---|------|----------|------------------------------------------| +| 0 | 32 | ident | `elem` | +| 4 | 32 | size | int: Length of section | +| 8 | 32 | count | int: Number of element string entries | +|12 | 32 | reserved | padding | + +Then for each element string: + +| ∆ | Bits | Name | Description | +|---|------|---------------|-----------------------------------------------| +|16+| 32 | offset | off: Offset to element string | +|20+| 32 | length | int: Number of elements in element string | + +The first entry in the element string list MUST be the null element string, +which has zero length and zero offset. + +Each element string is made up of elements with the following item structure: + +| ∆ | Bits | Name | Description | +|---|------|-----------|----------------------------------------------------------| +| 0 | 32 | element | str: output string OR UTF-32LE codepoint | +| 4 | 32 | flags | flags and order values | + +- `element`: either a UnicodeSet stored in a `strs` section entry, or a UTF-32LE + codepoint; see also `unicode_set` flag. +- `flags`: a 32-bit bitfield defined as below: + + | Bit position | Meaning | Description | + |--------------|---------------|----------------------------------| + | 0 | unicode_set | `element` is 0: UTF-32LE, 1: str | + | 1 | tertiary_base | 1: tertiary_base is true | + | 2 | prebase | 1: prebase is true | + | 3-15 | reserved | reserved | + | 16-23 | order | signed int: -128 to +127 | + | 24-31 | tertiary | signed int: -128 to +127 | + + For transforms, only `flags.unicode_set` will be used. The remaining flags are + used for reorders, `from` attribute only. + +### C7043.2.4 `finl`—Final transform + +See [C7043.2.11](#c7043211-tran-finl-and-bksptransforms). + +### C7043.2.5 `keys`—Keybag | ∆ | Bits | Name | Description | |---|------|-----------|------------------------------------------| @@ -96,7 +146,7 @@ For each key: - `to`: If `extend` is 0, `to` is a UTF-32LE codepoint. If `extend` is 1, `to` is a 32 bit index into the `strs` table. The string may be zero-length. -### C7043.2.3 `loca`—Locales +### C7043.2.6 `loca`—Locales | ∆ | Bits | Name | Description | |---|------|---------|------------------------------------------| @@ -116,7 +166,7 @@ For each locale ID in `count` The first locale ID is always the primary locale identifier. The rest of the locale IDs (starting at offset 16) are in sorted binary order. -### C7043.2.4 `meta`—Metadata +### C7043.2.7 `meta`—Metadata | ∆ | Bits | Name | Description | |---|------|---------------|-------------------------------------| @@ -138,7 +188,7 @@ The `settings` is a 32-bit bitfield as below: | 1 | transformFailure | transformFailure=omit | | 2 | transformPartial | transformPartial=hide | -### C7043.2.5 `name`—Names +### C7043.2.8 `name`—Names Defines the names of the keyboard as found in the source `` element. While this section is optional in the binary format, in practice it will always @@ -163,7 +213,27 @@ Note that the first name is repeated in the `meta` section. The remaining names are stored in source file order, and the semantic meaning of each name is not defined here. -### C7043.2.6 `strs`—Strings +### C7043.2.9 `ordr`—Reorders + +| ∆ | Bits | Name | Description | +|---|------|----------|------------------------------------------| +| 0 | 32 | ident | `ordr` | +| 4 | 32 | size | int: Length of section | +| 8 | 32 | count | int: Number of reorders | +|12 | 32 | reserved | padding | + +For each reorder item: + +| ∆ | Bits | Name | Description | +|---|------|----------|----------------------------------------------------------| +|16+| 32 | elements | elem: string of elements, index into `elem` section | +|20+| 32 | before | elem: look-behind required match, index into `elem` | + +- `elements`: index into the `elem` section, coding `from`, `order`, `tertiary`, + `tertiary_base`, and `prebase` properties. +- `before`: follows `transform/@before` + +### C7043.2.10 `strs`—Strings All strings are stored in the Strings section. @@ -198,7 +268,44 @@ A distinction between zero-length string and optional should be avoided (e.g. the difference between "" and null in Javascript). If this is truly required, a separate flag field must be used to denote the difference. -### C7043.2.7 `vkey`—VKey Map +### C7043.2.11 `tran`, `finl`, and `bksp`—Transforms + +All three of these tables have the same format. and differ only in their +identity. The simple transform table has the ident `tran`; the final transform +table has the ident `finl`, and the backspaces table has the ident `bksp`. + + +| ∆ | Bits | Name | Description | +|---|------|----------|------------------------------------------| +| 0 | 32 | ident | `tran` / `finl` / `bksp` | +| 4 | 32 | size | int: Length of section | +| 8 | 32 | count | int: Number of transforms | +|12 | 32 | reserved | padding | + + +The transforms are sorted in binary order based on the `from` field. + +For each transform: + +| ∆ | Bits | Name | Description | +|---|------|---------|------------------------------------------| +|16+| 32 | from | elem: combination of characters | +|20+| 32 | to | str: output text | +|24+| 32 | before | elem: look-behind text (0 = omitted) | +|28+| 32 | flags | int: per-transform flags | + +- `from`: the source text, index into `elem` section. +- `to`: sequence of Unicode codepoints that replace `from`. May be the null + string for `bksp` entries. +- `before`: optional look-behind text occurring before `from`, index into `elem` + section +- `flags`: a 32-bit bitfield defined as below: + + | Bit position | Meaning | Description | + |--------------|----------|----------------------| + | 0 | error | 1: `error="fail"` | + +### C7043.2.12 `vkey`—VKey Map | ∆ | Bits | Name | Description | |---|------|---------|------------------------------------------| @@ -219,6 +326,10 @@ For each key: - `vkey`: Is the standard vkey, 0-255 - `target`: Is the target (resolved) vkey, 0-255. -### C7043.2.8 Transforms and friends +## TODO-LDML: various things that need to be completed here or fixed in-spec -> TODO: transforms +> * UnicodeSets +> * spec: reference to `after` in reorders; various other @after refs +> * spec: ABNT2 key has hex value 0xC1 (even if kbdus.dll doesn't produce that) +> * spec: layerMaps.displayWidthInMm +> * `keys.key.mod`: TODO define this. 0 for no modifiers. \ No newline at end of file diff --git a/developer/src/kmldmlc/src/keyman/kmx/kmx-plus.ts b/developer/src/kmldmlc/src/keyman/kmx/kmx-plus.ts index b2a9005ea7..a1c78ecbbc 100644 --- a/developer/src/kmldmlc/src/keyman/kmx/kmx-plus.ts +++ b/developer/src/kmldmlc/src/keyman/kmx/kmx-plus.ts @@ -103,11 +103,16 @@ export default class KMXPlusFile extends KMXFile { public kmxplus: { sect?: Section; // sect is ignored here for writing + bksp?: Section; // TODO-LDML + elem?: Section; // TODO-LDML + finl?: Section; // TODO-LDML keys?: Keys; loca?: Loca; meta?: Meta; name?: Name; + ordr?: Section; // TODO-LDML strs?: Section; // strs is ignored here for writing + tran?: Section; // TODO-LDML vkey?: Vkey; } = { };