From ea6bd8f09fef03d015aeb8711447a19486ac2f2a Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Sun, 4 Sep 2022 06:10:34 +1000 Subject: [PATCH] chore(core): order sections according to binary format Just reorders all work around the various sections to match the binary format order -- that is, 'sect' first, then all other sections alphabetically. This also means the compiler now emits the sections in the expected order. No other changes to functionality. --- core/include/ldml/keyboardprocessor_ldml.ts | 4 +- core/src/kmx/kmx_plus.h | 191 ++++++----- core/src/ldml/C7043_ldml.md | 208 ++++++------ core/tests/unit/kmx/kmx_imsample.kmx | Bin 888 -> 884 bytes core/tests/unit/kmx/kmx_key_list.kmx | Bin 806 -> 802 bytes .../kmldmlc/src/keyman/compiler/compiler.ts | 4 +- .../src/keyman/kmx/kmx-plus-builder.ts | 298 ++++++++++-------- .../src/kmldmlc/src/keyman/kmx/kmx-plus.ts | 176 ++++++----- developer/src/kmldmlc/test/fixtures/basic.txt | 96 +++--- 9 files changed, 524 insertions(+), 453 deletions(-) diff --git a/core/include/ldml/keyboardprocessor_ldml.ts b/core/include/ldml/keyboardprocessor_ldml.ts index ecb332f618..ec6dc4c4fc 100644 --- a/core/include/ldml/keyboardprocessor_ldml.ts +++ b/core/include/ldml/keyboardprocessor_ldml.ts @@ -23,11 +23,11 @@ * through. */ export type SectionIdent = + 'sect' | 'keys' | 'loca' | 'meta' | 'name' | - 'sect' | 'strs' | 'vkey'; @@ -170,11 +170,11 @@ export const constants: Constants = { * All section IDs. */ section: { + sect: 'sect', keys: 'keys', loca: 'loca', meta: 'meta', name: 'name', - sect: 'sect', strs: 'strs', vkey: 'vkey', }, diff --git a/core/src/kmx/kmx_plus.h b/core/src/kmx/kmx_plus.h index 54c04d71aa..27d0159415 100644 --- a/core/src/kmx/kmx_plus.h +++ b/core/src/kmx/kmx_plus.h @@ -78,6 +78,101 @@ struct COMP_KMXPLUS_SECT { static_assert(sizeof(struct COMP_KMXPLUS_SECT) == LDML_LENGTH_SECT, "mismatched size of section sect"); static_assert(sizeof(struct COMP_KMXPLUS_SECT) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); +/* ------------------------------------------------------------------ + * keys section + ------------------------------------------------------------------ */ + +struct COMP_KMXPLUS_KEYS_ENTRY { + KMX_DWORD vkey; + KMX_DWORD mod; + KMX_DWORD to; // to may be KMXPLUS_STR or UTF32 char + KMX_DWORD flags; +}; + +struct COMP_KMXPLUS_KEYS { + static const KMX_DWORD IDENT = LDML_SECTIONID_KEYS; + COMP_KMXPLUS_HEADER header; + KMX_DWORD count; // number of keys + KMX_DWORD reserved; // padding + COMP_KMXPLUS_KEYS_ENTRY entries[]; + /** + * @brief True if section is valid. + */ + bool valid(KMX_DWORD length) const; +}; + +static_assert(sizeof(struct COMP_KMXPLUS_KEYS) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); +static_assert(sizeof(struct COMP_KMXPLUS_KEYS) == LDML_LENGTH_KEYS, "mismatched size of section keys"); + +/* ------------------------------------------------------------------ + * loca section + ------------------------------------------------------------------ */ + +struct COMP_KMXPLUS_LOCA_ENTRY { + KMXPLUS_STR locale; // 0010+ locale string entry +}; + +struct COMP_KMXPLUS_LOCA { + static const KMX_DWORD IDENT = LDML_SECTIONID_LOCA; + COMP_KMXPLUS_HEADER header; + KMX_DWORD count; // 0008 number of locales + KMX_DWORD reserved; + COMP_KMXPLUS_LOCA_ENTRY entries[]; + /** + * @brief True if section is valid. + */ + bool valid(KMX_DWORD length) const; +}; + +static_assert(sizeof(struct COMP_KMXPLUS_LOCA) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); +static_assert(sizeof(struct COMP_KMXPLUS_LOCA) == LDML_LENGTH_LOCA, "mismatched size of section loca"); + +/* ------------------------------------------------------------------ + * meta section + ------------------------------------------------------------------ */ + +struct COMP_KMXPLUS_META { + static const KMX_DWORD IDENT = LDML_SECTIONID_META; + COMP_KMXPLUS_HEADER header; + KMXPLUS_STR name; + KMXPLUS_STR author; + KMXPLUS_STR conform; + KMXPLUS_STR layout; + KMXPLUS_STR normalization; + KMXPLUS_STR indicator; + KMXPLUS_STR version; + KMX_DWORD settings; + /** + * @brief True if section is valid. + */ + bool valid(KMX_DWORD length) const; +}; + +static_assert(sizeof(struct COMP_KMXPLUS_META) == LDML_LENGTH_META, "mismatched size of section meta"); + +/* ------------------------------------------------------------------ + * name section + ------------------------------------------------------------------ */ + +struct COMP_KMXPLUS_NAME_ENTRY { + KMXPLUS_STR name; +}; + +struct COMP_KMXPLUS_NAME { + static const KMX_DWORD IDENT = LDML_SECTIONID_NAME; + COMP_KMXPLUS_HEADER header; + KMX_DWORD count; + KMX_DWORD reserved; + COMP_KMXPLUS_NAME_ENTRY entries[]; + /** + * @brief True if section is valid. + */ + bool valid(KMX_DWORD length) const; +}; + +static_assert(sizeof(struct COMP_KMXPLUS_NAME) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); +static_assert(sizeof(struct COMP_KMXPLUS_NAME) == LDML_LENGTH_NAME, "mismatched size of section name"); + /* ------------------------------------------------------------------ * strs section ------------------------------------------------------------------ */ @@ -112,78 +207,6 @@ struct COMP_KMXPLUS_STRS { static_assert(sizeof(struct COMP_KMXPLUS_STRS) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); static_assert(sizeof(struct COMP_KMXPLUS_STRS) == LDML_LENGTH_STRS, "mismatched size of section strs"); -/* ------------------------------------------------------------------ - * meta section - ------------------------------------------------------------------ */ - -struct COMP_KMXPLUS_META { - static const KMX_DWORD IDENT = LDML_SECTIONID_META; - COMP_KMXPLUS_HEADER header; - KMXPLUS_STR name; - KMXPLUS_STR author; - KMXPLUS_STR conform; - KMXPLUS_STR layout; - KMXPLUS_STR normalization; - KMXPLUS_STR indicator; - KMXPLUS_STR version; - KMX_DWORD settings; - /** - * @brief True if section is valid. - */ - bool valid(KMX_DWORD length) const; -}; - -static_assert(sizeof(struct COMP_KMXPLUS_META) == LDML_LENGTH_META, "mismatched size of section meta"); - -/* ------------------------------------------------------------------ - * loca section - ------------------------------------------------------------------ */ - -struct COMP_KMXPLUS_LOCA_ENTRY { - KMXPLUS_STR locale; // 0010+ locale string entry -}; - -struct COMP_KMXPLUS_LOCA { - static const KMX_DWORD IDENT = LDML_SECTIONID_LOCA; - COMP_KMXPLUS_HEADER header; - KMX_DWORD count; // 0008 number of locales - KMX_DWORD reserved; - COMP_KMXPLUS_LOCA_ENTRY entries[]; - /** - * @brief True if section is valid. - */ - bool valid(KMX_DWORD length) const; -}; - -static_assert(sizeof(struct COMP_KMXPLUS_LOCA) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); -static_assert(sizeof(struct COMP_KMXPLUS_LOCA) == LDML_LENGTH_LOCA, "mismatched size of section loca"); - -/* ------------------------------------------------------------------ - * keys section - ------------------------------------------------------------------ */ - -struct COMP_KMXPLUS_KEYS_ENTRY { - KMX_DWORD vkey; - KMX_DWORD mod; - KMX_DWORD to; // to may be KMXPLUS_STR or UTF32 char - KMX_DWORD flags; -}; - -struct COMP_KMXPLUS_KEYS { - static const KMX_DWORD IDENT = LDML_SECTIONID_KEYS; - COMP_KMXPLUS_HEADER header; - KMX_DWORD count; // number of keys - KMX_DWORD reserved; // padding - COMP_KMXPLUS_KEYS_ENTRY entries[]; - /** - * @brief True if section is valid. - */ - bool valid(KMX_DWORD length) const; -}; - -static_assert(sizeof(struct COMP_KMXPLUS_KEYS) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); -static_assert(sizeof(struct COMP_KMXPLUS_KEYS) == LDML_LENGTH_KEYS, "mismatched size of section keys"); - /* ------------------------------------------------------------------ * vkey section ------------------------------------------------------------------ */ @@ -208,30 +231,6 @@ struct COMP_KMXPLUS_VKEY { static_assert(sizeof(struct COMP_KMXPLUS_VKEY) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); static_assert(sizeof(struct COMP_KMXPLUS_VKEY) == LDML_LENGTH_VKEY, "mismatched size of section vkey"); -/* ------------------------------------------------------------------ - * name section - ------------------------------------------------------------------ */ - -struct COMP_KMXPLUS_NAME_ENTRY { - KMXPLUS_STR name; -}; - -struct COMP_KMXPLUS_NAME { - static const KMX_DWORD IDENT = LDML_SECTIONID_NAME; - COMP_KMXPLUS_HEADER header; - KMX_DWORD count; - KMX_DWORD reserved; - COMP_KMXPLUS_NAME_ENTRY entries[]; - /** - * @brief True if section is valid. - */ - bool valid(KMX_DWORD length) const; -}; - -static_assert(sizeof(struct COMP_KMXPLUS_NAME) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); -static_assert(sizeof(struct COMP_KMXPLUS_NAME) == LDML_LENGTH_NAME, "mismatched size of section name"); - - /** * @brief helper accessor object for * diff --git a/core/src/ldml/C7043_ldml.md b/core/src/ldml/C7043_ldml.md index 7ff4fe9dcd..0e69de493c 100644 --- a/core/src/ldml/C7043_ldml.md +++ b/core/src/ldml/C7043_ldml.md @@ -62,7 +62,108 @@ Then for `count` repetitions: This list is in sorted order based on the `sect` identifier. -### C7043.2.2 `strs`—Strings +### C7043.2.2 `keys`—Keybag + +| ∆ | Bits | Name | Description | +|---|------|-----------|------------------------------------------| +| 0 | 32 | ident | `keys` | +| 4 | 32 | size | int: Length of section | +| 8 | 32 | count | int: Number of keys | +|12 | 32 | reserved | reserved | + +The keys are sorted in binary order based on the `vkey` and `mod` fields. + +For each key: + +| ∆ | Bits | Name | Description | +|---|------|---------|------------------------------------------| +|16+| 32 | vkey | int: vkey ID | +|20+| 32 | mod | int: modifier key flags | +|24+| 32 | to | str: output string OR UTF-32LE codepoint | +|28+| 32 | flags | int: per-key flags | + +- `vkey`: If this is 0-255, it is the resolved standard/predefined vkey (K_A, + etc.). It is resolved because the `vkeyMap` from LDML has already been + applied. If this is 256 or above, it is a custom touch layout vkey generated + by the compiler. +- `mod`: TODO define this. 0 for no modifiers. +- `flags`: Flags is a 32-bit bitfield defined as below: + +| Bit position | Meaning | Description | +|--------------|----------|---------------------------------------------| +| 0 | extend | 0: `to` is a char, 1: `to` is a string | + +- `to`: If `extend` is 0, `to` is a UTF-32LE codepoint. If `extend` is 1, `to` + is a 32 bit index into the `strs` table. The string may be zero-length. + +### C7043.2.3 `loca`—Locales + +| ∆ | Bits | Name | Description | +|---|------|---------|------------------------------------------| +| 0 | 32 | ident | `loca` | +| 4 | 32 | size | int: Length of section | +| 8 | 32 | count | int: Number of locales | +|12 | 32 | reserved| padding | + +`count` is always ≥1, because a keyboard always has a primary locale identifier. + +For each locale ID in `count` + +| ∆ | Bits | Name | Description | +|---|------|---------|------------------------------------------| +|16+| 32 | locale | str: Locale ID in BCP47 format | + +The first locale ID is always the primary locale identifier. The rest of the +locale IDs (starting at offset 16) are in sorted binary order. + +### C7043.2.4 `meta`—Metadata + +| ∆ | Bits | Name | Description | +|---|------|---------------|-------------------------------------| +| 0 | 32 | ident | `meta` | +| 4 | 32 | size | int: Length of section | +| 8 | 32 | name | str: Keyboard name | +|12 | 32 | author | str: Keyboard author | +|16 | 32 | conform | str: CLDR 'conformsTo' version | +|20 | 32 | layout | str: layout type | +|24 | 32 | normalization | str: normalization mode | +|28 | 32 | indicator | str: indicator | +|32 | 32 | settings | int: keyboard settings | + +The `settings` is a 32-bit bitfield as below: + +| Bit position | Meaning | Description | +|--------------|------------------|------------------------------| +| 0 | fallback | fallback=omit | +| 1 | transformFailure | transformFailure=omit | +| 2 | transformPartial | transformPartial=hide | + +### C7043.2.5 `name`—Names + +Defines the names of the keyboard as found in the source `` element. +While this section is optional in the binary format, in practice it will always +be present, as the source format requires at least one name. + +| ∆ | Bits | Name | Description | +|---|------|---------|------------------------------------------| +| 0 | 32 | ident | `name` | +| 4 | 32 | size | int: Length of section | +| 8 | 32 | count | int: Number of names | +|12 | 32 | reserved| padding | + +Note that `count` is always ≥1, as the source format requires at least one name. + +For each name in `count`: + +| ∆ | Bits | Name | Description | +|---|------|---------|------------------------------------------| +|16+| 32 | name | str: A name for the keyboard | + +Note that the first name is repeated in the `meta` section. The remaining names +are stored in source file order, and the semantic meaning of each name is not +defined here. + +### C7043.2.6 `strs`—Strings All strings are stored in the Strings section. @@ -97,85 +198,7 @@ A distinction between zero-length string and optional should be avoided (e.g. the difference between "" and null in Javascript). If this is truly required, a separate flag field must be used to denote the difference. -### C7043.2.3 `meta`—Metadata - -| ∆ | Bits | Name | Description | -|---|------|---------------|-------------------------------------| -| 0 | 32 | ident | `meta` | -| 4 | 32 | size | int: Length of section | -| 8 | 32 | name | str: Keyboard name | -|12 | 32 | author | str: Keyboard author | -|16 | 32 | conform | str: CLDR 'conformsTo' version | -|20 | 32 | layout | str: layout type | -|24 | 32 | normalization | str: normalization mode | -|28 | 32 | indicator | str: indicator | -|32 | 32 | settings | int: keyboard settings | - -The `settings` is a 32-bit bitfield as below: - -| Bit position | Meaning | Description | -|--------------|------------------|------------------------------| -| 0 | fallback | fallback=omit | -| 1 | transformFailure | transformFailure=omit | -| 2 | transformPartial | transformPartial=hide | - - -### C7043.2.4 `loca`—Locales - - -| ∆ | Bits | Name | Description | -|---|------|---------|------------------------------------------| -| 0 | 32 | ident | `loca` | -| 4 | 32 | size | int: Length of section | -| 8 | 32 | count | int: Number of locales | -|12 | 32 | reserved| padding | - -`count` is always ≥1, because a keyboard always has a primary locale identifier. - -For each locale ID in `count` - -| ∆ | Bits | Name | Description | -|---|------|---------|------------------------------------------| -|16+| 32 | locale | str: Locale ID in BCP47 format | - -The first locale ID is always the primary locale identifier. The rest of the -locale IDs (starting at offset 16) are in sorted binary order. - -### C7043.2.5 `keys`—Keybag - -| ∆ | Bits | Name | Description | -|---|------|---------|------------------------------------------| -| 0 | 32 | ident | `keys` | -| 4 | 32 | size | int: Length of section | -| 8 | 32 | count | int: Number of keys | -|12 | 32 | reserved | reserved | - -The keys are sorted in binary order based on the `vkey` and `mod` fields. - -For each key: - -| ∆ | Bits | Name | Description | -|---|------|---------|------------------------------------------| -|16+| 32 | vkey | int: vkey ID | -|20+| 32 | mod | int: modifier key flags | -|24+| 32 | to | str: output string OR UTF-32LE codepoint | -|28+| 32 | flags | int: per-key flags | - -- `vkey`: If this is 0-255, it is the resolved standard/predefined vkey (K_A, - etc.). It is resolved because the `vkeyMap` from LDML has already been - applied. If this is 256 or above, it is a custom touch layout vkey generated - by the compiler. -- `mod`: TODO define this. 0 for no modifiers. -- `flags`: Flags is a 32-bit bitfield defined as below: - -| Bit position | Meaning | Description | -|--------------|----------|---------------------------------------------| -| 0 | extend | 0: `to` is a char, 1: `to` is a string | - -- `to`: If `extend` is 0, `to` is a UTF-32LE codepoint. If `extend` is 1, `to` - is a 32 bit index into the `strs` table. The string may be zero-length. - -### C7043.2.6 `vkey`—VKey Map +### C7043.2.7 `vkey`—VKey Map | ∆ | Bits | Name | Description | |---|------|---------|------------------------------------------| @@ -196,31 +219,6 @@ For each key: - `vkey`: Is the standard vkey, 0-255 - `target`: Is the target (resolved) vkey, 0-255. -### C7043.2.7 `name`—Names - -Defines the names of the keyboard as found in the source `` element. -While this section is optional in the binary format, in practice it will always -be present, as the source format requires at least one name. - -| ∆ | Bits | Name | Description | -|---|------|---------|------------------------------------------| -| 0 | 32 | ident | `name` | -| 4 | 32 | size | int: Length of section | -| 8 | 32 | count | int: Number of names | -|12 | 32 | reserved| padding | - -Note that `count` is always ≥1, as the source format requires at least one name. - -For each name in `count`: - -| ∆ | Bits | Name | Description | -|---|------|---------|------------------------------------------| -|16+| 32 | name | str: A name for the keyboard | - -Note that the first name is repeated in the `meta` section. The remaining names -are stored in source file order, and the semantic meaning of each name is not -defined here. - ### C7043.2.8 Transforms and friends > TODO: transforms diff --git a/core/tests/unit/kmx/kmx_imsample.kmx b/core/tests/unit/kmx/kmx_imsample.kmx index 754c0985ae2bf73e99d134f34a1254319968f309..4940a3c847ddb59053c5888dbed0d53c6080d66b 100644 GIT binary patch delta 361 zcmYk0%}N4c6o%h%U=W%zv@{WPBS8{j1}^-8Qdb$K7$UTh5NRPUBHX$c2}HUB*KX_v z1lQd`HxVt`=`-UjJn-=Te21CyZ~a+zwf)qTCHWtJLR0%}H1SCrMtk#a?!qaOG^8~e zh7|rBBKRD734VM9Ex;juh_*ccg?@Pc7yb5pnt$4GEX7sF-0yws{SV; diff --git a/core/tests/unit/kmx/kmx_key_list.kmx b/core/tests/unit/kmx/kmx_key_list.kmx index efb6540fcec6057653b03180095b2da757b0c222..63c211f4b0b0ee829b9c8b3d9fe4830640089ac7 100644 GIT binary patch literal 802 zcmZvaxk>|J6opTWf`}Uu7ox~uVHsi+w`xd)6funp*k%zDP$QGz(ut^8SXfx3@)4}9 zgO8vf7Gfb5zC`@)WR#%(@cs9kbN`KGGFuz#>XEAFPYp3bHZh141{WR(n+)X7n6f5g z7{=(|Q%7JD9feVJjOsu)dYKxBLG&7RHP9t$KF~*0FVGj%(?H)+>kwgYgL(%kdD>P^ zHRhrm9jc5jp)w@L+NHOLTfrT`jw;%xdSvE3RoPMT*UV^MF~wn4314v$-bVXr=4bqM z)|>uyPOB%SS>Y(l{bf$Kbcs1lb;!gGbG}dD^Qc8gYLm5P6?jpR*V(QG=Io!d{(EVp zgWnx`!A2*9;09k0GRy5Rv;2eXvn@Aid9&Fi)9r8PR1&$;9xV?N*uEgE;ZGom?+3?~ z&hVFz!ViE;c7?x&9Nz8;`*+!U@&VQ`7^c8h&$B74KX kgPqelxn1YUqJBpj;_4{a?s0HkX#(#+jyDzjb@o3|J6opTWf{4ZxZbXqm#8Q&r3Kl~mqzJJwf^8PTh#HA8E_I3&78Zgc7CwTd zPasy-7Gfb5V&N;q?@mSu>JQ)CbI$!YW+uI!T~&`1db~IgBVZ2(k%HjD17VYa{25c$ zWDG)t{vCA)9P}6rqodRb=teJ5;}Aw?sVlzTq85F9Laq9`L9P4xhI$19ti7k+LQ)lN zD5omVq8uHngx*A@NshHcZx^?WtHHL)+M`y;%vZF}in2E+rddrZ3USRT=4p;1vC+Pt z`ItA;dQ-o4YW1WPGaO~OL5ck>UF4afI%K>D&3HbJ&!ZL~p*7}~RNziU?q{p!d1n6X z_5WQ*I(XL*1{<9afJ^+fpIL67&GHYj&$is8<;`Z7Ot;@&>LrmYRcO^9j_vcaBYYha z_z*a*)WDxY65kIl*%|&Ea(Fu@=$&Qv$@^HR09L=T|I5Y`E1w~O_Wm