diff --git a/core/include/ldml/keyboardprocessor_ldml.ts b/core/include/ldml/keyboardprocessor_ldml.ts index ecb332f618..ec6dc4c4fc 100644 --- a/core/include/ldml/keyboardprocessor_ldml.ts +++ b/core/include/ldml/keyboardprocessor_ldml.ts @@ -23,11 +23,11 @@ * through. */ export type SectionIdent = + 'sect' | 'keys' | 'loca' | 'meta' | 'name' | - 'sect' | 'strs' | 'vkey'; @@ -170,11 +170,11 @@ export const constants: Constants = { * All section IDs. */ section: { + sect: 'sect', keys: 'keys', loca: 'loca', meta: 'meta', name: 'name', - sect: 'sect', strs: 'strs', vkey: 'vkey', }, diff --git a/core/src/kmx/kmx_plus.h b/core/src/kmx/kmx_plus.h index 54c04d71aa..27d0159415 100644 --- a/core/src/kmx/kmx_plus.h +++ b/core/src/kmx/kmx_plus.h @@ -78,6 +78,101 @@ struct COMP_KMXPLUS_SECT { static_assert(sizeof(struct COMP_KMXPLUS_SECT) == LDML_LENGTH_SECT, "mismatched size of section sect"); static_assert(sizeof(struct COMP_KMXPLUS_SECT) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); +/* ------------------------------------------------------------------ + * keys section + ------------------------------------------------------------------ */ + +struct COMP_KMXPLUS_KEYS_ENTRY { + KMX_DWORD vkey; + KMX_DWORD mod; + KMX_DWORD to; // to may be KMXPLUS_STR or UTF32 char + KMX_DWORD flags; +}; + +struct COMP_KMXPLUS_KEYS { + static const KMX_DWORD IDENT = LDML_SECTIONID_KEYS; + COMP_KMXPLUS_HEADER header; + KMX_DWORD count; // number of keys + KMX_DWORD reserved; // padding + COMP_KMXPLUS_KEYS_ENTRY entries[]; + /** + * @brief True if section is valid. + */ + bool valid(KMX_DWORD length) const; +}; + +static_assert(sizeof(struct COMP_KMXPLUS_KEYS) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); +static_assert(sizeof(struct COMP_KMXPLUS_KEYS) == LDML_LENGTH_KEYS, "mismatched size of section keys"); + +/* ------------------------------------------------------------------ + * loca section + ------------------------------------------------------------------ */ + +struct COMP_KMXPLUS_LOCA_ENTRY { + KMXPLUS_STR locale; // 0010+ locale string entry +}; + +struct COMP_KMXPLUS_LOCA { + static const KMX_DWORD IDENT = LDML_SECTIONID_LOCA; + COMP_KMXPLUS_HEADER header; + KMX_DWORD count; // 0008 number of locales + KMX_DWORD reserved; + COMP_KMXPLUS_LOCA_ENTRY entries[]; + /** + * @brief True if section is valid. + */ + bool valid(KMX_DWORD length) const; +}; + +static_assert(sizeof(struct COMP_KMXPLUS_LOCA) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); +static_assert(sizeof(struct COMP_KMXPLUS_LOCA) == LDML_LENGTH_LOCA, "mismatched size of section loca"); + +/* ------------------------------------------------------------------ + * meta section + ------------------------------------------------------------------ */ + +struct COMP_KMXPLUS_META { + static const KMX_DWORD IDENT = LDML_SECTIONID_META; + COMP_KMXPLUS_HEADER header; + KMXPLUS_STR name; + KMXPLUS_STR author; + KMXPLUS_STR conform; + KMXPLUS_STR layout; + KMXPLUS_STR normalization; + KMXPLUS_STR indicator; + KMXPLUS_STR version; + KMX_DWORD settings; + /** + * @brief True if section is valid. + */ + bool valid(KMX_DWORD length) const; +}; + +static_assert(sizeof(struct COMP_KMXPLUS_META) == LDML_LENGTH_META, "mismatched size of section meta"); + +/* ------------------------------------------------------------------ + * name section + ------------------------------------------------------------------ */ + +struct COMP_KMXPLUS_NAME_ENTRY { + KMXPLUS_STR name; +}; + +struct COMP_KMXPLUS_NAME { + static const KMX_DWORD IDENT = LDML_SECTIONID_NAME; + COMP_KMXPLUS_HEADER header; + KMX_DWORD count; + KMX_DWORD reserved; + COMP_KMXPLUS_NAME_ENTRY entries[]; + /** + * @brief True if section is valid. + */ + bool valid(KMX_DWORD length) const; +}; + +static_assert(sizeof(struct COMP_KMXPLUS_NAME) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); +static_assert(sizeof(struct COMP_KMXPLUS_NAME) == LDML_LENGTH_NAME, "mismatched size of section name"); + /* ------------------------------------------------------------------ * strs section ------------------------------------------------------------------ */ @@ -112,78 +207,6 @@ struct COMP_KMXPLUS_STRS { static_assert(sizeof(struct COMP_KMXPLUS_STRS) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); static_assert(sizeof(struct COMP_KMXPLUS_STRS) == LDML_LENGTH_STRS, "mismatched size of section strs"); -/* ------------------------------------------------------------------ - * meta section - ------------------------------------------------------------------ */ - -struct COMP_KMXPLUS_META { - static const KMX_DWORD IDENT = LDML_SECTIONID_META; - COMP_KMXPLUS_HEADER header; - KMXPLUS_STR name; - KMXPLUS_STR author; - KMXPLUS_STR conform; - KMXPLUS_STR layout; - KMXPLUS_STR normalization; - KMXPLUS_STR indicator; - KMXPLUS_STR version; - KMX_DWORD settings; - /** - * @brief True if section is valid. - */ - bool valid(KMX_DWORD length) const; -}; - -static_assert(sizeof(struct COMP_KMXPLUS_META) == LDML_LENGTH_META, "mismatched size of section meta"); - -/* ------------------------------------------------------------------ - * loca section - ------------------------------------------------------------------ */ - -struct COMP_KMXPLUS_LOCA_ENTRY { - KMXPLUS_STR locale; // 0010+ locale string entry -}; - -struct COMP_KMXPLUS_LOCA { - static const KMX_DWORD IDENT = LDML_SECTIONID_LOCA; - COMP_KMXPLUS_HEADER header; - KMX_DWORD count; // 0008 number of locales - KMX_DWORD reserved; - COMP_KMXPLUS_LOCA_ENTRY entries[]; - /** - * @brief True if section is valid. - */ - bool valid(KMX_DWORD length) const; -}; - -static_assert(sizeof(struct COMP_KMXPLUS_LOCA) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); -static_assert(sizeof(struct COMP_KMXPLUS_LOCA) == LDML_LENGTH_LOCA, "mismatched size of section loca"); - -/* ------------------------------------------------------------------ - * keys section - ------------------------------------------------------------------ */ - -struct COMP_KMXPLUS_KEYS_ENTRY { - KMX_DWORD vkey; - KMX_DWORD mod; - KMX_DWORD to; // to may be KMXPLUS_STR or UTF32 char - KMX_DWORD flags; -}; - -struct COMP_KMXPLUS_KEYS { - static const KMX_DWORD IDENT = LDML_SECTIONID_KEYS; - COMP_KMXPLUS_HEADER header; - KMX_DWORD count; // number of keys - KMX_DWORD reserved; // padding - COMP_KMXPLUS_KEYS_ENTRY entries[]; - /** - * @brief True if section is valid. - */ - bool valid(KMX_DWORD length) const; -}; - -static_assert(sizeof(struct COMP_KMXPLUS_KEYS) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); -static_assert(sizeof(struct COMP_KMXPLUS_KEYS) == LDML_LENGTH_KEYS, "mismatched size of section keys"); - /* ------------------------------------------------------------------ * vkey section ------------------------------------------------------------------ */ @@ -208,30 +231,6 @@ struct COMP_KMXPLUS_VKEY { static_assert(sizeof(struct COMP_KMXPLUS_VKEY) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); static_assert(sizeof(struct COMP_KMXPLUS_VKEY) == LDML_LENGTH_VKEY, "mismatched size of section vkey"); -/* ------------------------------------------------------------------ - * name section - ------------------------------------------------------------------ */ - -struct COMP_KMXPLUS_NAME_ENTRY { - KMXPLUS_STR name; -}; - -struct COMP_KMXPLUS_NAME { - static const KMX_DWORD IDENT = LDML_SECTIONID_NAME; - COMP_KMXPLUS_HEADER header; - KMX_DWORD count; - KMX_DWORD reserved; - COMP_KMXPLUS_NAME_ENTRY entries[]; - /** - * @brief True if section is valid. - */ - bool valid(KMX_DWORD length) const; -}; - -static_assert(sizeof(struct COMP_KMXPLUS_NAME) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); -static_assert(sizeof(struct COMP_KMXPLUS_NAME) == LDML_LENGTH_NAME, "mismatched size of section name"); - - /** * @brief helper accessor object for * diff --git a/core/src/ldml/C7043_ldml.md b/core/src/ldml/C7043_ldml.md index 7ff4fe9dcd..0e69de493c 100644 --- a/core/src/ldml/C7043_ldml.md +++ b/core/src/ldml/C7043_ldml.md @@ -62,7 +62,108 @@ Then for `count` repetitions: This list is in sorted order based on the `sect` identifier. -### C7043.2.2 `strs`—Strings +### C7043.2.2 `keys`—Keybag + +| ∆ | Bits | Name | Description | +|---|------|-----------|------------------------------------------| +| 0 | 32 | ident | `keys` | +| 4 | 32 | size | int: Length of section | +| 8 | 32 | count | int: Number of keys | +|12 | 32 | reserved | reserved | + +The keys are sorted in binary order based on the `vkey` and `mod` fields. + +For each key: + +| ∆ | Bits | Name | Description | +|---|------|---------|------------------------------------------| +|16+| 32 | vkey | int: vkey ID | +|20+| 32 | mod | int: modifier key flags | +|24+| 32 | to | str: output string OR UTF-32LE codepoint | +|28+| 32 | flags | int: per-key flags | + +- `vkey`: If this is 0-255, it is the resolved standard/predefined vkey (K_A, + etc.). It is resolved because the `vkeyMap` from LDML has already been + applied. If this is 256 or above, it is a custom touch layout vkey generated + by the compiler. +- `mod`: TODO define this. 0 for no modifiers. +- `flags`: Flags is a 32-bit bitfield defined as below: + +| Bit position | Meaning | Description | +|--------------|----------|---------------------------------------------| +| 0 | extend | 0: `to` is a char, 1: `to` is a string | + +- `to`: If `extend` is 0, `to` is a UTF-32LE codepoint. If `extend` is 1, `to` + is a 32 bit index into the `strs` table. The string may be zero-length. + +### C7043.2.3 `loca`—Locales + +| ∆ | Bits | Name | Description | +|---|------|---------|------------------------------------------| +| 0 | 32 | ident | `loca` | +| 4 | 32 | size | int: Length of section | +| 8 | 32 | count | int: Number of locales | +|12 | 32 | reserved| padding | + +`count` is always ≥1, because a keyboard always has a primary locale identifier. + +For each locale ID in `count` + +| ∆ | Bits | Name | Description | +|---|------|---------|------------------------------------------| +|16+| 32 | locale | str: Locale ID in BCP47 format | + +The first locale ID is always the primary locale identifier. The rest of the +locale IDs (starting at offset 16) are in sorted binary order. + +### C7043.2.4 `meta`—Metadata + +| ∆ | Bits | Name | Description | +|---|------|---------------|-------------------------------------| +| 0 | 32 | ident | `meta` | +| 4 | 32 | size | int: Length of section | +| 8 | 32 | name | str: Keyboard name | +|12 | 32 | author | str: Keyboard author | +|16 | 32 | conform | str: CLDR 'conformsTo' version | +|20 | 32 | layout | str: layout type | +|24 | 32 | normalization | str: normalization mode | +|28 | 32 | indicator | str: indicator | +|32 | 32 | settings | int: keyboard settings | + +The `settings` is a 32-bit bitfield as below: + +| Bit position | Meaning | Description | +|--------------|------------------|------------------------------| +| 0 | fallback | fallback=omit | +| 1 | transformFailure | transformFailure=omit | +| 2 | transformPartial | transformPartial=hide | + +### C7043.2.5 `name`—Names + +Defines the names of the keyboard as found in the source `` element. +While this section is optional in the binary format, in practice it will always +be present, as the source format requires at least one name. + +| ∆ | Bits | Name | Description | +|---|------|---------|------------------------------------------| +| 0 | 32 | ident | `name` | +| 4 | 32 | size | int: Length of section | +| 8 | 32 | count | int: Number of names | +|12 | 32 | reserved| padding | + +Note that `count` is always ≥1, as the source format requires at least one name. + +For each name in `count`: + +| ∆ | Bits | Name | Description | +|---|------|---------|------------------------------------------| +|16+| 32 | name | str: A name for the keyboard | + +Note that the first name is repeated in the `meta` section. The remaining names +are stored in source file order, and the semantic meaning of each name is not +defined here. + +### C7043.2.6 `strs`—Strings All strings are stored in the Strings section. @@ -97,85 +198,7 @@ A distinction between zero-length string and optional should be avoided (e.g. the difference between "" and null in Javascript). If this is truly required, a separate flag field must be used to denote the difference. -### C7043.2.3 `meta`—Metadata - -| ∆ | Bits | Name | Description | -|---|------|---------------|-------------------------------------| -| 0 | 32 | ident | `meta` | -| 4 | 32 | size | int: Length of section | -| 8 | 32 | name | str: Keyboard name | -|12 | 32 | author | str: Keyboard author | -|16 | 32 | conform | str: CLDR 'conformsTo' version | -|20 | 32 | layout | str: layout type | -|24 | 32 | normalization | str: normalization mode | -|28 | 32 | indicator | str: indicator | -|32 | 32 | settings | int: keyboard settings | - -The `settings` is a 32-bit bitfield as below: - -| Bit position | Meaning | Description | -|--------------|------------------|------------------------------| -| 0 | fallback | fallback=omit | -| 1 | transformFailure | transformFailure=omit | -| 2 | transformPartial | transformPartial=hide | - - -### C7043.2.4 `loca`—Locales - - -| ∆ | Bits | Name | Description | -|---|------|---------|------------------------------------------| -| 0 | 32 | ident | `loca` | -| 4 | 32 | size | int: Length of section | -| 8 | 32 | count | int: Number of locales | -|12 | 32 | reserved| padding | - -`count` is always ≥1, because a keyboard always has a primary locale identifier. - -For each locale ID in `count` - -| ∆ | Bits | Name | Description | -|---|------|---------|------------------------------------------| -|16+| 32 | locale | str: Locale ID in BCP47 format | - -The first locale ID is always the primary locale identifier. The rest of the -locale IDs (starting at offset 16) are in sorted binary order. - -### C7043.2.5 `keys`—Keybag - -| ∆ | Bits | Name | Description | -|---|------|---------|------------------------------------------| -| 0 | 32 | ident | `keys` | -| 4 | 32 | size | int: Length of section | -| 8 | 32 | count | int: Number of keys | -|12 | 32 | reserved | reserved | - -The keys are sorted in binary order based on the `vkey` and `mod` fields. - -For each key: - -| ∆ | Bits | Name | Description | -|---|------|---------|------------------------------------------| -|16+| 32 | vkey | int: vkey ID | -|20+| 32 | mod | int: modifier key flags | -|24+| 32 | to | str: output string OR UTF-32LE codepoint | -|28+| 32 | flags | int: per-key flags | - -- `vkey`: If this is 0-255, it is the resolved standard/predefined vkey (K_A, - etc.). It is resolved because the `vkeyMap` from LDML has already been - applied. If this is 256 or above, it is a custom touch layout vkey generated - by the compiler. -- `mod`: TODO define this. 0 for no modifiers. -- `flags`: Flags is a 32-bit bitfield defined as below: - -| Bit position | Meaning | Description | -|--------------|----------|---------------------------------------------| -| 0 | extend | 0: `to` is a char, 1: `to` is a string | - -- `to`: If `extend` is 0, `to` is a UTF-32LE codepoint. If `extend` is 1, `to` - is a 32 bit index into the `strs` table. The string may be zero-length. - -### C7043.2.6 `vkey`—VKey Map +### C7043.2.7 `vkey`—VKey Map | ∆ | Bits | Name | Description | |---|------|---------|------------------------------------------| @@ -196,31 +219,6 @@ For each key: - `vkey`: Is the standard vkey, 0-255 - `target`: Is the target (resolved) vkey, 0-255. -### C7043.2.7 `name`—Names - -Defines the names of the keyboard as found in the source `` element. -While this section is optional in the binary format, in practice it will always -be present, as the source format requires at least one name. - -| ∆ | Bits | Name | Description | -|---|------|---------|------------------------------------------| -| 0 | 32 | ident | `name` | -| 4 | 32 | size | int: Length of section | -| 8 | 32 | count | int: Number of names | -|12 | 32 | reserved| padding | - -Note that `count` is always ≥1, as the source format requires at least one name. - -For each name in `count`: - -| ∆ | Bits | Name | Description | -|---|------|---------|------------------------------------------| -|16+| 32 | name | str: A name for the keyboard | - -Note that the first name is repeated in the `meta` section. The remaining names -are stored in source file order, and the semantic meaning of each name is not -defined here. - ### C7043.2.8 Transforms and friends > TODO: transforms diff --git a/core/tests/unit/kmx/kmx_imsample.kmx b/core/tests/unit/kmx/kmx_imsample.kmx index 754c0985ae..4940a3c847 100644 Binary files a/core/tests/unit/kmx/kmx_imsample.kmx and b/core/tests/unit/kmx/kmx_imsample.kmx differ diff --git a/core/tests/unit/kmx/kmx_key_list.kmx b/core/tests/unit/kmx/kmx_key_list.kmx index efb6540fce..63c211f4b0 100644 Binary files a/core/tests/unit/kmx/kmx_key_list.kmx and b/core/tests/unit/kmx/kmx_key_list.kmx differ diff --git a/developer/src/kmldmlc/src/keyman/compiler/compiler.ts b/developer/src/kmldmlc/src/keyman/compiler/compiler.ts index 811b04c11f..643bc86442 100644 --- a/developer/src/kmldmlc/src/keyman/compiler/compiler.ts +++ b/developer/src/kmldmlc/src/keyman/compiler/compiler.ts @@ -9,9 +9,9 @@ import { MetaCompiler } from './meta'; import { NameCompiler } from './name'; const SECTION_COMPILERS = [ - MetaCompiler, - LocaCompiler, KeysCompiler, + LocaCompiler, + MetaCompiler, NameCompiler ]; diff --git a/developer/src/kmldmlc/src/keyman/kmx/kmx-plus-builder.ts b/developer/src/kmldmlc/src/keyman/kmx/kmx-plus-builder.ts index d6c7b477f9..3bf52380fe 100644 --- a/developer/src/kmldmlc/src/keyman/kmx/kmx-plus-builder.ts +++ b/developer/src/kmldmlc/src/keyman/kmx/kmx-plus-builder.ts @@ -8,32 +8,64 @@ interface BUILDER_SECTION { _offset: number; // used only for building the output }; +/* ------------------------------------------------------------------ + * sect section + ------------------------------------------------------------------ */ + interface BUILDER_SECT_ITEM { sect: number; offset: number; //? new r.VoidPointer(r.uint32le, {type: 'global'}) }; -// Builder for the 'sect' (Section table of contents) section +/** + * Builder for the 'sect' (Section table of contents) section + */ interface BUILDER_SECT extends BUILDER_SECTION { total: number; count: number; items: BUILDER_SECT_ITEM[]; }; -interface BUILDER_STRS_ITEM { - // While we use length which is number of utf-16 code units excluding null terminator, - // we always write a null terminator, so we can get restructure to do that for us here - offset: number; //? new r.Pointer(r.uint32le, new r.String(null, 'utf16le')), - length: number; // in UTF-16 code units - _value: string +/* ------------------------------------------------------------------ + * keys section + ------------------------------------------------------------------ */ + +interface BUILDER_KEYS_ITEM { + vkey: number; + mod: number; + to: number; //str or UTF-32 char depending on value of 'extend' + flags: number; //bitfield }; -interface BUILDER_STRS extends BUILDER_SECTION { +/** + * Builder for the 'keys' section + */ +interface BUILDER_KEYS extends BUILDER_SECTION { count: number; reserved: number; - items: BUILDER_STRS_ITEM[]; + items: BUILDER_KEYS_ITEM[]; }; +/* ------------------------------------------------------------------ + * loca section + ------------------------------------------------------------------ */ + +/** + * Builder for the 'loca' section + */ +interface BUILDER_LOCA extends BUILDER_SECTION { + count: number; + reserved: number; + items: number[]; //str[] +}; + +/* ------------------------------------------------------------------ + * meta section + ------------------------------------------------------------------ */ + +/** + * Builder for the 'meta' section + */ interface BUILDER_META extends BUILDER_SECTION { name: number; //str author: number; //str @@ -45,31 +77,47 @@ interface BUILDER_META extends BUILDER_SECTION { settings: number; //bitfield }; +/* ------------------------------------------------------------------ + * name section + ------------------------------------------------------------------ */ + +/** + * Builder for the 'name' section + */ interface BUILDER_NAME extends BUILDER_SECTION { count: number; reserved: number; items: number[]; //str[] }; -interface BUILDER_LOCA extends BUILDER_SECTION { +/* ------------------------------------------------------------------ + * strs section + ------------------------------------------------------------------ */ + +interface BUILDER_STRS_ITEM { + // While we use length which is number of utf-16 code units excluding null terminator, + // we always write a null terminator, so we can get restructure to do that for us here + offset: number; //? new r.Pointer(r.uint32le, new r.String(null, 'utf16le')), + length: number; // in UTF-16 code units + _value: string +}; + +/** + * Builder for the 'strs' section + */ +interface BUILDER_STRS extends BUILDER_SECTION { count: number; reserved: number; - items: number[]; //str[] + items: BUILDER_STRS_ITEM[]; }; -interface BUILDER_KEYS_ITEM { - vkey: number; - mod: number; - to: number; //str or UTF-32 char depending on value of 'extend' - flags: number; //bitfield -}; - -interface BUILDER_KEYS extends BUILDER_SECTION { - count: number; - reserved: number; - items: BUILDER_KEYS_ITEM[]; -}; +/* ------------------------------------------------------------------ + * vkey section + ------------------------------------------------------------------ */ +/** + * Builder for the 'vkey' section + */ interface BUILDER_VKEY_ITEM { vkey: number; target: number; @@ -92,11 +140,11 @@ export default class KMXPlusBuilder { } private sect_sect: BUILDER_SECT; - private sect_strs: BUILDER_STRS; + private sect_keys: BUILDER_KEYS; + private sect_loca: BUILDER_LOCA; private sect_meta: BUILDER_META; private sect_name: BUILDER_NAME; - private sect_loca: BUILDER_LOCA; - private sect_keys: BUILDER_KEYS; + private sect_strs: BUILDER_STRS; private sect_vkey: BUILDER_VKEY; private alloc_string(value: string) { @@ -133,72 +181,6 @@ export default class KMXPlusBuilder { }; } - private build_strs(): BUILDER_STRS { - return { - ident: constants.hex_section_id(constants.section.strs), - size: 0, // finalized later - _offset: 0, - count: 0, // finalized later - reserved: 0, - items: [], // finalized later - }; - } - - private build_meta(): BUILDER_META { - return { - ident: constants.hex_section_id(constants.section.meta), - size: constants.length_meta, - _offset: 0, - name: this.alloc_string(this.file.kmxplus.meta.name), - author: this.alloc_string(this.file.kmxplus.meta.author), - conform: this.alloc_string(this.file.kmxplus.meta.conform), - layout: this.alloc_string(this.file.kmxplus.meta.layout), - normalization: this.alloc_string(this.file.kmxplus.meta.normalization), - indicator: this.alloc_string(this.file.kmxplus.meta.indicator), - version: this.alloc_string(this.file.kmxplus.meta.version), - settings: this.file.kmxplus.meta.settings ?? 0, - }; - } - - private build_loca(): BUILDER_LOCA { - let loca: BUILDER_LOCA = { - ident: constants.hex_section_id(constants.section.loca), - size: constants.length_loca + constants.length_loca_item * this.file.kmxplus.loca.locales.length, - _offset: 0, - count: this.file.kmxplus.loca.locales.length, - reserved: 0, - items: [] - }; - - for(let item of this.file.kmxplus.loca.locales) { - loca.items.push(this.alloc_string(item)); - } - - return loca; - } - - private build_name(): BUILDER_NAME { - if(!this.file.kmxplus.name.names.length) { - return null; - } - - let name: BUILDER_NAME = { - ident: constants.hex_section_id(constants.section.name), - size: constants.length_name + constants.length_name_item * this.file.kmxplus.name.names.length, - _offset: 0, - count: this.file.kmxplus.name.names.length, - reserved: 0, - items: [] - }; - - for(let item of this.file.kmxplus.name.names) { - name.items.push(this.alloc_string(item)); - } - - return name; - } - - private build_keys(): BUILDER_KEYS { if(!this.file.kmxplus.keys.keys.length) { return null; @@ -226,6 +208,71 @@ export default class KMXPlusBuilder { return keys; } + private build_loca(): BUILDER_LOCA { + let loca: BUILDER_LOCA = { + ident: constants.hex_section_id(constants.section.loca), + size: constants.length_loca + constants.length_loca_item * this.file.kmxplus.loca.locales.length, + _offset: 0, + count: this.file.kmxplus.loca.locales.length, + reserved: 0, + items: [] + }; + + for(let item of this.file.kmxplus.loca.locales) { + loca.items.push(this.alloc_string(item)); + } + + return loca; + } + + private build_meta(): BUILDER_META { + return { + ident: constants.hex_section_id(constants.section.meta), + size: constants.length_meta, + _offset: 0, + name: this.alloc_string(this.file.kmxplus.meta.name), + author: this.alloc_string(this.file.kmxplus.meta.author), + conform: this.alloc_string(this.file.kmxplus.meta.conform), + layout: this.alloc_string(this.file.kmxplus.meta.layout), + normalization: this.alloc_string(this.file.kmxplus.meta.normalization), + indicator: this.alloc_string(this.file.kmxplus.meta.indicator), + version: this.alloc_string(this.file.kmxplus.meta.version), + settings: this.file.kmxplus.meta.settings ?? 0, + }; + } + + private build_name(): BUILDER_NAME { + if(!this.file.kmxplus.name.names.length) { + return null; + } + + let name: BUILDER_NAME = { + ident: constants.hex_section_id(constants.section.name), + size: constants.length_name + constants.length_name_item * this.file.kmxplus.name.names.length, + _offset: 0, + count: this.file.kmxplus.name.names.length, + reserved: 0, + items: [] + }; + + for(let item of this.file.kmxplus.name.names) { + name.items.push(this.alloc_string(item)); + } + + return name; + } + + private build_strs(): BUILDER_STRS { + return { + ident: constants.hex_section_id(constants.section.strs), + size: 0, // finalized later + _offset: 0, + count: 0, // finalized later + reserved: 0, + items: [], // finalized later + }; + } + private build_vkey(): BUILDER_VKEY { if(!this.file.kmxplus.vkey.vkeys.length) { return null; @@ -250,6 +297,29 @@ export default class KMXPlusBuilder { return vkey; } + private build() { + // Required sections: sect, strs, loca, meta + + this.sect_sect = this.build_sect(); + this.sect_strs = this.build_strs(); + + // per C7043, the first string in sect_strs MUST be the zero-length string. + this.alloc_string(''); + + this.sect_meta = this.build_meta(); + this.sect_name = this.build_name(); + this.sect_loca = this.build_loca(); + this.sect_keys = this.build_keys(); + this.sect_vkey = this.build_vkey(); + + // Finalize all sections + + this.finalize_strs(); // must be done after all strings allocated + this.finalize_sect(); // must be done last + + return this.sect_sect.total; + } + private finalize_strs() { this.sect_strs.count = this.sect_strs.items.length; let offset = constants.length_strs + constants.length_strs_item * this.sect_strs.count; @@ -274,56 +344,34 @@ export default class KMXPlusBuilder { private finalize_sect() { // 'sect' section - // TODO: order sects alpha + + // We always have 'loca', 'meta' and 'strs' this.sect_sect.count = 3; // Handle optional sections if(this.sect_keys) { this.sect_sect.count++; } - if(this.sect_vkey) { + if(this.sect_name) { this.sect_sect.count++; } - if(this.sect_name) { + if(this.sect_vkey) { this.sect_sect.count++; } this.sect_sect.size = constants.length_sect + constants.length_sect_item * this.sect_sect.count; let offset = this.sect_sect.size; - offset = this.finalize_sect_item(this.sect_strs, offset); + offset = this.finalize_sect_item(this.sect_keys, offset); + offset = this.finalize_sect_item(this.sect_loca, offset); offset = this.finalize_sect_item(this.sect_meta, offset); offset = this.finalize_sect_item(this.sect_name, offset); - offset = this.finalize_sect_item(this.sect_loca, offset); - offset = this.finalize_sect_item(this.sect_keys, offset); + offset = this.finalize_sect_item(this.sect_strs, offset); offset = this.finalize_sect_item(this.sect_vkey, offset); this.sect_sect.total = offset; } - private build() { - // Required sections: sect, strs, loca, meta - - this.sect_sect = this.build_sect(); - this.sect_strs = this.build_strs(); - - // per C7043, the first string in sect_strs MUST be the zero-length string. - this.alloc_string(''); - - this.sect_meta = this.build_meta(); - this.sect_name = this.build_name(); - this.sect_loca = this.build_loca(); - this.sect_keys = this.build_keys(); - this.sect_vkey = this.build_vkey(); - - // Finalize all sections - - this.finalize_strs(); // must be done after all strings allocated - this.finalize_sect(); // must be done last - - return this.sect_sect.total; - } - private emitSection(file: Uint8Array, comp: any, sect: BUILDER_SECTION) { if(sect) { file.set(comp.toBuffer(sect), sect._offset); @@ -348,12 +396,12 @@ export default class KMXPlusBuilder { let file: Uint8Array = new Uint8Array(fileSize); this.emitSection(file, this.file.COMP_PLUS_SECT, this.sect_sect); - this.emitSection(file, this.file.COMP_PLUS_STRS, this.sect_strs); - this.emitStrings(file); + this.emitSection(file, this.file.COMP_PLUS_KEYS, this.sect_keys); + this.emitSection(file, this.file.COMP_PLUS_LOCA, this.sect_loca); this.emitSection(file, this.file.COMP_PLUS_META, this.sect_meta); this.emitSection(file, this.file.COMP_PLUS_NAME, this.sect_name); - this.emitSection(file, this.file.COMP_PLUS_LOCA, this.sect_loca); - this.emitSection(file, this.file.COMP_PLUS_KEYS, this.sect_keys); + this.emitSection(file, this.file.COMP_PLUS_STRS, this.sect_strs); + this.emitStrings(file); this.emitSection(file, this.file.COMP_PLUS_VKEY, this.sect_vkey); return file; diff --git a/developer/src/kmldmlc/src/keyman/kmx/kmx-plus.ts b/developer/src/kmldmlc/src/keyman/kmx/kmx-plus.ts index ee23717001..b2a9005ea7 100644 --- a/developer/src/kmldmlc/src/keyman/kmx/kmx-plus.ts +++ b/developer/src/kmldmlc/src/keyman/kmx/kmx-plus.ts @@ -9,6 +9,34 @@ import KMXFile from './kmx'; export class Section { } +// 'sect' +// 'keys' + +export enum KeyFlags { + none = 0, + extend = 1<<0, // note, this should be used only when streaming, ignored in-memory + // additional flags reserved for future use +}; + +export class KeysItem { + vkey: number; + mod: number; + to: string; + flags: KeyFlags; +}; + +export class Keys extends Section { + keys: KeysItem[] = []; +}; + +// 'loca' + +export class Loca extends Section { + locales: string[] = []; +}; + +// 'meta' + export enum KeyboardSettings { none = 0, fallback = 1<<0, @@ -29,30 +57,14 @@ export class Meta extends Section { settings: KeyboardSettings; }; -export class Loca extends Section { - locales: string[] = []; -}; +// 'name' export class Name extends Section { names: string[] = []; }; -export enum KeyFlags { - none = 0, - extend = 1<<0, // note, this should be used only when streaming, ignored in-memory - // additional flags reserved for future use -}; - -export class KeysItem { - vkey: number; - mod: number; - to: string; - flags: KeyFlags; -}; - -export class Keys extends Section { - keys: KeysItem[] = []; -}; +// 'strs' +// 'vkey' export class VkeyItem { vkey: number; @@ -70,19 +82,19 @@ export default class KMXPlusFile extends KMXFile { public readonly COMP_PLUS_SECT_ITEM: any; public readonly COMP_PLUS_SECT: any; - public readonly COMP_PLUS_STRS_ITEM: any; - public readonly COMP_PLUS_STRS: any; + public readonly COMP_PLUS_KEYS_ITEM: any; + public readonly COMP_PLUS_KEYS: any; + + public readonly COMP_PLUS_LOCA_ITEM: any; + public readonly COMP_PLUS_LOCA: any; public readonly COMP_PLUS_META: any; public readonly COMP_PLUS_NAME_ITEM: any; public readonly COMP_PLUS_NAME: any; - public readonly COMP_PLUS_LOCA_ITEM: any; - public readonly COMP_PLUS_LOCA: any; - - public readonly COMP_PLUS_KEYS_ITEM: any; - public readonly COMP_PLUS_KEYS: any; + public readonly COMP_PLUS_STRS_ITEM: any; + public readonly COMP_PLUS_STRS: any; public readonly COMP_PLUS_VKEY_ITEM: any; public readonly COMP_PLUS_VKEY: any; @@ -91,11 +103,11 @@ export default class KMXPlusFile extends KMXFile { public kmxplus: { sect?: Section; // sect is ignored here for writing - strs?: Section; // strs is ignored here for writing + keys?: Keys; loca?: Loca; meta?: Meta; name?: Name; - keys?: Keys; + strs?: Section; // strs is ignored here for writing vkey?: Vkey; } = { }; @@ -105,6 +117,8 @@ export default class KMXPlusFile extends KMXFile { // Binary-correct structures matching kmx_plus.h + // 'sect' + this.COMP_PLUS_SECT_ITEM = new r.Struct({ sect: r.uint32le, offset: r.uint32le //? new r.VoidPointer(r.uint32le, {type: 'global'}) @@ -118,53 +132,7 @@ export default class KMXPlusFile extends KMXFile { items: new r.Array(this.COMP_PLUS_SECT_ITEM, 'count') }); - this.COMP_PLUS_STRS_ITEM = new r.Struct({ - // While we use length which is number of utf-16 code units excluding null terminator, - // we always write a null terminator, so we can get restructure to do that for us here - offset: r.uint32le, //? new r.Pointer(r.uint32le, new r.String(null, 'utf16le')), - length: r.uint32le - }); - - this.COMP_PLUS_STRS = new r.Struct({ - ident: r.uint32le, - size: r.uint32le, - count: r.uint32le, - reserved: new r.Reserved(r.uint32le), // padding - items: new r.Array(this.COMP_PLUS_STRS_ITEM, 'count') - }); - - this.COMP_PLUS_META = new r.Struct({ - ident: r.uint32le, - size: r.uint32le, - name: r.uint32le, //str - author: r.uint32le, //str - conform: r.uint32le, //str - layout: r.uint32le, //str - normalization: r.uint32le, //str - indicator: r.uint32le, //str - version: r.uint32le, //str - settings: r.uint32le, //new r.Bitfield(r.uint32le, ['fallback', 'transformFailure', 'transformPartial']) - }); - - this.COMP_PLUS_NAME_ITEM = r.uint32le; //str - - this.COMP_PLUS_NAME = new r.Struct({ - ident: r.uint32le, - size: r.uint32le, - count: r.uint32le, - reserved: new r.Reserved(r.uint32le), // padding - items: new r.Array(this.COMP_PLUS_NAME_ITEM, 'count') - }); - - this.COMP_PLUS_LOCA_ITEM = r.uint32le; //str - - this.COMP_PLUS_LOCA = new r.Struct({ - ident: r.uint32le, - size: r.uint32le, - count: r.uint32le, - reserved: new r.Reserved(r.uint32le), // padding - items: new r.Array(this.COMP_PLUS_LOCA_ITEM, 'count') - }); + // 'keys' this.COMP_PLUS_KEYS_ITEM = new r.Struct({ vkey: r.uint32le, @@ -181,6 +149,64 @@ export default class KMXPlusFile extends KMXFile { items: new r.Array(this.COMP_PLUS_KEYS_ITEM, 'count') }); + // 'loca' + + this.COMP_PLUS_LOCA_ITEM = r.uint32le; //str + + this.COMP_PLUS_LOCA = new r.Struct({ + ident: r.uint32le, + size: r.uint32le, + count: r.uint32le, + reserved: new r.Reserved(r.uint32le), // padding + items: new r.Array(this.COMP_PLUS_LOCA_ITEM, 'count') + }); + + // 'meta' + + this.COMP_PLUS_META = new r.Struct({ + ident: r.uint32le, + size: r.uint32le, + name: r.uint32le, //str + author: r.uint32le, //str + conform: r.uint32le, //str + layout: r.uint32le, //str + normalization: r.uint32le, //str + indicator: r.uint32le, //str + version: r.uint32le, //str + settings: r.uint32le, //new r.Bitfield(r.uint32le, ['fallback', 'transformFailure', 'transformPartial']) + }); + + // 'name' + + this.COMP_PLUS_NAME_ITEM = r.uint32le; //str + + this.COMP_PLUS_NAME = new r.Struct({ + ident: r.uint32le, + size: r.uint32le, + count: r.uint32le, + reserved: new r.Reserved(r.uint32le), // padding + items: new r.Array(this.COMP_PLUS_NAME_ITEM, 'count') + }); + + // 'strs' + + this.COMP_PLUS_STRS_ITEM = new r.Struct({ + // While we use length which is number of utf-16 code units excluding null terminator, + // we always write a null terminator, so we can get restructure to do that for us here + offset: r.uint32le, //? new r.Pointer(r.uint32le, new r.String(null, 'utf16le')), + length: r.uint32le + }); + + this.COMP_PLUS_STRS = new r.Struct({ + ident: r.uint32le, + size: r.uint32le, + count: r.uint32le, + reserved: new r.Reserved(r.uint32le), // padding + items: new r.Array(this.COMP_PLUS_STRS_ITEM, 'count') + }); + + // 'vkey' + this.COMP_PLUS_VKEY_ITEM = new r.Struct({ vkey: r.uint32le, target: r.uint32le diff --git a/developer/src/kmldmlc/test/fixtures/basic.txt b/developer/src/kmldmlc/test/fixtures/basic.txt index 864acbec82..b4ab71ecca 100644 --- a/developer/src/kmldmlc/test/fixtures/basic.txt +++ b/developer/src/kmldmlc/test/fixtures/basic.txt @@ -20,7 +20,7 @@ block(kmxheader) # struct COMP_KEYBOARD { 00 10 00 00 # KMX_DWORD dwFileVersion; // 0004 Version of the file - Keyman 4.0 is 0x0400 - 46 bf 48 f2 # KMX_DWORD dwCheckSum; // 0008 As stored in keyboard + 62 78 ee 07 # KMX_DWORD dwCheckSum; // 0008 As stored in keyboard 00 00 00 00 # KMX_DWORD KeyboardID; // 000C as stored in HKEY_LOCAL_MACHINE//system//currentcontrolset//control//keyboard layouts 01 00 00 00 # KMX_DWORD IsRegistered; // 0010 00 00 00 00 # KMX_DWORD version; // 0014 keyboard version @@ -58,8 +58,11 @@ block(sect) # struct COMP_KMXPLUS_SECT { # KMX_DWORD sect; // 0010+ Section identity # KMX_DWORD offset; // 0014+ Section offset relative to dpKMXPlus of section - 73 74 72 73 - diff(sect,strs) + 6b 65 79 73 + diff(sect,keys) + + 6c 6f 63 61 + diff(sect,loca) 6d 65 74 61 diff(sect,meta) @@ -67,11 +70,49 @@ block(sect) # struct COMP_KMXPLUS_SECT { 6e 61 6d 65 diff(sect,name) - 6c 6f 63 61 - diff(sect,loca) + 73 74 72 73 + diff(sect,strs) - 6b 65 79 73 - diff(sect,keys) +block(keys) # struct COMP_KMXPLUS_KEYS { + 6b 65 79 73 # KMX_DWORD header.ident; // 0000 Section name - keys + sizeof(keys) # KMX_DWORD header.size; // 0004 Section length + 02 00 00 00 # KMX_DWORD count; // 0008 number of keys + 00 00 00 00 # KMX_DWORD reserved; // 000C padding + # }; + +# Keys data: + + c0 00 00 00 00 00 00 00 index(strNull,strKey1,2) 01 00 00 00 # KMX_DWORD vkey, mod, to, flags; + 31 00 00 00 00 00 00 00 index(strNull,strKey2,2) 01 00 00 00 # KMX_DWORD vkey, mod, to, flags; + +block(loca) # struct COMP_KMXPLUS_LOCA { + 6c 6f 63 61 # KMX_DWORD header.ident; // 0000 Section name - loca + sizeof(loca) # KMX_DWORD header.size; // 0004 Section length + 01 00 00 00 # KMX_DWORD count; // 0008 number of locales + 00 00 00 00 # KMX_DWORD reserved; // 000C padding + index(strNull,strLocale,2) # KMXPLUS_STR locale; // 0010+ locale string entry = 'mt' + # }; + +block(meta) # struct COMP_KMXPLUS_META { + 6d 65 74 61 # KMX_DWORD header.ident; // 0000 Section name - meta + sizeof(meta) # KMX_DWORD header.size; // 0004 Section length + index(strNull,strName,2) # KMXPLUS_STR name; + index(strNull,strAuthor,2) # KMXPLUS_STR author; + index(strNull,strConformsTo,2) # KMXPLUS_STR conform; + index(strNull,strLayout,2) # KMXPLUS_STR layout; + index(strNull,strNorm,2) # KMXPLUS_STR normalization; + index(strNull,strIndicator,2) # KMXPLUS_STR indicator; + index(strNull,strVersion,2) # KMXPLUS_STR version; + 00 00 00 00 # KMX_DWORD settings; + # }; + +block(name) # struct COMP_KMXPLUS_META { + 6e 61 6d 65 # KMX_DWORD header.ident; // 0000 Section name - name + sizeof(name) # KMX_DWORD header.size; // 0004 Section length + 01 00 00 00 # KMX_DWORD count; // 0008 number of names + 00 00 00 00 # KMX_DWORD reserved; // 000C padding + index(strNull,strName,2) # KMXPLUS_STR name; // 0010+ name string entry = 'TestKbd' + # }; block(strs) # struct COMP_KMXPLUS_STRS { 73 74 72 73 # KMX_DWORD header.ident; // 0000 Section name - strs @@ -114,45 +155,4 @@ block(strs) # struct COMP_KMXPLUS_STRS { block(endstrs) # end of strs block -block(meta) # struct COMP_KMXPLUS_META { - 6d 65 74 61 # KMX_DWORD header.ident; // 0000 Section name - meta - sizeof(meta) # KMX_DWORD header.size; // 0004 Section length - index(strNull,strName,2) # KMXPLUS_STR name; - index(strNull,strAuthor,2) # KMXPLUS_STR author; - index(strNull,strConformsTo,2) # KMXPLUS_STR conform; - index(strNull,strLayout,2) # KMXPLUS_STR layout; - index(strNull,strNorm,2) # KMXPLUS_STR normalization; - index(strNull,strIndicator,2) # KMXPLUS_STR indicator; - index(strNull,strVersion,2) # KMXPLUS_STR version; - 00 00 00 00 # KMX_DWORD settings; - # }; - -block(name) # struct COMP_KMXPLUS_META { - 6e 61 6d 65 # KMX_DWORD header.ident; // 0000 Section name - name - sizeof(name) # KMX_DWORD header.size; // 0004 Section length - 01 00 00 00 # KMX_DWORD count; // 0008 number of names - 00 00 00 00 # KMX_DWORD reserved; // 000C padding - index(strNull,strName,2) # KMXPLUS_STR name; // 0010+ name string entry = 'TestKbd' - # }; - -block(loca) # struct COMP_KMXPLUS_LOCA { - 6c 6f 63 61 # KMX_DWORD header.ident; // 0000 Section name - loca - sizeof(loca) # KMX_DWORD header.size; // 0004 Section length - 01 00 00 00 # KMX_DWORD count; // 0008 number of locales - 00 00 00 00 # KMX_DWORD reserved; // 000C padding - index(strNull,strLocale,2) # KMXPLUS_STR locale; // 0010+ locale string entry = 'mt' - # }; - -block(keys) # struct COMP_KMXPLUS_KEYS { - 6b 65 79 73 # KMX_DWORD header.ident; // 0000 Section name - keys - sizeof(keys) # KMX_DWORD header.size; // 0004 Section length - 02 00 00 00 # KMX_DWORD count; // 0008 number of keys - 00 00 00 00 # KMX_DWORD reserved; // 000C padding - # }; - -# Keys data: - - c0 00 00 00 00 00 00 00 index(strNull,strKey1,2) 01 00 00 00 # KMX_DWORD vkey, mod, to, flags; - 31 00 00 00 00 00 00 00 index(strNull,strKey2,2) 01 00 00 00 # KMX_DWORD vkey, mod, to, flags; - block(eof) # end of file \ No newline at end of file