erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-10-06 14:07:32 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 11

feat(core): ldml keyboard transform elements

Browse source 
Adds definitions for LDML keyboard transform elements `transforms`,
`backspaces`, and `reorders`, and the binary format sections `bksp`,
`elem`, `finl`, `ordr`, and `tran`.

Minimally updates the compiler so that it will build, but no other
changes to the compiler.

Compiler and Typescript updates to support building these sections will
come in the next PR.
This commit is contained in:
Marc Durdin 2022-09-05 13:11:51 +10:00
parent a708b2f85d
commit 9f5f8e976e
5 changed files with 412 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
core/include/ldml/keyboardprocessor_ldml.h
View file

@ -15,7 +15,21 @@
#pragma once
#define LDML_ELEM_FLAGS_ORDER_BITSHIFT 0x10
#define LDML_ELEM_FLAGS_ORDER_MASK 0xFF0000
#define LDML_ELEM_FLAGS_PREBASE 0x4
#define LDML_ELEM_FLAGS_TERTIARY_BASE 0x2
#define LDML_ELEM_FLAGS_TERTIARY_BITSHIFT 0x18
#define LDML_ELEM_FLAGS_TERTIARY_MASK 0xFF000000
#define LDML_ELEM_FLAGS_UNICODE_SET 0x1
#define LDML_KEYS_FLAGS_EXTEND 0x1
#define LDML_LENGTH_BKSP 0x10
#define LDML_LENGTH_BKSP_ITEM 0x10
#define LDML_LENGTH_ELEM 0x10
#define LDML_LENGTH_ELEM_ITEM 0x8
#define LDML_LENGTH_ELEM_ITEM_ELEMENT 0x8
#define LDML_LENGTH_FINL 0x10
#define LDML_LENGTH_FINL_ITEM 0x10
#define LDML_LENGTH_HEADER 0x8
#define LDML_LENGTH_KEYS 0x10
#define LDML_LENGTH_KEYS_ITEM 0x10
@ -24,15 +38,25 @@
#define LDML_LENGTH_META 0x28
#define LDML_LENGTH_NAME 0x10
#define LDML_LENGTH_NAME_ITEM 0x4
#define LDML_LENGTH_ORDR 0x10
#define LDML_LENGTH_ORDR_ITEM 0x8
#define LDML_LENGTH_SECT 0x10
#define LDML_LENGTH_SECT_ITEM 0x8
#define LDML_LENGTH_STRS 0x10
#define LDML_LENGTH_STRS_ITEM 0x8
#define LDML_LENGTH_TRAN 0x10
#define LDML_LENGTH_TRAN_ITEM 0x10
#define LDML_LENGTH_VKEY 0x10
#define LDML_LENGTH_VKEY_ITEM 0x8
#define LDML_META_SETTINGS_FALLBACK_OMIT 0x1
#define LDML_META_SETTINGS_TRANSFORMFAILURE_OMIT 0x2
#define LDML_META_SETTINGS_TRANSFORMPARTIAL_HIDE 0x4
#define LDML_SECTIONID_BKSP 0x70736B62 /* "bksp" */
#define LDML_SECTIONNAME_BKSP "bksp"
#define LDML_SECTIONID_ELEM 0x6D656C65 /* "elem" */
#define LDML_SECTIONNAME_ELEM "elem"
#define LDML_SECTIONID_FINL 0x6C6E6966 /* "finl" */
#define LDML_SECTIONNAME_FINL "finl"
#define LDML_SECTIONID_KEYS 0x7379656B /* "keys" */
#define LDML_SECTIONNAME_KEYS "keys"
#define LDML_SECTIONID_LOCA 0x61636F6C /* "loca" */
@ -41,10 +65,14 @@
#define LDML_SECTIONNAME_META "meta"
#define LDML_SECTIONID_NAME 0x656D616E /* "name" */
#define LDML_SECTIONNAME_NAME "name"
#define LDML_SECTIONID_ORDR 0x7264726F /* "ordr" */
#define LDML_SECTIONNAME_ORDR "ordr"
#define LDML_SECTIONID_SECT 0x74636573 /* "sect" */
#define LDML_SECTIONNAME_SECT "sect"
#define LDML_SECTIONID_STRS 0x73727473 /* "strs" */
#define LDML_SECTIONNAME_STRS "strs"
#define LDML_SECTIONID_TRAN 0x6E617274 /* "tran" */
#define LDML_SECTIONNAME_TRAN "tran"
#define LDML_SECTIONID_VKEY 0x79656B76 /* "vkey" */
#define LDML_SECTIONNAME_VKEY "vkey"
#define LDML_VERSION "1.0"

165
core/include/ldml/keyboardprocessor_ldml.ts
View file

@ -24,13 +24,19 @@
*/
export type SectionIdent =
'sect' |
'bksp' |
'elem' |
'finl' |
'keys' |
'loca' |
'meta' |
'name' |
'ordr' |
'strs' |
'tran' |
'vkey';
type SectionMap = {
[id in SectionIdent]: SectionIdent;
}
@ -74,6 +80,116 @@ export const constants: Constants = {
*/
length_sect_item: 8,
/* ------------------------------------------------------------------
* bksp section
------------------------------------------------------------------ */
/**
* Minimum length of the 'bksp' section, not including entries
*/
length_bksp: 16,
/**
* Length of each item in the 'bksp' section variable part
*/
length_bksp_item: 16,
/* ------------------------------------------------------------------
* elem section
------------------------------------------------------------------ */
/**
* Minimum length of the 'elem' section, not including entries
*/
length_elem: 16,
/**
* Length of each elem string in the 'elem' section variable part
*/
length_elem_item: 8,
/**
* Length of each element in an elem string
*/
length_elem_item_element: 8,
/**
* bitwise or value for unicode_set in elem[elemstr][element].flags.
* If bit is 1, then 'element' is a UnicodeSet string.
* If bit is 0, then 'element' is a UTF-32LE codepoint
*
* `unicode_set = flags & elem_flags_unicode_set`
*/
elem_flags_unicode_set: 0x00000001,
/**
* bitwise or value for tertiary_base in elem[elemstr][element].flags.
* If bit is 1, then tertiary_base is true.
* If bit is 0, then tertiary_base is false.
*
* Used only for `ordr`-type element strings.
*
* `tertiary_base = flags & elem_flags_tertiary_base`
*/
elem_flags_tertiary_base: 0x00000002,
/**
* bitwise or value for tertiary_base in elem[elemstr][element].flags.
* If bit is 1, then prebase is true.
* If bit is 0, then prebase is false.
*
* Used only for `ordr`-type element strings.
*
* `prebase = flags & elem_flags_prebase`
*/
elem_flags_prebase: 0x00000004,
/**
* bitwise mask for order in elem[elemstr][element].flags.
*
* Used only for `ordr`-type element strings. 1 byte signed integer.
*
* `order = (flags & elem_flags_order_mask) >> elem_flags_order_bitshift`
*/
elem_flags_order_mask: 0x00FF0000,
/**
* bit shift for order in elem[elemstr][element].flags.
*
* Used only for `ordr`-type element strings.
*
* `order = (flags & elem_flags_order_mask) >> elem_flags_order_bitshift`
*/
elem_flags_order_bitshift: 16,
/**
* bitwise mask for tertiary sort in elem[elemstr][element].flags.
*
* Used only for `ordr`-type element strings. 1 byte signed integer.
*
* `tertiary = (flags & elem_flags_tertiary_mask) >> elem_flags_tertiary_bitshift`
*/
elem_flags_tertiary_mask: 0xFF000000,
/**
* bit shift for tertiary sort in elem[elemstr][element].flags.
*
* Used only for `ordr`-type element strings. 1 byte signed integer.
*
* `order = (flags & elem_flags_tertiary_mask) >> elem_flags_tertiary_bitshift`
*/
elem_flags_tertiary_bitshift: 24,
/* ------------------------------------------------------------------
* finl section
------------------------------------------------------------------ */
/**
* Minimum length of the 'finl' section, not including entries
*/
length_finl: 16,
/**
* Length of each item in the 'finl' section variable part
*/
length_finl_item: 16,
/* ------------------------------------------------------------------
* keys section
------------------------------------------------------------------ */
@ -93,7 +209,7 @@ export const constants: Constants = {
*
* `extend = flags & keys_flags_extend`
*/
keys_flags_extend: 1,
keys_flags_extend: 1,
/* ------------------------------------------------------------------
* loca section
@ -136,11 +252,24 @@ export const constants: Constants = {
/**
* Minimum length of the 'name' section not including variable parts
*/
length_name: 16,
/**
* Length of each item in the 'name' section variable part
*/
length_name_item: 4,
length_name: 16,
/**
* Length of each item in the 'name' section variable part
*/
length_name_item: 4,
/* ------------------------------------------------------------------
* ordr section
------------------------------------------------------------------ */
/**
* Minimum length of the 'ordr' section, not including entries
*/
length_ordr: 16,
/**
* Length of each item in the 'ordr' section variable part
*/
length_ordr_item: 8,
/* ------------------------------------------------------------------
* strs section
@ -155,15 +284,30 @@ export const constants: Constants = {
*/
length_strs_item: 8,
/* ------------------------------------------------------------------
* tran section
------------------------------------------------------------------ */
/**
* Minimum length of the 'tran' section, not including entries
*/
length_tran: 16,
/**
* Length of each item in the 'tran' section variable part
*/
length_tran_item: 16,
/* ------------------------------------------------------------------
* vkey section
------------------------------------------------------------------ */
/**
* Minimum length of the 'vkey' section
* not including variable parts
* Minimum length of the 'vkey' section not including variable parts
*/
length_vkey: 16,
/**
* Length of each item in the 'vkey' section variable part
*/
length_vkey_item: 8,
/**
@ -171,11 +315,16 @@ export const constants: Constants = {
*/
section: {
sect: 'sect',
bksp: 'bksp',
elem: 'elem',
finl: 'finl',
keys: 'keys',
loca: 'loca',
meta: 'meta',
name: 'name',
ordr: 'ordr',
strs: 'strs',
tran: 'tran',
vkey: 'vkey',
},

103
core/src/kmx/kmx_plus.h
View file

@ -35,6 +35,11 @@ namespace kmx {
#endif
typedef KMX_DWORD KMXPLUS_STR;
typedef KMX_DWORD KMXPLUS_ELEM;
// forward declarations
struct COMP_KMXPLUS_TRAN_ENTRY;
struct COMP_KMXPLUS_TRAN;
struct COMP_KMXPLUS_HEADER {
KMX_DWORD ident; // 0000 Section name
@ -78,6 +83,53 @@ struct COMP_KMXPLUS_SECT {
static_assert(sizeof(struct COMP_KMXPLUS_SECT) == LDML_LENGTH_SECT, "mismatched size of section sect");
static_assert(sizeof(struct COMP_KMXPLUS_SECT) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary");
/* ------------------------------------------------------------------
* bksp section
------------------------------------------------------------------ */
typedef COMP_KMXPLUS_TRAN_ENTRY COMP_KMXPLUS_BKSP_ENTRY;
typedef COMP_KMXPLUS_TRAN COMP_KMXPLUS_BKSP;
/* ------------------------------------------------------------------
* elem section
------------------------------------------------------------------ */
struct COMP_KMXPLUS_ELEM_ELEMENT {
KMX_DWORD element; // str: output string or UTF-32LE codepoint
KMX_DWORD flags; // flag and order values
};
struct COMP_KMXPLUS_ELEM_ENTRY {
KMX_DWORD offset; // 0010+ offset from this blob
KMX_DWORD length; // 0014+ str length (ELEMENT units)
};
struct COMP_KMXPLUS_ELEM {
static const KMX_DWORD IDENT = LDML_SECTIONID_ELEM;
COMP_KMXPLUS_HEADER header;
KMX_DWORD count; // 0008 count of str entries
KMX_DWORD reserved; // 000C padding
COMP_KMXPLUS_ELEM_ENTRY entries[]; // 0010+ entries
/**
* @brief True if section is valid.
*/
bool valid(KMX_DWORD length) const;
};
static_assert(sizeof(struct COMP_KMXPLUS_ELEM) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary");
static_assert(sizeof(struct COMP_KMXPLUS_ELEM) == LDML_LENGTH_ELEM, "mismatched size of section elem");
/* ------------------------------------------------------------------
* finl section
------------------------------------------------------------------ */
// TODO-LDML: IDENT
typedef COMP_KMXPLUS_TRAN_ENTRY COMP_KMXPLUS_FINL_ENTRY;
// TODO-LDML: IDENT
typedef COMP_KMXPLUS_TRAN COMP_KMXPLUS_FINL;
/* ------------------------------------------------------------------
* keys section
------------------------------------------------------------------ */
@ -173,6 +225,30 @@ struct COMP_KMXPLUS_NAME {
static_assert(sizeof(struct COMP_KMXPLUS_NAME) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary");
static_assert(sizeof(struct COMP_KMXPLUS_NAME) == LDML_LENGTH_NAME, "mismatched size of section name");
/* ------------------------------------------------------------------
* ordr section
------------------------------------------------------------------ */
struct COMP_KMXPLUS_ORDR_ENTRY {
KMXPLUS_ELEM elements;
KMXPLUS_ELEM before;
};
struct COMP_KMXPLUS_ORDR {
static const KMX_DWORD IDENT = LDML_SECTIONID_ORDR;
COMP_KMXPLUS_HEADER header;
KMX_DWORD count;
KMX_DWORD reserved;
COMP_KMXPLUS_ORDR_ENTRY entries[];
/**
* @brief True if section is valid.
*/
bool valid(KMX_DWORD length) const;
};
static_assert(sizeof(struct COMP_KMXPLUS_ORDR) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary");
static_assert(sizeof(struct COMP_KMXPLUS_ORDR) == LDML_LENGTH_ORDR, "mismatched size of section ordr");
/* ------------------------------------------------------------------
* strs section
------------------------------------------------------------------ */
@ -207,6 +283,33 @@ struct COMP_KMXPLUS_STRS {
static_assert(sizeof(struct COMP_KMXPLUS_STRS) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary");
static_assert(sizeof(struct COMP_KMXPLUS_STRS) == LDML_LENGTH_STRS, "mismatched size of section strs");
/* ------------------------------------------------------------------
* tran section
------------------------------------------------------------------ */
struct COMP_KMXPLUS_TRAN_ENTRY {
KMXPLUS_ELEM from;
KMXPLUS_STR to;
KMXPLUS_ELEM before;
KMX_DWORD flags;
};
struct COMP_KMXPLUS_TRAN {
static const KMX_DWORD IDENT = LDML_SECTIONID_TRAN;
COMP_KMXPLUS_HEADER header;
KMX_DWORD count;
KMX_DWORD reserved;
COMP_KMXPLUS_TRAN_ENTRY entries[];
/**
* @brief True if section is valid.
*/
bool valid(KMX_DWORD length) const;
};
static_assert(sizeof(struct COMP_KMXPLUS_TRAN) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary");
static_assert(sizeof(struct COMP_KMXPLUS_TRAN) == LDML_LENGTH_TRAN, "mismatched size of section tran");
/* ------------------------------------------------------------------
* vkey section
------------------------------------------------------------------ */

127
core/src/ldml/C7043_ldml.md
View file

@ -62,7 +62,57 @@ Then for `count` repetitions:
This list is in sorted order based on the `sect` identifier.
### C7043.2.2 `keys`—Keybag
### C7043.2.2 `bksp`—Backspace transform
See [C7043.2.11](#c7043211-tran-finl-and-bksptransforms).
### C7043.2.3 `elem`—Transform and Reorder element strings
| ∆ | Bits | Name | Description |
|---|------|----------|------------------------------------------|
| 0 | 32 | ident | `elem` |
| 4 | 32 | size | int: Length of section |
| 8 | 32 | count | int: Number of element string entries |
|12 | 32 | reserved | padding |
Then for each element string:
| ∆ | Bits | Name | Description |
|---|------|---------------|-----------------------------------------------|
|16+| 32 | offset | off: Offset to element string |
|20+| 32 | length | int: Number of elements in element string |
The first entry in the element string list MUST be the null element string,
which has zero length and zero offset.
Each element string is made up of elements with the following item structure:
| ∆ | Bits | Name | Description |
|---|------|-----------|----------------------------------------------------------|
| 0 | 32 | element | str: output string OR UTF-32LE codepoint |
| 4 | 32 | flags | flags and order values |
- `element`: either a UnicodeSet stored in a `strs` section entry, or a UTF-32LE
codepoint; see also `unicode_set` flag.
- `flags`: a 32-bit bitfield defined as below:
| Bit position | Meaning | Description |
|--------------|---------------|----------------------------------|
| 0 | unicode_set | `element` is 0: UTF-32LE, 1: str |
| 1 | tertiary_base | 1: tertiary_base is true |
| 2 | prebase | 1: prebase is true |
| 3-15 | reserved | reserved |
| 16-23 | order | signed int: -128 to +127 |
| 24-31 | tertiary | signed int: -128 to +127 |
For transforms, only `flags.unicode_set` will be used. The remaining flags are
used for reorders, `from` attribute only.
### C7043.2.4 `finl`—Final transform
See [C7043.2.11](#c7043211-tran-finl-and-bksptransforms).
### C7043.2.5 `keys`—Keybag
| ∆ | Bits | Name | Description |
|---|------|-----------|------------------------------------------|
@ -96,7 +146,7 @@ For each key:
- `to`: If `extend` is 0, `to` is a UTF-32LE codepoint. If `extend` is 1, `to`
is a 32 bit index into the `strs` table. The string may be zero-length.
### C7043.2.3 `loca`—Locales
### C7043.2.6 `loca`—Locales
| ∆ | Bits | Name | Description |
|---|------|---------|------------------------------------------|
@ -116,7 +166,7 @@ For each locale ID in `count`
The first locale ID is always the primary locale identifier. The rest of the
locale IDs (starting at offset 16) are in sorted binary order.
### C7043.2.4 `meta`—Metadata
### C7043.2.7 `meta`—Metadata
| ∆ | Bits | Name | Description |
|---|------|---------------|-------------------------------------|
@ -138,7 +188,7 @@ The `settings` is a 32-bit bitfield as below:
| 1 | transformFailure | transformFailure=omit |
| 2 | transformPartial | transformPartial=hide |
### C7043.2.5 `name`—Names
### C7043.2.8 `name`—Names
Defines the names of the keyboard as found in the source `<names>` element.
While this section is optional in the binary format, in practice it will always
@ -163,7 +213,27 @@ Note that the first name is repeated in the `meta` section. The remaining names
are stored in source file order, and the semantic meaning of each name is not
defined here.
### C7043.2.6 `strs`—Strings
### C7043.2.9 `ordr`—Reorders
| ∆ | Bits | Name | Description |
|---|------|----------|------------------------------------------|
| 0 | 32 | ident | `ordr` |
| 4 | 32 | size | int: Length of section |
| 8 | 32 | count | int: Number of reorders |
|12 | 32 | reserved | padding |
For each reorder item:
| ∆ | Bits | Name | Description |
|---|------|----------|----------------------------------------------------------|
|16+| 32 | elements | elem: string of elements, index into `elem` section |
|20+| 32 | before | elem: look-behind required match, index into `elem` |
- `elements`: index into the `elem` section, coding `from`, `order`, `tertiary`,
`tertiary_base`, and `prebase` properties.
- `before`: follows `transform/@before`
### C7043.2.10 `strs`—Strings
All strings are stored in the Strings section.
@ -198,7 +268,44 @@ A distinction between zero-length string and optional should be avoided (e.g.
the difference between "" and null in Javascript). If this is truly required, a
separate flag field must be used to denote the difference.
### C7043.2.7 `vkey`—VKey Map
### C7043.2.11 `tran`, `finl`, and `bksp`—Transforms
All three of these tables have the same format. and differ only in their
identity. The simple transform table has the ident `tran`; the final transform
table has the ident `finl`, and the backspaces table has the ident `bksp`.
| ∆ | Bits | Name | Description |
|---|------|----------|------------------------------------------|
| 0 | 32 | ident | `tran` / `finl` / `bksp` |
| 4 | 32 | size | int: Length of section |
| 8 | 32 | count | int: Number of transforms |
|12 | 32 | reserved | padding |
The transforms are sorted in binary order based on the `from` field.
For each transform:
| ∆ | Bits | Name | Description |
|---|------|---------|------------------------------------------|
|16+| 32 | from | elem: combination of characters |
|20+| 32 | to | str: output text |
|24+| 32 | before | elem: look-behind text (0 = omitted) |
|28+| 32 | flags | int: per-transform flags |
- `from`: the source text, index into `elem` section.
- `to`: sequence of Unicode codepoints that replace `from`. May be the null
string for `bksp` entries.
- `before`: optional look-behind text occurring before `from`, index into `elem`
section
- `flags`: a 32-bit bitfield defined as below:
| Bit position | Meaning | Description |
|--------------|----------|----------------------|
| 0 | error | 1: `error="fail"` |
### C7043.2.12 `vkey`—VKey Map
| ∆ | Bits | Name | Description |
|---|------|---------|------------------------------------------|
@ -219,6 +326,10 @@ For each key:
- `vkey`: Is the standard vkey, 0-255
- `target`: Is the target (resolved) vkey, 0-255.
### C7043.2.8 Transforms and friends
## TODO-LDML: various things that need to be completed here or fixed in-spec
> TODO: transforms
> * UnicodeSets
> * spec: reference to `after` in reorders; various other @after refs
> * spec: ABNT2 key has hex value 0xC1 (even if kbdus.dll doesn't produce that)
> * spec: layerMaps.displayWidthInMm
> * `keys.key.mod`: TODO define this. 0 for no modifiers.

5
developer/src/kmldmlc/src/keyman/kmx/kmx-plus.ts
View file

@ -103,11 +103,16 @@ export default class KMXPlusFile extends KMXFile {
public kmxplus: {
sect?: Section; // sect is ignored here for writing
bksp?: Section; // TODO-LDML
elem?: Section; // TODO-LDML
finl?: Section; // TODO-LDML
keys?: Keys;
loca?: Loca;
meta?: Meta;
name?: Name;
ordr?: Section; // TODO-LDML
strs?: Section; // strs is ignored here for writing
tran?: Section; // TODO-LDML
vkey?: Vkey;
} = { };