erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-09-16 04:37:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

Merge pull request #7083 from keymanapp/feat/core/5015-process4

Browse source 
feat(core): build out ldml structures / initial scaffolding implementation 🙀
This commit is contained in:
Steven R. Loomis 2022-08-22 17:15:33 -05:00 • committed by GitHub
commit 6d717454ea
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 864 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
core/include/ldml/keyboardprocessor_ldml.h
View file

@ -1,3 +1,11 @@
/*
Copyright: Copyright (C) 2022 SIL International.
Authors: srl295
This file provides constants for the KMX Plus (LDML support) binary format,
to be shared between TypeScript and C++ via the generator (below)
*/
//
// Generated File - do not edit
//
@ -6,6 +14,14 @@
//
#pragma once
#define LDML_KEYS_FLAGS_EXTEND 1
#define LDML_LENGTH_HEADER 8
#define LDML_LENGTH_KEYS 16
#define LDML_LENGTH_LOCA 12
#define LDML_LENGTH_META 36
#define LDML_LENGTH_SECT 16
#define LDML_LENGTH_STRS 16
#define LDML_LENGTH_VKEY 12
#define LDML_META_SETTINGS_FALLBACK_OMIT 1
#define LDML_META_SETTINGS_TRANSFORMFAILURE_OMIT 2
#define LDML_META_SETTINGS_TRANSFORMPARTIAL_HIDE 4
@ -19,5 +35,6 @@
#define LDML_SECTION_SECT ((uint32_t)'sect')
// Section ID
#define LDML_SECTION_STRS ((uint32_t)'strs')
#define LDML_STRS_FLAGS_EXTEND 1
// Section ID
#define LDML_SECTION_VKEY ((uint32_t)'vkey')
#define LDML_VERSION "1.0"

142
core/include/ldml/keyboardprocessor_ldml.ts
View file

@ -1,3 +1,10 @@
/*
Copyright: Copyright (C) 2022 SIL International.
Authors: srl295
This file provides constants for the KMX Plus (LDML support) binary format,
to be shared between TypeScript and C++ via the generator (below)
*/
// Notice!
//
@ -10,57 +17,90 @@
// It is not updated automatically.
// TODO-LDML: namespace com.keyman.core.ldml {
/**
* Constants for the KMXPlus data format
* These are shared between the data access layer and the compiler
*/
export const constants = {
/**
* Constants for the KMXPlus data format
* These are shared between the data access layer and the compiler
* The version of the LDML processor
*/
export const constants = {
/**
* The version of the LDML processor
*/
version: '1.0',
/**
* Section ID for the keybag
*/
section_keys: 'keys',
/**
* Section ID for the locale list
*/
section_loca: 'loca',
/**
* Section ID for the metadata
*/
section_meta: 'meta',
/**
* Section ID for the section header
*/
section_sect: 'sect',
/**
* Section ID for the string table
*/
section_strs: 'strs',
/**
* bitwise or value for fallback=omit in meta.settings
*/
meta_settings_fallback_omit: 1,
/**
* bitwise or value for transformFailure=omit in meta.settings
*/
meta_settings_transformFailure_omit: 2,
/**
* bitwise or value for transformPartial=hide in meta.settings
*/
meta_settings_transformPartial_hide: 4,
/**
* bitwise or value for extend in strs[key].flags.
* If bit is 1, then 'to' is a string.
* If bit is 0, then 'to' is an offset.
*
* `extend = flags & strs_flags_extend`
*/
strs_flags_extend: 1,
};
version: '1.0',
/**
* Length of a raw section header, in bytes
*/
length_header: 8,
/**
* Section ID for the keybag
*/
section_keys: 'keys',
/**
* Minimum length of the 'keys' section
* not including variable parts
*/
length_keys: 16,
/**
* Section ID for the locale list
*/
section_loca: 'loca',
/**
* Minimum length of the 'loca' section
* not including variable parts
*/
length_loca: 12,
/**
* Section ID for the metadata
*/
section_meta: 'meta',
/**
* length of the 'meta' section
*/
length_meta: 36,
/**
* bitwise or value for fallback=omit in meta.settings
*/
meta_settings_fallback_omit: 1,
/**
* bitwise or value for transformFailure=omit in meta.settings
*/
meta_settings_transformFailure_omit: 2,
/**
* bitwise or value for transformPartial=hide in meta.settings
*/
meta_settings_transformPartial_hide: 4,
/**
* Section ID for the section header
*/
section_sect: 'sect',
/**
* Minimum length of the 'sect' section, not including entries
*/
length_sect: 16,
/**
* Section ID for the string table
*/
section_strs: 'strs',
/**
* Minimum length of the 'strs' section
* not including variable parts
*/
length_strs: 16,
/**
* bitwise or value for extend in keys[key].flags.
* If bit is 1, then 'to' is a string.
* If bit is 0, then 'to' is a UTF-32LE codepoint
*
* `extend = flags & keys_flags_extend`
*/
keys_flags_extend: 1,
/**
* Section ID for the vkeys map
*/
section_vkey: 'vkey',
/**
* Minimum length of the 'vkey' section
* not including variable parts
*/
length_vkey: 12,
};
// }

16
core/include/ldml/ldml-const-builder.ts
View file

@ -1,8 +1,22 @@
/*
Copyright: Copyright (C) 2022 SIL International.
Authors: srl295
This tool generates a .h version of the keyboardprocessor_ldml.ts file
*/
import { constants } from './keyboardprocessor_ldml';
const keys = Object.keys(constants);
keys.sort();
console.log(`//
console.log(`
/*
Copyright: Copyright (C) 2022 SIL International.
Authors: srl295
This file provides constants for the KMX Plus (LDML support) binary format,
to be shared between TypeScript and C++ via the generator (below)
*/
//
// Generated File - do not edit
//
// This file is generated by core/tools/ldml-const-builder/build.sh

230
core/src/kmx/kmx_plus.cpp Normal file
View file

@ -0,0 +1,230 @@
/*
Copyright: Copyright (C) 2022 SIL International.
Authors: srl295
Implementation for the KMX Plus utilities
*/
#include <kmx/kmx_plus.h>
/**
* @def KMXPLUS_DEBUG Set to 1 to enable debug output
*/
#define KMXPLUS_DEBUG 1
#if KMXPLUS_DEBUG
#include <stdio.h>
#endif
#include <assert.h>
namespace km {
namespace kbp {
namespace kmx {
#if KMXPLUS_DEBUG
static void
dump_section_name(KMX_DWORD ident) {
for (int i = 0; i < 4; i++) {
putchar((ident & 0xFF000000) >> 24);
ident <<= 8;
}
}
static void
dump_kmxplus_header(const COMP_KMXPLUS_HEADER* hdr) {
if (hdr == NULL) {
printf("! dump_kmxplus_header: NULL header\n");
}
dump_section_name(hdr->ident);
printf(": (%X) size 0x%X\n", hdr->ident, hdr->size);
}
static void
dump_kmxplus_keys(const uint8_t* /*data*/, const COMP_KMXPLUS_KEYS* keys) {
if(keys == NULL) {
printf("! could not load 'keys' section\n");
return;
}
dump_kmxplus_header((const COMP_KMXPLUS_HEADER*)keys);
printf(" count: #0x%X\n", keys->count);
for (KMX_DWORD i = 0; i<keys->count; i++) {
printf(" #0x%d\n", i);
const COMP_KMXPLUS_KEYS_ENTRY& entry = keys->entries[i];
printf(" vkey\t0x%X\n", entry.vkey);
printf(" mod\t0x%X\n", entry.mod);
printf(" flags\t0x%X\n", entry.flags);
if (entry.flags & LDML_KEYS_FLAGS_EXTEND) {
printf(" \t Extend: String #0x%X\n", entry.to);
} else {
printf(" \t UTF-32:U+%04X\n", entry.to);
}
}
}
static void
dump_kmxplus_loca(const uint8_t* /*data*/, const COMP_KMXPLUS_LOCA* loca) {
if(loca == NULL) {
printf("! could not load 'loca' section\n");
return;
}
dump_kmxplus_header((const COMP_KMXPLUS_HEADER*)loca);
// TODO-LDML
for(KMX_DWORD i=0; i<loca->count; i++) {
printf(" Locale #%d: #0x%X\n", i, loca->entries[i].locale);
}
}
static void
dump_kmxplus_meta(const uint8_t* /*data*/, const COMP_KMXPLUS_META* meta) {
if(meta == NULL) {
printf("! could not load 'meta' section\n");
return;
}
dump_kmxplus_header((const COMP_KMXPLUS_HEADER*)meta);
printf(" name:\t#0x%X\n", meta->name);
printf(" author:\t#0x%X\n", meta->author);
printf(" conform:\t#0x%X\n", meta->conform);
printf(" layout:\t#0x%X\n", meta->layout);
printf(" normalization:\t#0x%X\n", meta->normalization);
printf(" indicator:\t#0x%X\n", meta->indicator);
printf(" settings:\t0x%X\n", meta->settings);
}
static void
dump_kmxplus_vkey(const uint8_t* /*data*/, const COMP_KMXPLUS_VKEY* vkey) {
if(vkey == NULL) {
printf("! could not load 'vkey' section\n");
return;
}
dump_kmxplus_header((const COMP_KMXPLUS_HEADER*)vkey);
// TODO-LDML
}
static void
dump_kmxplus_strs(const uint8_t* /*data*/, const COMP_KMXPLUS_STRS* strs) {
dump_kmxplus_header((const COMP_KMXPLUS_HEADER*)strs);
printf("strs: count 0x%X\n", strs->count);
for (KMX_DWORD i=0; i<strs->count; i++) {
KMX_WCHAR buf[BUFSIZ];
printf("#0x%X: ", i);
PKMX_WCHAR str = strs->get(i, buf, BUFSIZ);
if (!str) {
printf("NULL/ERR\n");
continue;
}
for(int j=0; str[j] && j<0x30; j++) {
if (str[j] < 0x7F && str[j] != 0x0020 && str[j] > 0x20) {
putchar(str[j]);
} else {
printf("U+%04X ", str[j]);
}
}
printf("\n");
}
}
static void
dump_kmxplus_sect(const uint8_t* data, const COMP_KMXPLUS_SECT* sect) {
dump_kmxplus_header((const COMP_KMXPLUS_HEADER*)sect);
printf("sect: total 0x%X\n", sect->total);
printf("sect: count 0x%X\n", sect->count);
for (KMX_DWORD i = 0; i < sect->count; i++) {
const COMP_KMXPLUS_SECT_ENTRY& entry = sect->entries[i];
dump_section_name(entry.sect);
printf(" sect#%d: %X @ %X\n", i, entry.sect, entry.offset);
const uint8_t* entrydata = (data+entry.offset);
switch(entry.sect) {
case LDML_SECTION_KEYS:
dump_kmxplus_keys(data, as_kmxplus_keys(entrydata));
break;
case LDML_SECTION_LOCA:
dump_kmxplus_loca(data, as_kmxplus_loca(entrydata));
break;
case LDML_SECTION_META:
dump_kmxplus_meta(data, as_kmxplus_meta(entrydata));
break;
case LDML_SECTION_SECT:
printf("! Cowardly refusing to dump nested 'sect' section.\n");
break;
case LDML_SECTION_STRS:
dump_kmxplus_strs(data, as_kmxplus_strs(entrydata));
break;
case LDML_SECTION_VKEY:
dump_kmxplus_vkey(data, as_kmxplus_vkey(entrydata));
break;
default:
printf("Unknown section %X", entry.sect);
}
}
}
#endif
void
dump_kmxplus_data(const uint8_t* data) {
#if KMXPLUS_DEBUG
const COMP_KMXPLUS_SECT* sect = as_kmxplus_sect(data);
if (sect == NULL) {
printf("Err: 'sect' null from %p\n", data);
return;
}
dump_kmxplus_sect(data, sect);
#endif
}
void
dump_kmxplus_data(kmx::PCOMP_KEYBOARD keyboard) {
#if KMXPLUS_DEBUG
printf("dump_kmxplus_data(): Got a PCOMP_KEYBOARD at %p\n", keyboard);
if (!(keyboard->dwFlags & KF_KMXPLUS)) {
printf("Err: flags KF_KMXPLUS not set\n");
return;
}
const COMP_KEYBOARD_EX* ex = reinterpret_cast<const COMP_KEYBOARD_EX*>(keyboard);
printf("KMXPlus offset 0x%X, KMXPlus size 0x%X\n", ex->kmxplus.dpKMXPlus, ex->kmxplus.dwKMXPlusSize);
const uint8_t* rawdata = reinterpret_cast<const uint8_t*>(keyboard);
dump_kmxplus_data(rawdata + ex->kmxplus.dpKMXPlus);
#endif
}
const COMP_KMXPLUS_KEYS_ENTRY *COMP_KMXPLUS_KEYS::find(KMX_DWORD vkey, KMX_DWORD mod) const {
// TODO-LDML: eventually, assume sorted order & binary search
for (KMX_DWORD i=0; i<count; i++) {
if(entries[i].vkey == vkey && entries[i].mod == mod) {
return &entries[i];
}
}
return NULL;
}
KMX_DWORD COMP_KMXPLUS_SECT::find(KMX_DWORD ident) const {
for (KMX_DWORD i = 0; i < count; i++) {
if (ident == entries[i].sect) {
return entries[i].offset;
}
}
return 0;
}
PKMX_WCHAR
COMP_KMXPLUS_STRS::get(KMX_DWORD entry, PKMX_WCHAR buf, KMX_DWORD bufsiz) const {
assert(entry < count);
if (entry >= count) {
return NULL;
}
KMX_DWORD offset = entries[entry].offset;
KMX_DWORD length = entries[entry].length;
assert(bufsiz > (length+1)); // assert bufsiz big enough
assert(offset+((length+1)*2) <= header.size); // assert not out of bounds
const uint8_t* thisptr = reinterpret_cast<const uint8_t*>(this);
const KMX_WCHAR* start = reinterpret_cast<const KMX_WCHAR*>(thisptr+offset);
for(KMX_DWORD i=0;i<=length;i++) {
buf[i] = start[i];
}
return buf;
}
} // namespace kmx
} // namespace kbp
} // namespace km

221
core/src/kmx/kmx_plus.h Normal file
View file

@ -0,0 +1,221 @@
/*
Copyright: Copyright (C) 2022 SIL International.
Authors: srl295
This file defines the structure of a kmx_plus file, starting at COMP_KEYBOARD_KMXPLUSINFO.dpKMXPlus
*/
#pragma once
#include <assert.h>
#include <kmx/kmx_file.h>
#include <ldml/keyboardprocessor_ldml.h>
namespace km {
namespace kbp {
namespace kmx {
struct COMP_KMXPLUS_HEADER {
KMX_DWORD ident; // 0000 Section name
KMX_DWORD size; // 0004 Section length
};
// Assert that the length matches the declared length
static_assert(sizeof(struct COMP_KMXPLUS_HEADER) == LDML_LENGTH_HEADER, "mismatched size of section header");
struct COMP_KMXPLUS_SECT_ENTRY {
KMX_DWORD sect; // 0010+ Section identity
KMX_DWORD offset; // 0014+ Section offset relative to dpKMXPlus of section
};
struct COMP_KMXPLUS_SECT {
COMP_KMXPLUS_HEADER header;
KMX_DWORD total; // 0008 KMXPlus entire length
KMX_DWORD count; // 000B number of section headers
COMP_KMXPLUS_SECT_ENTRY entries[0]; // 0010 section entries
/**
* @brief Get the offset of a section, or 0
*
* @param ident section id such as 'strs'. Never 'sect'
* @return KMX_DWORD offset from beginning of kmxplus
*/
KMX_DWORD find(KMX_DWORD ident) const;
};
// Assert that the length matches the declared length
static_assert(sizeof(struct COMP_KMXPLUS_SECT) == LDML_LENGTH_SECT, "mismatched size of section sect");
struct COMP_KMXPLUS_STRS_ENTRY {
KMX_DWORD offset; // 0010+ offset from this blob
KMX_DWORD length; // 0014+ str length (UTF-16LE units)
};
struct COMP_KMXPLUS_STRS {
COMP_KMXPLUS_HEADER header;
KMX_DWORD count; // 0008 count of str entries
KMX_DWORD reserved; // 000C padding
COMP_KMXPLUS_STRS_ENTRY entries[0]; // 0010+ entries
/**
* @brief Get a string entry
*
* @param entry entry number
* @param buf output buffer
* @param bufsiz buffer size in bytes
* @return NULL or a pointer to the output buffer
*/
PKMX_WCHAR get(KMX_DWORD entry, PKMX_WCHAR buf, KMX_DWORD bufsiz) const;
};
static_assert(sizeof(struct COMP_KMXPLUS_STRS) == LDML_LENGTH_STRS, "mismatched size of section strs");
struct COMP_KMXPLUS_META {
COMP_KMXPLUS_HEADER header;
KMX_DWORD name;
KMX_DWORD author;
KMX_DWORD conform;
KMX_DWORD layout;
KMX_DWORD normalization;
KMX_DWORD indicator;
KMX_DWORD settings;
};
static_assert(sizeof(struct COMP_KMXPLUS_META) == LDML_LENGTH_META, "mismatched size of section meta");
struct COMP_KMXPLUS_LOCA_ENTRY {
KMX_DWORD locale; // 000C+ locale string entry
};
struct COMP_KMXPLUS_LOCA {
COMP_KMXPLUS_HEADER header;
KMX_DWORD count; // 0008 number of locales
COMP_KMXPLUS_LOCA_ENTRY entries[0];
};
static_assert(sizeof(struct COMP_KMXPLUS_LOCA) == LDML_LENGTH_LOCA, "mismatched size of section loca");
struct COMP_KMXPLUS_KEYS_ENTRY {
KMX_DWORD vkey;
KMX_DWORD mod;
KMX_DWORD to;
KMX_DWORD flags;
};
struct COMP_KMXPLUS_KEYS {
COMP_KMXPLUS_HEADER header;
KMX_DWORD count; // number of keys
KMX_DWORD reserved; // padding
COMP_KMXPLUS_KEYS_ENTRY entries[0];
const COMP_KMXPLUS_KEYS_ENTRY *find(KMX_DWORD vkey, KMX_DWORD mod) const;
};
static_assert(sizeof(struct COMP_KMXPLUS_KEYS) == LDML_LENGTH_KEYS, "mismatched size of section keys");
struct COMP_KMXPLUS_VKEY_ENTRY {
KMX_DWORD vkey;
KMX_DWORD target;
};
struct COMP_KMXPLUS_VKEY {
COMP_KMXPLUS_HEADER header;
KMX_DWORD count;
COMP_KMXPLUS_VKEY_ENTRY entries[0];
};
static_assert(sizeof(struct COMP_KMXPLUS_VKEY) == LDML_LENGTH_VKEY, "mismatched size of section vkey");
/**
* @brief Validate that this data is the named section.
*
* @param data raw data
* @param ident 4-byte section type
* @return COMP_KMXPLUS_ALLDATA* or null
*/
static inline const COMP_KMXPLUS_HEADER *
validate_as_section(const uint8_t *data, uint32_t ident) {
if (!data) {
return NULL;
}
const COMP_KMXPLUS_HEADER *all = reinterpret_cast<const COMP_KMXPLUS_HEADER *>(data);
// TODO-LDML these fail on 000null ..
assert(all->size >= LDML_LENGTH_HEADER);
assert(ident == all->ident);
if (ident != all->ident || (all->size < LDML_LENGTH_HEADER)) {
return NULL; // invalid header or wrong section
}
return all;
}
/**
* cast raw data to section
* @return section data or null on error
*/
static inline const COMP_KMXPLUS_SECT *
as_kmxplus_sect(const uint8_t *data) {
const COMP_KMXPLUS_HEADER *all = validate_as_section(data, LDML_SECTION_SECT);
return reinterpret_cast<const COMP_KMXPLUS_SECT *>(all);
}
/**
* cast raw data to section
* @return section data or null on error
*/
static inline const COMP_KMXPLUS_STRS *
as_kmxplus_strs(const uint8_t *data) {
const COMP_KMXPLUS_HEADER *all = validate_as_section(data, LDML_SECTION_STRS);
return reinterpret_cast<const COMP_KMXPLUS_STRS *>(all);
}
/**
* cast raw data to section
* @return section data or null on error
*/
static inline const COMP_KMXPLUS_KEYS *
as_kmxplus_keys(const uint8_t *data) {
const COMP_KMXPLUS_HEADER *all = validate_as_section(data, LDML_SECTION_KEYS);
return reinterpret_cast<const COMP_KMXPLUS_KEYS *>(all);
}
/**
* cast raw data to section
* @return section data or null on error
*/
static inline const COMP_KMXPLUS_LOCA *
as_kmxplus_loca(const uint8_t *data) {
const COMP_KMXPLUS_HEADER *all = validate_as_section(data, LDML_SECTION_LOCA);
return reinterpret_cast<const COMP_KMXPLUS_LOCA *>(all);
}
/**
* cast raw data to section
* @return section data or null on error
*/
static inline const COMP_KMXPLUS_META *
as_kmxplus_meta(const uint8_t *data) {
const COMP_KMXPLUS_HEADER *all = validate_as_section(data, LDML_SECTION_META);
return reinterpret_cast<const COMP_KMXPLUS_META *>(all);
}
/**
* convert raw data to section
* @return section data or null on error
*/
static inline const COMP_KMXPLUS_VKEY *
as_kmxplus_vkey(const uint8_t *data) {
const COMP_KMXPLUS_HEADER *all = validate_as_section(data, LDML_SECTION_VKEY);
return reinterpret_cast<const COMP_KMXPLUS_VKEY *>(all);
}
/**
* @brief Temporary function to dump raw data
* May have no output if KMXPLUS_DEBUG is 0
* @param kmxplusdata data from the beginning of the KMXPlus section
*/
void dump_kmxplus_data(void *kmxplusdata);
/**
* @brief Temporary functino to dump raw data
* May have no output if KMXPLUS_DEBUG is 0
* @param keyboard pointer to PCOMP_KEYBOARD with plus data following
*/
void dump_kmxplus_data(kmx::PCOMP_KEYBOARD keyboard);
} // namespace kmx
} // namespace kbp
} // namespace km

165
core/src/ldml/C7043_ldml.md Normal file
View file

@ -0,0 +1,165 @@
# KMX Plus Binary Format #7043
- copied from <https://github.com/keymanapp/keyman/issues/7043>
- Authors: MD SL
## C7043.0 Introduction
This document discusses the binary format for KMXPlus, which contains keyboards converted from source LDML data.
Draft spec PR: <https://github.com/unicode-org/cldr/pull/1847>
## C7043.1 Principles
- The data described here is located at byte offset `dpKMXPlus`.
- All integer values are unsigned 32-bit little-endian unless otherwise specified.
- All strings are UTF-16LE unless otherwise specified. (See the `strs` section.) String data items are identified with `str:`, indicating a 32 bit index into the `strs` table.
- All offsets are 32-bit little-endian values. For all sections except for the `'sect'` section (which see), offsets are relative to the beginning of each section.
## C7043.2 Sections
- Data is divided into several sections. The very first section is the `'sect'` section which is the table of contents.
- All sections, including the first, begin on a 128-bit boundary with zero padding as needed.
- All sections begin with a 32-bit (four character) section identifier, and a 32-bit section size.
- Other than the `sect` table itself, the rest of the sections follow in binary order in the file. In other words, the binary ordering of the section identifiers determines the order of the file layout.
### C7043.2.1 `sect`—Section Table of contents
The very first section is a table of contents listing the rest of the sections. The table of contents does not list itself.
This is the only section where all byte offsets are relative to the value of `dpKMXPlus`.
| ∆ | Bits | Name | Description |
|---|------|---------|------------------------------------------|
| 0 | 32 | ident | `sect` |
| 4 | 32 | size | int: Length of section |
| 8 | 32 | total | int: KMXPlus entire length |
|12 | 32 | count | int: Number of following section headers |
Then for `count` repetitions:
| ∆ | Bits | Name | Description |
|---|------|---------|---------------------------------------------------|
|16+| 32 | sect | Section identity, e.g. `loca` |
|20+| 32 | offset | off: offset relative to `dpKMXPlus` of section |
This list is in sorted order based on the `sect` identifier.
### C7043.2.2 `strs`—Strings
All strings are stored in the Strings section.
| ∆ | Bits | Name | Description |
|---|------|---------------|-------------------------------------|
| 0 | 32 | ident | `strs` |
| 4 | 32 | size | int: Length of section |
| 8 | 32 | count | int: Number of strings |
|12 | 32 | reserved | Padding |
Then for each string:
| ∆ | Bits | Name | Description |
|---|------|---------------|-----------------------------------------------|
|16+| 32 | offset | off: Offset to string |
|20+| 32 | length | int: Length of string in UTF-16LE code units |
After the string offset table comes the actual UTF-16LE data. There is a null (\u0000) after each string, which is _not_ included in the string length.
The string offset table, and then strings themselves, are sorted according to a binary codepoint sort, not including the null.
### C7043.2.3 `meta`—Metadata
| ∆ | Bits | Name | Description |
|---|------|---------------|-------------------------------------|
| 0 | 32 | ident | `meta` |
| 4 | 32 | size | int: Length of section |
| 8 | 32 | name | str: Keyboard name |
|12 | 32 | author | str: Keyboard author |
|16 | 32 | conform | str: CLDR 'conformsTo' version |
|20 | 32 | layout | str: layout type |
|24 | 32 | normalization | str: normalization mode |
|28 | 32 | indicator | str: indicator |
|32 | 32 | settings | int: keyboard settings |
The `settings` is a 32-bit bitfield as below:
| Bit position | Meaning | Description |
|--------------|----------|---------------------------------------------|
| 0 | fallback | fallback=omit |
| 1 | transformFailure | transformFailure=omit |
| 2 | transformPartial | transformPartial=hide |
### C7043.2.4 `loca`—Locales
| ∆ | Bits | Name | Description |
|---|------|---------|------------------------------------------|
| 0 | 32 | ident | `loca` |
| 4 | 32 | size | int: Length of section |
| 8 | 32 | count | int: Number of locales |
`count` is always ≥1, because a keyboard always has a primary locale identifier.
For each locale ID in `count`
| ∆ | Bits | Name | Description |
|---|------|---------|------------------------------------------|
|12+| 32 | locale | str: Locale ID in BCP47 format |
The first locale ID is always the primary locale identifier. The rest of the locale IDs (starting at offset 16) are in sorted binary order.
### C7043.2.5 `keys`—Keybag
| ∆ | Bits | Name | Description |
|---|------|---------|------------------------------------------|
| 0 | 32 | ident | `keys` |
| 4 | 32 | size | int: Length of section |
| 8 | 32 | count | int: Number of keys |
|12 | 32 | reserved | reserved |
The keys are sorted in binary order based on the `vkey` and `mod` fields.
For each key:
| ∆ | Bits | Name | Description |
|---|------|---------|------------------------------------------|
|16+| 32 | vkey | int: vkey ID |
|20+| 32 | mod | int: modifier key flags |
|24+| 32 | to | str: output string OR UTF-32LE codepoint |
|28+| 32 | flags | int: per-key flags |
- `vkey`: If this is 0-255, it is the resolved standard/predefined vkey (K_A, etc.). It is resolved because the `vkeyMap` from LDML has already been applied. If this is 256 or above, it is a custom touch layout vkey generated by the compiler.
- `mod`: TODO define this. 0 for no modifiers.
- `flags`: Flags is a 32-bit bitfield defined as below:
| Bit position | Meaning | Description |
|--------------|----------|---------------------------------------------|
| 0 | extend | 0: `to` is a char, 1: `to` is a string |
- `to`: If `extend` is 0, `to` is a UTF-32LE codepoint. If `extend` is 1, `to` is a 32 bit index into the `strs` table.
### C7043.2.6 `vkey`—VKey Map
| ∆ | Bits | Name | Description |
|---|------|---------|------------------------------------------|
| 0 | 32 | ident | `vkey` |
| 4 | 32 | size | int: Length of section |
| 8 | 32 | count | int: Number of vkeys |
The keys are sorted in binary order based on the `vkey` field.
For each key:
| ∆ | Bits | Name | Description |
|---|------|---------|------------------------------------------|
|12+| 32 | vkey | int: source vkey ID (0…255) |
|16+| 32 | target | int: target vkey ID (0…255) |
- `vkey`: Is the standard vkey, 0-255
- `target`: Is the target (resolved) vkey, 0-255.
### C7043.2.7 Transforms and friends
> TODO: transforms

107
core/src/ldml/ldml_processor.cpp
View file

@ -9,8 +9,41 @@
#include "ldml/ldml_processor.hpp"
#include "state.hpp"
#include "../kmx/kmx_file.h"
#include "../kmx/kmx_plus.h"
#include "ldml/keyboardprocessor_ldml.h"
// extern "C" {
// #include "../../../common/windows/cpp/src/ConvertUTF.c"
// }
// HACK
enum ConversionResult {
conversionOK,
conversionNotOk
};
enum ConversionFlags {
strictConversion
};
typedef KMX_WCHAR UTF16;
typedef KMX_DWORD UTF32;
ConversionResult ConvertUTF32toUTF16(
UTF32** sourceStart, const UTF32* sourceEnd,
UTF16** targetStart, const UTF16* targetEnd, const ConversionFlags /*flags*/) {
if(sourceEnd > (*sourceStart+1)) return conversionNotOk; // Don't support >1 char
if(**sourceStart & 0xFFFF0000) {
// Don't support supplemental chars yet
return conversionNotOk;
}
*((*targetStart)++) = (UTF16)((*(*sourceStart)++) & 0xFFFF); // BMP
assert(*sourceStart==sourceEnd);
assert(*targetStart <= targetEnd);
return(conversionOK);
}
namespace {
constexpr km_kbp_attr const engine_attrs = {
256,
@ -25,13 +58,14 @@ namespace {
namespace km {
namespace kbp {
ldml_processor::ldml_processor(path const & kb_path, const std::vector<uint8_t> _kmn_unused(data))
ldml_processor::ldml_processor(path const & kb_path, const std::vector<uint8_t> &data)
: abstract_processor(
keyboard_attributes(kb_path.stem(), KM_KBP_LMDL_PROCESSOR_VERSION, kb_path.parent(), {})
)
), rawdata(data) // TODO-LDML: parse the data, don't just copy it
{
// TODO-LDML: load the file from the buffer (KMXPlus format)
// Note: kb_path is essentially debug metadata here
assert(data.size() != 0);
}
bool ldml_processor::is_kmxplus_file(path const & kb_path, std::vector<uint8_t>& data) {
@ -50,7 +84,7 @@ bool ldml_processor::is_kmxplus_file(path const & kb_path, std::vector<uint8_t>&
file.seekg(0, std::ios::beg);
data.reserve((size_t)size);
data.resize((size_t)size);
if(!file.read((char *) data.data(), size)) {
return false;
}
@ -67,6 +101,9 @@ bool ldml_processor::is_kmxplus_file(path const & kb_path, std::vector<uint8_t>&
return false;
}
// Dump data. This also does some validation.
dump_kmxplus_data(comp_keyboard);
// A KMXPlus file is in the buffer (although more validation is required)
return true;
}
@ -98,7 +135,7 @@ km_kbp_status
ldml_processor::process_event(
km_kbp_state *state,
km_kbp_virtual_key vk,
uint16_t _kmn_unused(modifier_state),
uint16_t modifier_state,
uint8_t is_key_down,
uint16_t /*event_flags*/ // TODO-LDML: unused... for now...
) {
@ -123,9 +160,65 @@ ldml_processor::process_event(
state->actions().push_backspace(KM_KBP_BT_UNKNOWN); // Assuming we don't know the character
break;
default:
/* We're going to push an 'a' for a passing unit test */
state->context().push_character('a');
state->actions().push_character('a');
// TODO-LDML: temporary code here
// Don't want to do this work each time
const kmx::PCOMP_KEYBOARD comp_keyboard = (kmx::PCOMP_KEYBOARD)rawdata.data();
assert(comp_keyboard->dwFlags & KF_KMXPLUS);
const kmx::COMP_KEYBOARD_EX* ex = reinterpret_cast<const kmx::COMP_KEYBOARD_EX*>(comp_keyboard);
// printf("KMXPlus offset 0x%X, KMXPlus size 0x%X\n", ex->kmxplus.dpKMXPlus, ex->kmxplus.dwKMXPlusSize);
const uint8_t* kmxplusdata = rawdata.data() + ex->kmxplus.dpKMXPlus;
// Get out the SECT header
const kmx::COMP_KMXPLUS_SECT *sect = kmx::as_kmxplus_sect(kmxplusdata);
assert(sect != NULL);
assert(sect->header.ident == LDML_SECTION_SECT);
KMX_DWORD offset;
// Fill out the other sections we need.
offset = sect->find(LDML_SECTION_STRS);
assert(offset != 0); // or else section not found
const kmx::COMP_KMXPLUS_STRS *strs = kmx::as_kmxplus_strs(kmxplusdata+offset);
assert(strs->header.ident == LDML_SECTION_STRS);
offset = sect->find(LDML_SECTION_KEYS);
assert(offset != 0); // or else section not found
const kmx::COMP_KMXPLUS_KEYS *keys = kmx::as_kmxplus_keys(kmxplusdata+offset);
assert(keys->header.ident == LDML_SECTION_KEYS);
// Look up the key
const kmx::COMP_KMXPLUS_KEYS_ENTRY *key = keys->find(vk, modifier_state);
assert(key != NULL);
if (!key) {
return KM_KBP_STATUS_KEY_ERROR;
}
// Prepare output chars
KMX_DWORD len = 0;
KMX_WCHAR out[BUFSIZ];
if (key->flags && LDML_KEYS_FLAGS_EXTEND) {
// It's a string.
assert(NULL != strs->get(key->to, out, BUFSIZ));
// u_strlen()
for(len=0; len<BUFSIZ && out[len]; len++);
} else {
UTF32 buf32[2];
buf32[0] = key->to; // UTF-32
buf32[1] = 0; // to avoid UMR warning
UTF32 *sourceStart = &buf32[0];
const UTF32 *sourceEnd = &buf32[1]; // Reference off the end. NULL to avoid UMR.
UTF16 *targetStart = (UTF16*)out;
const UTF16 *targetEnd = (UTF16*)out+BUFSIZ-1;
ConversionResult result = ::ConvertUTF32toUTF16(&sourceStart, sourceEnd, &targetStart, targetEnd, strictConversion);
assert(result == conversionOK);
*targetStart = 0;
len = 1; // TODO=LDML calculate
// len = (targetStart - out);
assert(len>=1 && len <= 2);
}
assert(len>=0);
assert(len<BUFSIZ);
assert(out[len] == 0); // null termination
for(KMX_DWORD i=0; i<len; i++) {
state->context().push_character(out[i]);
state->actions().push_character(out[i]);
}
}
state->actions().commit();

6
core/src/ldml/ldml_processor.hpp
View file

@ -11,7 +11,6 @@
#include <vector>
#include <unordered_map>
#include <keyman/keyboardprocessor.h>
#include "processor.hpp"
#include "option.hpp"
@ -21,11 +20,12 @@ namespace kbp {
#define KM_KBP_LMDL_PROCESSOR_VERSION u"1.0"
class ldml_processor : public abstract_processor {
private:
std::vector<uint8_t> rawdata; // TODO-LDML: should be 'unpacked' format instead.
public:
ldml_processor(
path const & kb_path,
const std::vector<uint8_t> data
const std::vector<uint8_t> & data
);
// ~ldml_processor() override;

1
core/src/meson.build
View file

@ -83,6 +83,7 @@ lib = library('kmnkbp0',
'kmx/kmx_file.cpp',
'kmx/kmx_modifiers.cpp',
'kmx/kmx_options.cpp',
'kmx/kmx_plus.cpp',
'kmx/kmx_processor.cpp',
'kmx/kmx_xstring.cpp',
'utfcodec.cpp',

BIN
core/tests/unit/ldml/001_tiny.kmx
View file

Binary file not shown.

11
core/tests/unit/ldml/meson.build
View file

@ -33,7 +33,7 @@ ldml = executable('ldml',
objects: lib.extract_all_objects())
tests = [
'000_null_keyboard',
# '000_null_keyboard',
'001_tiny'
]
@ -166,3 +166,12 @@ else
test(kbd, ldml, args: [kbd_src, kbd_obj])
endforeach
endif
e = executable('test_kmx_plus', 'test_kmx_plus.cpp',
cpp_args: defns + warns,
include_directories: [inc, libsrc],
link_args: links + tests_flags,
objects: lib.extract_all_objects())
test('test_kmx_plus', e)

10
core/tests/unit/ldml/test_kmx_plus.cpp Normal file
View file

@ -0,0 +1,10 @@
#include <stdio.h>
#include <assert.h>
#include <kmx/kmx_plus.h>
using namespace km::kbp::kmx;
int main(int argc, const char *argv[]) {
printf("OKAY\n");
return 0;
}