diff --git a/core/include/ldml/keyboardprocessor_ldml.h b/core/include/ldml/keyboardprocessor_ldml.h index dd77344ce8..71e5d0dd1c 100644 --- a/core/include/ldml/keyboardprocessor_ldml.h +++ b/core/include/ldml/keyboardprocessor_ldml.h @@ -22,6 +22,8 @@ #define LDML_LENGTH_LOCA 0x10 #define LDML_LENGTH_LOCA_ITEM 0x4 #define LDML_LENGTH_META 0x28 +#define LDML_LENGTH_NAME 0x10 +#define LDML_LENGTH_NAME_ITEM 0x4 #define LDML_LENGTH_SECT 0x10 #define LDML_LENGTH_SECT_ITEM 0x8 #define LDML_LENGTH_STRS 0x10 @@ -37,6 +39,8 @@ #define LDML_SECTIONNAME_LOCA "loca" #define LDML_SECTIONID_META 0x6174656D /* "meta" */ #define LDML_SECTIONNAME_META "meta" +#define LDML_SECTIONID_NAME 0x656D616E /* "name" */ +#define LDML_SECTIONNAME_NAME "name" #define LDML_SECTIONID_SECT 0x74636573 /* "sect" */ #define LDML_SECTIONNAME_SECT "sect" #define LDML_SECTIONID_STRS 0x73727473 /* "strs" */ diff --git a/core/include/ldml/keyboardprocessor_ldml.ts b/core/include/ldml/keyboardprocessor_ldml.ts index 446c191372..ecb332f618 100644 --- a/core/include/ldml/keyboardprocessor_ldml.ts +++ b/core/include/ldml/keyboardprocessor_ldml.ts @@ -1,18 +1,32 @@ /* Copyright: Copyright (C) 2022 SIL International. - Authors: srl295 + Authors: srl295, mcdurdin This file provides constants for the KMX Plus (LDML support) binary format, to be shared between TypeScript and C++ via the generator (below) */ -// This defines the section identifiers and ensures that we include each and -// every one of them in the `sections` block and gives us a type which we can -// iterate through +// NOTICE! +// +// If you update this file, you *must* be sure to re-run +// +// core/tools/ldml-const-builder/build.sh clean build run +// +// To update keyboardprocessor_ldml.h, and commit the result. +// +// It is not updated automatically. + + +/** + * Defines the section identifiers and ensures that we include each and every + * one of them in the `sections` block and gives us a type which we can iterate + * through. + */ export type SectionIdent = 'keys' | 'loca' | 'meta' | + 'name' | 'sect' | 'strs' | 'vkey'; @@ -22,22 +36,12 @@ type SectionMap = { } interface Constants_Section { - // For now, only defining one property as other types - // can be inferred + // For now, only defining one property as other types can be inferred section: SectionMap; } type Constants = Constants_Section & {[id:string]:any}; -// Notice! -// -// If you update this file, you *must* be sure to re-run -// -// core/tools/ldml-const-builder/build.sh clean build run -// -// To update keyboardprocessor_ldml.h, and commit the result. -// -// It is not updated automatically. // TODO-LDML: namespace com.keyman.core.ldml { /** @@ -57,26 +61,57 @@ export const constants: Constants = { */ length_header: 8, + /* ------------------------------------------------------------------ + * sect section + ------------------------------------------------------------------ */ + /** * Minimum length of the 'sect' section, not including entries */ length_sect: 16, + /** + * Length of each item in the 'sect' section variable part + */ length_sect_item: 8, - /** - * Minimum length of the 'keys' section - * not including variable parts - */ - length_keys: 16, - length_keys_item: 16, + /* ------------------------------------------------------------------ + * keys section + ------------------------------------------------------------------ */ /** - * Minimum length of the 'loca' section - * not including variable parts + * Minimum length of the 'keys' section not including variable parts + */ + length_keys: 16, + /** + * Length of each item in the 'keys' section variable part + */ + length_keys_item: 16, + /** + * bitwise or value for extend in keys[key].flags. + * If bit is 1, then 'to' is a string. + * If bit is 0, then 'to' is a UTF-32LE codepoint + * + * `extend = flags & keys_flags_extend` + */ + keys_flags_extend: 1, + + /* ------------------------------------------------------------------ + * loca section + ------------------------------------------------------------------ */ + + /** + * Minimum length of the 'loca' section not including variable parts */ length_loca: 16, + /** + * Length of each item in the 'loca' section variable part + */ length_loca_item: 4, + /* ------------------------------------------------------------------ + * meta section + ------------------------------------------------------------------ */ + /** * length of the 'meta' section */ @@ -94,25 +129,36 @@ export const constants: Constants = { */ meta_settings_transformPartial_hide: 4, + /* ------------------------------------------------------------------ + * name section + ------------------------------------------------------------------ */ + /** - * Minimum length of the 'strs' section - * not including variable parts + * Minimum length of the 'name' section not including variable parts + */ + length_name: 16, + /** + * Length of each item in the 'name' section variable part + */ + length_name_item: 4, + + /* ------------------------------------------------------------------ + * strs section + ------------------------------------------------------------------ */ + + /** + * Minimum length of the 'strs' section not including variable parts */ length_strs: 16, /** - * Length of each item in the 'strs' section - * variable part + * Length of each item in the 'strs' section variable part */ length_strs_item: 8, - /** - * bitwise or value for extend in keys[key].flags. - * If bit is 1, then 'to' is a string. - * If bit is 0, then 'to' is a UTF-32LE codepoint - * - * `extend = flags & keys_flags_extend` - */ - keys_flags_extend: 1, + /* ------------------------------------------------------------------ + * vkey section + ------------------------------------------------------------------ */ + /** * Minimum length of the 'vkey' section * not including variable parts @@ -127,6 +173,7 @@ export const constants: Constants = { keys: 'keys', loca: 'loca', meta: 'meta', + name: 'name', sect: 'sect', strs: 'strs', vkey: 'vkey', diff --git a/core/src/kmx/kmx_plus.h b/core/src/kmx/kmx_plus.h index 747a72b524..54c04d71aa 100644 --- a/core/src/kmx/kmx_plus.h +++ b/core/src/kmx/kmx_plus.h @@ -45,6 +45,10 @@ struct COMP_KMXPLUS_HEADER { // Assert that the length matches the declared length static_assert(sizeof(struct COMP_KMXPLUS_HEADER) == LDML_LENGTH_HEADER, "mismatched size of section header"); +/* ------------------------------------------------------------------ + * sect section + ------------------------------------------------------------------ */ + struct COMP_KMXPLUS_SECT_ENTRY { KMX_DWORD sect; // 0010+ Section identity KMX_DWORD offset; // 0014+ Section offset relative to dpKMXPlus of section @@ -74,6 +78,10 @@ struct COMP_KMXPLUS_SECT { static_assert(sizeof(struct COMP_KMXPLUS_SECT) == LDML_LENGTH_SECT, "mismatched size of section sect"); static_assert(sizeof(struct COMP_KMXPLUS_SECT) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); +/* ------------------------------------------------------------------ + * strs section + ------------------------------------------------------------------ */ + struct COMP_KMXPLUS_STRS_ENTRY { KMX_DWORD offset; // 0010+ offset from this blob KMX_DWORD length; // 0014+ str length (UTF-16LE units) @@ -104,6 +112,10 @@ struct COMP_KMXPLUS_STRS { static_assert(sizeof(struct COMP_KMXPLUS_STRS) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); static_assert(sizeof(struct COMP_KMXPLUS_STRS) == LDML_LENGTH_STRS, "mismatched size of section strs"); +/* ------------------------------------------------------------------ + * meta section + ------------------------------------------------------------------ */ + struct COMP_KMXPLUS_META { static const KMX_DWORD IDENT = LDML_SECTIONID_META; COMP_KMXPLUS_HEADER header; @@ -123,6 +135,10 @@ struct COMP_KMXPLUS_META { static_assert(sizeof(struct COMP_KMXPLUS_META) == LDML_LENGTH_META, "mismatched size of section meta"); +/* ------------------------------------------------------------------ + * loca section + ------------------------------------------------------------------ */ + struct COMP_KMXPLUS_LOCA_ENTRY { KMXPLUS_STR locale; // 0010+ locale string entry }; @@ -142,6 +158,10 @@ struct COMP_KMXPLUS_LOCA { static_assert(sizeof(struct COMP_KMXPLUS_LOCA) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); static_assert(sizeof(struct COMP_KMXPLUS_LOCA) == LDML_LENGTH_LOCA, "mismatched size of section loca"); +/* ------------------------------------------------------------------ + * keys section + ------------------------------------------------------------------ */ + struct COMP_KMXPLUS_KEYS_ENTRY { KMX_DWORD vkey; KMX_DWORD mod; @@ -164,6 +184,10 @@ struct COMP_KMXPLUS_KEYS { static_assert(sizeof(struct COMP_KMXPLUS_KEYS) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); static_assert(sizeof(struct COMP_KMXPLUS_KEYS) == LDML_LENGTH_KEYS, "mismatched size of section keys"); +/* ------------------------------------------------------------------ + * vkey section + ------------------------------------------------------------------ */ + struct COMP_KMXPLUS_VKEY_ENTRY { KMX_DWORD vkey; KMX_DWORD target; @@ -184,6 +208,30 @@ struct COMP_KMXPLUS_VKEY { static_assert(sizeof(struct COMP_KMXPLUS_VKEY) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); static_assert(sizeof(struct COMP_KMXPLUS_VKEY) == LDML_LENGTH_VKEY, "mismatched size of section vkey"); +/* ------------------------------------------------------------------ + * name section + ------------------------------------------------------------------ */ + +struct COMP_KMXPLUS_NAME_ENTRY { + KMXPLUS_STR name; +}; + +struct COMP_KMXPLUS_NAME { + static const KMX_DWORD IDENT = LDML_SECTIONID_NAME; + COMP_KMXPLUS_HEADER header; + KMX_DWORD count; + KMX_DWORD reserved; + COMP_KMXPLUS_NAME_ENTRY entries[]; + /** + * @brief True if section is valid. + */ + bool valid(KMX_DWORD length) const; +}; + +static_assert(sizeof(struct COMP_KMXPLUS_NAME) % 0x10 == 0, "Structs prior to entries[] should align to 128-bit boundary"); +static_assert(sizeof(struct COMP_KMXPLUS_NAME) == LDML_LENGTH_NAME, "mismatched size of section name"); + + /** * @brief helper accessor object for * diff --git a/core/src/ldml/C7043_ldml.md b/core/src/ldml/C7043_ldml.md index 9aaf39deb5..7ff4fe9dcd 100644 --- a/core/src/ldml/C7043_ldml.md +++ b/core/src/ldml/C7043_ldml.md @@ -196,6 +196,31 @@ For each key: - `vkey`: Is the standard vkey, 0-255 - `target`: Is the target (resolved) vkey, 0-255. -### C7043.2.7 Transforms and friends +### C7043.2.7 `name`—Names + +Defines the names of the keyboard as found in the source `` element. +While this section is optional in the binary format, in practice it will always +be present, as the source format requires at least one name. + +| ∆ | Bits | Name | Description | +|---|------|---------|------------------------------------------| +| 0 | 32 | ident | `name` | +| 4 | 32 | size | int: Length of section | +| 8 | 32 | count | int: Number of names | +|12 | 32 | reserved| padding | + +Note that `count` is always ≥1, as the source format requires at least one name. + +For each name in `count`: + +| ∆ | Bits | Name | Description | +|---|------|---------|------------------------------------------| +|16+| 32 | name | str: A name for the keyboard | + +Note that the first name is repeated in the `meta` section. The remaining names +are stored in source file order, and the semantic meaning of each name is not +defined here. + +### C7043.2.8 Transforms and friends > TODO: transforms diff --git a/developer/src/kmcmpdll/kcframe/kcframe.vcxproj b/developer/src/kmcmpdll/kcframe/kcframe.vcxproj index cc92af4fc1..24cb578d73 100644 --- a/developer/src/kmcmpdll/kcframe/kcframe.vcxproj +++ b/developer/src/kmcmpdll/kcframe/kcframe.vcxproj @@ -114,7 +114,7 @@ MaxSpeed OnlyExplicitInline - .;..\..\..\global\inc;%(AdditionalIncludeDirectories) + .;..;%(AdditionalIncludeDirectories) WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true MultiThreaded @@ -149,7 +149,7 @@ MaxSpeed OnlyExplicitInline - .;..\..\..\global\inc;%(AdditionalIncludeDirectories) + .;..;%(AdditionalIncludeDirectories) WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true MultiThreaded @@ -182,7 +182,7 @@ Disabled - .;..\..\..\global\inc;%(AdditionalIncludeDirectories) + .;..;%(AdditionalIncludeDirectories) WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true EnableFastChecks @@ -218,7 +218,7 @@ Disabled - .;..\..\..\global\inc;%(AdditionalIncludeDirectories) + .;..;%(AdditionalIncludeDirectories) WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) EnableFastChecks MultiThreadedDebug diff --git a/developer/src/kmldmlc/src/keyman/compiler/compiler.ts b/developer/src/kmldmlc/src/keyman/compiler/compiler.ts index b1251a80ae..811b04c11f 100644 --- a/developer/src/kmldmlc/src/keyman/compiler/compiler.ts +++ b/developer/src/kmldmlc/src/keyman/compiler/compiler.ts @@ -6,11 +6,13 @@ import CompilerCallbacks from './callbacks'; import { KeysCompiler } from './keys'; import { LocaCompiler } from './loca'; import { MetaCompiler } from './meta'; +import { NameCompiler } from './name'; const SECTION_COMPILERS = [ MetaCompiler, LocaCompiler, KeysCompiler, + NameCompiler ]; export default class Compiler { diff --git a/developer/src/kmldmlc/src/keyman/compiler/name.ts b/developer/src/kmldmlc/src/keyman/compiler/name.ts new file mode 100644 index 0000000000..6a16ac5309 --- /dev/null +++ b/developer/src/kmldmlc/src/keyman/compiler/name.ts @@ -0,0 +1,22 @@ +import { constants } from "@keymanapp/ldml-keyboard-constants"; +import { Name } from "../kmx/kmx-plus"; +import { SectionCompiler } from "./section-compiler"; + +export class NameCompiler extends SectionCompiler { + + public get id() { + return constants.section.name; + } + + public validate(): boolean { + let valid = true; + valid = (this.keyboard.names?.name?.length ?? 0) > 0; + return valid; + } + + public compile(): Name { + let result = new Name(); + result.names = this.keyboard.names?.name?.map(v => v.value) ?? []; + return result; + } +} diff --git a/developer/src/kmldmlc/src/keyman/kmx/kmx-plus-builder.ts b/developer/src/kmldmlc/src/keyman/kmx/kmx-plus-builder.ts index f325c9397f..d6c7b477f9 100644 --- a/developer/src/kmldmlc/src/keyman/kmx/kmx-plus-builder.ts +++ b/developer/src/kmldmlc/src/keyman/kmx/kmx-plus-builder.ts @@ -45,6 +45,12 @@ interface BUILDER_META extends BUILDER_SECTION { settings: number; //bitfield }; +interface BUILDER_NAME extends BUILDER_SECTION { + count: number; + reserved: number; + items: number[]; //str[] +}; + interface BUILDER_LOCA extends BUILDER_SECTION { count: number; reserved: number; @@ -88,6 +94,7 @@ export default class KMXPlusBuilder { private sect_sect: BUILDER_SECT; private sect_strs: BUILDER_STRS; private sect_meta: BUILDER_META; + private sect_name: BUILDER_NAME; private sect_loca: BUILDER_LOCA; private sect_keys: BUILDER_KEYS; private sect_vkey: BUILDER_VKEY; @@ -170,6 +177,28 @@ export default class KMXPlusBuilder { return loca; } + private build_name(): BUILDER_NAME { + if(!this.file.kmxplus.name.names.length) { + return null; + } + + let name: BUILDER_NAME = { + ident: constants.hex_section_id(constants.section.name), + size: constants.length_name + constants.length_name_item * this.file.kmxplus.name.names.length, + _offset: 0, + count: this.file.kmxplus.name.names.length, + reserved: 0, + items: [] + }; + + for(let item of this.file.kmxplus.name.names) { + name.items.push(this.alloc_string(item)); + } + + return name; + } + + private build_keys(): BUILDER_KEYS { if(!this.file.kmxplus.keys.keys.length) { return null; @@ -255,12 +284,16 @@ export default class KMXPlusBuilder { if(this.sect_vkey) { this.sect_sect.count++; } + if(this.sect_name) { + this.sect_sect.count++; + } this.sect_sect.size = constants.length_sect + constants.length_sect_item * this.sect_sect.count; let offset = this.sect_sect.size; offset = this.finalize_sect_item(this.sect_strs, offset); offset = this.finalize_sect_item(this.sect_meta, offset); + offset = this.finalize_sect_item(this.sect_name, offset); offset = this.finalize_sect_item(this.sect_loca, offset); offset = this.finalize_sect_item(this.sect_keys, offset); offset = this.finalize_sect_item(this.sect_vkey, offset); @@ -278,6 +311,7 @@ export default class KMXPlusBuilder { this.alloc_string(''); this.sect_meta = this.build_meta(); + this.sect_name = this.build_name(); this.sect_loca = this.build_loca(); this.sect_keys = this.build_keys(); this.sect_vkey = this.build_vkey(); @@ -317,6 +351,7 @@ export default class KMXPlusBuilder { this.emitSection(file, this.file.COMP_PLUS_STRS, this.sect_strs); this.emitStrings(file); this.emitSection(file, this.file.COMP_PLUS_META, this.sect_meta); + this.emitSection(file, this.file.COMP_PLUS_NAME, this.sect_name); this.emitSection(file, this.file.COMP_PLUS_LOCA, this.sect_loca); this.emitSection(file, this.file.COMP_PLUS_KEYS, this.sect_keys); this.emitSection(file, this.file.COMP_PLUS_VKEY, this.sect_vkey); diff --git a/developer/src/kmldmlc/src/keyman/kmx/kmx-plus.ts b/developer/src/kmldmlc/src/keyman/kmx/kmx-plus.ts index 7d53b6261e..ee23717001 100644 --- a/developer/src/kmldmlc/src/keyman/kmx/kmx-plus.ts +++ b/developer/src/kmldmlc/src/keyman/kmx/kmx-plus.ts @@ -33,6 +33,10 @@ export class Loca extends Section { locales: string[] = []; }; +export class Name extends Section { + names: string[] = []; +}; + export enum KeyFlags { none = 0, extend = 1<<0, // note, this should be used only when streaming, ignored in-memory @@ -71,6 +75,9 @@ export default class KMXPlusFile extends KMXFile { public readonly COMP_PLUS_META: any; + public readonly COMP_PLUS_NAME_ITEM: any; + public readonly COMP_PLUS_NAME: any; + public readonly COMP_PLUS_LOCA_ITEM: any; public readonly COMP_PLUS_LOCA: any; @@ -87,6 +94,7 @@ export default class KMXPlusFile extends KMXFile { strs?: Section; // strs is ignored here for writing loca?: Loca; meta?: Meta; + name?: Name; keys?: Keys; vkey?: Vkey; } = { }; @@ -138,6 +146,16 @@ export default class KMXPlusFile extends KMXFile { settings: r.uint32le, //new r.Bitfield(r.uint32le, ['fallback', 'transformFailure', 'transformPartial']) }); + this.COMP_PLUS_NAME_ITEM = r.uint32le; //str + + this.COMP_PLUS_NAME = new r.Struct({ + ident: r.uint32le, + size: r.uint32le, + count: r.uint32le, + reserved: new r.Reserved(r.uint32le), // padding + items: new r.Array(this.COMP_PLUS_NAME_ITEM, 'count') + }); + this.COMP_PLUS_LOCA_ITEM = r.uint32le; //str this.COMP_PLUS_LOCA = new r.Struct({ diff --git a/developer/src/kmldmlc/test/fixtures/basic.txt b/developer/src/kmldmlc/test/fixtures/basic.txt index e9b51b8f85..864acbec82 100644 --- a/developer/src/kmldmlc/test/fixtures/basic.txt +++ b/developer/src/kmldmlc/test/fixtures/basic.txt @@ -10,7 +10,7 @@ # # cd developer/src/kmldmlc # # ./build.sh build # if necessary -# node . test/fixtures/basic.xml /tmp/basic.kmx +# node . test/fixtures/basic.xml -o /tmp/basic.kmx # xxd -g 1 -l 12 /tmp/basic.kmx | cut -d' ' -f 10-13 # # rm /tmp/basic.kmx # or you may wish to examine it in more detail # @@ -20,7 +20,7 @@ block(kmxheader) # struct COMP_KEYBOARD { 00 10 00 00 # KMX_DWORD dwFileVersion; // 0004 Version of the file - Keyman 4.0 is 0x0400 - 94 95 98 0d # KMX_DWORD dwCheckSum; // 0008 As stored in keyboard + 46 bf 48 f2 # KMX_DWORD dwCheckSum; // 0008 As stored in keyboard 00 00 00 00 # KMX_DWORD KeyboardID; // 000C as stored in HKEY_LOCAL_MACHINE//system//currentcontrolset//control//keyboard layouts 01 00 00 00 # KMX_DWORD IsRegistered; // 0010 00 00 00 00 # KMX_DWORD version; // 0014 keyboard version @@ -52,7 +52,7 @@ block(sect) # struct COMP_KMXPLUS_SECT { 73 65 63 74 # KMX_DWORD header.ident; // 0000 Section name sizeof(sect) # KMX_DWORD header.size; // 0004 Section length diff(sect,eof) # KMX_DWORD total; // 0008 KMXPlus entire length - 04 00 00 00 # KMX_DWORD count; // 000C number of section headers + 05 00 00 00 # KMX_DWORD count; // 000C number of section headers # }; # Next sections are sect entries # KMX_DWORD sect; // 0010+ Section identity @@ -64,6 +64,9 @@ block(sect) # struct COMP_KMXPLUS_SECT { 6d 65 74 61 diff(sect,meta) + 6e 61 6d 65 + diff(sect,name) + 6c 6f 63 61 diff(sect,loca) @@ -124,6 +127,14 @@ block(meta) # struct COMP_KMXPLUS_META { 00 00 00 00 # KMX_DWORD settings; # }; +block(name) # struct COMP_KMXPLUS_META { + 6e 61 6d 65 # KMX_DWORD header.ident; // 0000 Section name - name + sizeof(name) # KMX_DWORD header.size; // 0004 Section length + 01 00 00 00 # KMX_DWORD count; // 0008 number of names + 00 00 00 00 # KMX_DWORD reserved; // 000C padding + index(strNull,strName,2) # KMXPLUS_STR name; // 0010+ name string entry = 'TestKbd' + # }; + block(loca) # struct COMP_KMXPLUS_LOCA { 6c 6f 63 61 # KMX_DWORD header.ident; // 0000 Section name - loca sizeof(loca) # KMX_DWORD header.size; // 0004 Section length