From aef38f12df102913f6184dfcbb87fe2aced7927d Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Sat, 3 Sep 2022 06:11:24 +1000 Subject: [PATCH] chore(core): define zero length string vs null string for kmxplus --- core/src/ldml/C7043_ldml.md | 17 ++++- .../src/keyman/kmx/kmx-plus-builder.ts | 4 ++ developer/src/kmldmlc/test/fixtures/basic.txt | 63 ++++++++++++------- 3 files changed, 59 insertions(+), 25 deletions(-) diff --git a/core/src/ldml/C7043_ldml.md b/core/src/ldml/C7043_ldml.md index 45097a5406..06b59edfb5 100644 --- a/core/src/ldml/C7043_ldml.md +++ b/core/src/ldml/C7043_ldml.md @@ -66,11 +66,22 @@ Then for each string: |16+| 32 | offset | off: Offset to string | |20+| 32 | length | int: Length of string in UTF-16LE code units | -After the string offset table comes the actual UTF-16LE data. There is a null (\u0000) after each string, which is _not_ included in the string length. +After the string offset table comes the actual UTF-16LE data. There is a null +(\u0000) after each string, which is _not_ included in the string length. -The string offset table, and then strings themselves, are sorted according to a binary codepoint sort, not including the null. +The string offset table, and then strings themselves, are sorted according to a +binary codepoint sort, not including the null. -A string may be zero length. +The first string in the string table MUST always be the zero-length string. A +zero-length string is considered the same as a null string. For metadata fields, +references to this zero-length string will have the dword index value 0, which +can safely be interpreted as "not set". There may be other locations which have +required strings, but for which a zero-length string is permissible, and this +index value of 0 can be used for that purpose. + +A distinction between zero-length string and optional should be avoided (e.g. +the difference between "" and null in Javascript). If this is truly required, a +separate flag field must be used to denote the difference. ### C7043.2.3 `meta`—Metadata diff --git a/developer/src/kmldmlc/src/keyman/kmx/kmx-plus-builder.ts b/developer/src/kmldmlc/src/keyman/kmx/kmx-plus-builder.ts index 287ad7f0e3..f325c9397f 100644 --- a/developer/src/kmldmlc/src/keyman/kmx/kmx-plus-builder.ts +++ b/developer/src/kmldmlc/src/keyman/kmx/kmx-plus-builder.ts @@ -273,6 +273,10 @@ export default class KMXPlusBuilder { this.sect_sect = this.build_sect(); this.sect_strs = this.build_strs(); + + // per C7043, the first string in sect_strs MUST be the zero-length string. + this.alloc_string(''); + this.sect_meta = this.build_meta(); this.sect_loca = this.build_loca(); this.sect_keys = this.build_keys(); diff --git a/developer/src/kmldmlc/test/fixtures/basic.txt b/developer/src/kmldmlc/test/fixtures/basic.txt index d8a6213fdb..8f660adac6 100644 --- a/developer/src/kmldmlc/test/fixtures/basic.txt +++ b/developer/src/kmldmlc/test/fixtures/basic.txt @@ -1,9 +1,26 @@ +# +# basic.txt describes the expected output of running kmldmlc against basic.xml. It is used in +# the end-to-end test test-compiler-e2e.ts. +# +# Any changes to the compiler or basic.xml will likely result in changes to the compiled file. +# While structural differences should be updated manually in this file to ensure that we are +# getting the expected result for the e2e test, the checksum can be safely retrieved from the +# updated compilation result. The following may be helpful for getting the updated checksum +# value: +# +# cd developer/src/kmldmlc +# # ./build.sh build # if necessary +# node . test/fixtures/basic.xml /tmp/basic.kmx +# xxd -g 1 -l 12 /tmp/basic.kmx | cut -d' ' -f 10-13 +# # rm /tmp/basic.kmx # or you may wish to examine it in more detail +# + block(kmxheader) # struct COMP_KEYBOARD { 4b 58 54 53 # KMX_DWORD dwIdentifier; // 0000 Keyman compiled keyboard id 00 10 00 00 # KMX_DWORD dwFileVersion; // 0004 Version of the file - Keyman 4.0 is 0x0400 - 13 06 7c 19 # KMX_DWORD dwCheckSum; // 0008 As stored in keyboard + 94 95 98 0d # KMX_DWORD dwCheckSum; // 0008 As stored in keyboard 00 00 00 00 # KMX_DWORD KeyboardID; // 000C as stored in HKEY_LOCAL_MACHINE//system//currentcontrolset//control//keyboard layouts 01 00 00 00 # KMX_DWORD IsRegistered; // 0010 00 00 00 00 # KMX_DWORD version; // 0014 keyboard version @@ -53,13 +70,14 @@ block(sect) # struct COMP_KMXPLUS_SECT { block(strs) # struct COMP_KMXPLUS_STRS { 73 74 72 73 # KMX_DWORD header.ident; // 0000 Section name - strs diff(strs,endstrs) # KMX_DWORD header.size; // 0004 Section length - 0A 00 00 00 # KMX_DWORD count; // 0008 count of str entries + 0B 00 00 00 # KMX_DWORD count; // 0008 count of str entries 00 00 00 00 # KMX_DWORD reserved; // 000C padding # Next sections are string entries # KMX_DWORD offset; // 0010+ offset from this blob # KMX_DWORD length; // 0014+ str length (UTF-16LE units) + diff(strs,strNull) sizeof(strNull,2) diff(strs,strName) sizeof(strName,2) diff(strs,strAuthor) sizeof(strAuthor,2) diff(strs,strConformsTo) sizeof(strConformsTo,2) @@ -75,30 +93,31 @@ block(strs) # struct COMP_KMXPLUS_STRS { # String table -- block(x) is used to store the null u16char at end of each string # without interfering with sizeof() calculation above -block(strName) 54 00 65 00 73 00 74 00 4b 00 62 00 64 00 block(x) 00 00 # 0:TestKbd -block(strAuthor) 73 00 72 00 6c 00 32 00 39 00 35 00 block(x) 00 00 # 1:srl295 +block(strNull) block(x) 00 00 # the zero-length string +block(strName) 54 00 65 00 73 00 74 00 4b 00 62 00 64 00 block(x) 00 00 # 'TestKbd' +block(strAuthor) 73 00 72 00 6c 00 32 00 39 00 35 00 block(x) 00 00 # 'srl295' block(strConformsTo) 74 00 65 00 63 00 68 00 70 00 72 00 65 00 76 00 - 69 00 65 00 77 00 block(x) 00 00 # 2:techpreview -block(strLayout) 71 00 77 00 65 00 72 00 74 00 79 00 block(x) 00 00 # 3:qwerty -block(strNorm) 4e 00 46 00 43 00 block(x) 00 00 # 4:NFC -block(strIndicator) 3d d8 40 de block(x) 00 00 # 5:🙀 -block(strVersion) 30 00 block(x) 00 00 # 6:0 -block(strLocale) 6d 00 74 00 block(x) 00 00 # 7:mt -block(strKey1) 27 01 block(x) 00 00 # 8:ħ -block(strKey2) 90 17 b6 17 block(x) 00 00 # 9:ថា + 69 00 65 00 77 00 block(x) 00 00 # 'techpreview' +block(strLayout) 71 00 77 00 65 00 72 00 74 00 79 00 block(x) 00 00 # 'qwerty' +block(strNorm) 4e 00 46 00 43 00 block(x) 00 00 # 'NFC' +block(strIndicator) 3d d8 40 de block(x) 00 00 # '🙀' +block(strVersion) 30 00 block(x) 00 00 # '0' +block(strLocale) 6d 00 74 00 block(x) 00 00 # 'mt' +block(strKey1) 27 01 block(x) 00 00 # 'ħ' +block(strKey2) 90 17 b6 17 block(x) 00 00 # 'ថា' block(endstrs) # end of strs block block(meta) # struct COMP_KMXPLUS_META { 6d 65 74 61 # KMX_DWORD header.ident; // 0000 Section name - meta sizeof(meta) # KMX_DWORD header.size; // 0004 Section length - 00 00 00 00 # KMX_DWORD name; - 01 00 00 00 # KMX_DWORD author; - 02 00 00 00 # KMX_DWORD conform; - 03 00 00 00 # KMX_DWORD layout; - 04 00 00 00 # KMX_DWORD normalization; - 05 00 00 00 # KMX_DWORD indicator; - 06 00 00 00 # KMX_DWORD version; + 01 00 00 00 # KMXPLUS_STR name; + 02 00 00 00 # KMXPLUS_STR author; + 03 00 00 00 # KMXPLUS_STR conform; + 04 00 00 00 # KMXPLUS_STR layout; + 05 00 00 00 # KMXPLUS_STR normalization; + 06 00 00 00 # KMXPLUS_STR indicator; + 07 00 00 00 # KMXPLUS_STR version; 00 00 00 00 # KMX_DWORD settings; # }; @@ -107,7 +126,7 @@ block(loca) # struct COMP_KMXPLUS_LOCA { sizeof(loca) # KMX_DWORD header.size; // 0004 Section length 01 00 00 00 # KMX_DWORD count; // 0008 number of locales 00 00 00 00 # KMX_DWORD reserved; - 07 00 00 00 # KMX_DWORD locale; // 0010+ locale string entry = 'mt' + 08 00 00 00 # KMXPLUS_STR locale; // 0010+ locale string entry = 'mt' # }; block(keys) # struct COMP_KMXPLUS_KEYS { @@ -119,7 +138,7 @@ block(keys) # struct COMP_KMXPLUS_KEYS { # Keys data: - c0 00 00 00 00 00 00 00 08 00 00 00 01 00 00 00 # KMX_DWORD vkey, mod, to, flags; - 31 00 00 00 00 00 00 00 09 00 00 00 01 00 00 00 # KMX_DWORD vkey, mod, to, flags; + c0 00 00 00 00 00 00 00 09 00 00 00 01 00 00 00 # KMX_DWORD vkey, mod, to, flags; + 31 00 00 00 00 00 00 00 0A 00 00 00 01 00 00 00 # KMX_DWORD vkey, mod, to, flags; block(eof) # end of file \ No newline at end of file