mirror of
https://github.com/keymanapp/keyman.git
synced 2026-08-08 09:55:32 +00:00
feat(common): add size and offset calculators to hextobin
This makes it easier to write a binary file which contains offsets and size values. Using this for unit tests for the LDML keyboard compiler. Example kmldmlc basic.txt is updated and included. Future enhancement suggestion: include count() for repeated blocks and index() for block indices within these repeated blocks. Would be very useful for string tables in the example ref.
This commit is contained in:
Marc Durdin
parent
918d261f99
commit
17371c3801
3 changed files with 239 additions and 107 deletions
|
|
@ -9,11 +9,19 @@ let outputFilename: string = "";
|
|||
program
|
||||
.description(
|
||||
`Will convert the input file which is a hex dump to the output file. Hex dump can contain
|
||||
blank lines, offsets and comments. File writing will start at zero if no offset given;
|
||||
any blank spaces in the file will be filled with nul bytes.
|
||||
blank lines, offsets, commands and comments. File writing will start at zero, padding is
|
||||
not currently supported.
|
||||
|
||||
0x1234: 11 22 33 44 55 aa bb # This is a comment
|
||||
# offset(0xhex|dec): hex bytes # comment`
|
||||
Format:
|
||||
|
||||
# This is a comment
|
||||
block(name) # each part of file must be defined as a block
|
||||
11 22 33 44 55 aa bb # inserts hex bytes
|
||||
offset(block) # inserts 4 byte offset of block from BOF
|
||||
sizeof(block[,divisor]) # inserts 4 byte size of block, optionally divided by divisor
|
||||
diff(block1,block2) # inserts 4 byte offset diff between start of block1 and block2
|
||||
# block names are required, but if not referenced can be reused (e.g. block(x))
|
||||
`
|
||||
)
|
||||
.arguments('<infile>')
|
||||
.arguments('<outfile>')
|
||||
|
|
@ -34,4 +42,4 @@ function exitDueToUsageError(message: string): never {
|
|||
|
||||
import hextobin from './index';
|
||||
|
||||
hextobin(inputFilename, outputFilename, {silent: true});
|
||||
hextobin(inputFilename, outputFilename, {silent: false});
|
||||
|
|
|
|||
|
|
@ -8,70 +8,194 @@ export default async function hextobin(inputFilename: string, outputFilename?: s
|
|||
|
||||
let reader = rd.createInterface(fs.createReadStream(inputFilename));
|
||||
|
||||
interface HexBlock {
|
||||
offset: number;
|
||||
hex: string;
|
||||
interface HexBlockRef {
|
||||
type: 'sizeof' | 'offset' | 'diff';
|
||||
blockName: string;
|
||||
blockName2?: string; // used only by diff
|
||||
divisor?: number; // used only by sizeof
|
||||
offset: number; // actual byte offset in hex data (i.e. offset in string is * 2)
|
||||
};
|
||||
|
||||
let data: HexBlock[] = [];
|
||||
interface HexBlock {
|
||||
name: string;
|
||||
offset: number; // calculated during reconciliation phase
|
||||
hex: string;
|
||||
refs: HexBlockRef[];
|
||||
};
|
||||
|
||||
let lineNumber = 0;
|
||||
let blocks: HexBlock[] = [];
|
||||
|
||||
for await (const l of reader) {
|
||||
lineNumber++;
|
||||
let tokens = l.split(/[ \t]+/);
|
||||
let currentLine = '';
|
||||
let currentLineNumber = 0;
|
||||
|
||||
while(tokens.length && tokens[0] == '') {
|
||||
tokens.shift();
|
||||
if(!await load()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if(!reconciliation()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return save();
|
||||
|
||||
function reportError(message) {
|
||||
if(!options.silent) {
|
||||
console.error(`Invalid input file: ${message} on line #${currentLineNumber}: "${currentLine}"`);
|
||||
}
|
||||
}
|
||||
|
||||
if(tokens.length == 0) {
|
||||
continue;
|
||||
}
|
||||
function currentBlock(): HexBlock {
|
||||
return blocks.length ? blocks[blocks.length-1] : null;
|
||||
}
|
||||
|
||||
if(tokens[0].endsWith(':')) {
|
||||
let f = tokens.shift();
|
||||
if(!f) {
|
||||
continue;
|
||||
}
|
||||
data.push({offset: parseInt(f), hex:''});
|
||||
}
|
||||
|
||||
if(data.length == 0) {
|
||||
// no offset given, starting at 0
|
||||
data.push({offset: 0, hex: ''});
|
||||
}
|
||||
|
||||
for(let token of tokens) {
|
||||
if(token == '') {
|
||||
continue;
|
||||
}
|
||||
if(token.startsWith('#')) {
|
||||
break;
|
||||
}
|
||||
function parseToken(token: string): { command: string, parameters: string[] } {
|
||||
let m = /^([a-z]+)\((.+)\)$/.exec(token);
|
||||
if(!m) {
|
||||
if(!token.match(/^[a-fA-F0-9]{2}$/)) {
|
||||
if(!options.silent) {
|
||||
console.error(`Invalid input file: expected hex for component "${token}" on line #${lineNumber}: "${l}"`);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
data[data.length-1].hex += token;
|
||||
// hex byte
|
||||
return { command: 'data', parameters: [token] };
|
||||
}
|
||||
|
||||
// processing command
|
||||
return { command: m[1], parameters: m[2].split(',') };
|
||||
}
|
||||
|
||||
let total = data.reduce((total: number, item: HexBlock) => Math.max(item.offset + item.hex.length/2, total), 0);
|
||||
function token(token: string) {
|
||||
const t = parseToken(token);
|
||||
if(!t) {
|
||||
reportError(`expected command or hex at token "${token}"`);
|
||||
return false;
|
||||
}
|
||||
|
||||
if(!options.silent) {
|
||||
console.log(`${lineNumber} lines read; ${data.length} sections to write. Total file size = ${total} bytes.` );
|
||||
}
|
||||
let buffer = new Uint8Array(total);
|
||||
data.forEach(item => {
|
||||
let buf = Buffer.from(item.hex, 'hex');
|
||||
buffer.set(buf, item.offset);
|
||||
});
|
||||
if(t.command == 'block') {
|
||||
blocks.push({name: t.parameters[0], offset: 0, hex: '', refs: []});
|
||||
return true;
|
||||
}
|
||||
|
||||
if(outputFilename) {
|
||||
fs.writeFileSync(outputFilename, buffer);
|
||||
const b = currentBlock();
|
||||
if(!b) {
|
||||
reportError(`expected block() before data`);
|
||||
return false;
|
||||
}
|
||||
|
||||
switch(t.command) {
|
||||
case 'offset':
|
||||
b.refs.push({type: 'offset', blockName: t.parameters[0], offset: b.hex.length / 2});
|
||||
b.hex += "_".repeat(8); // always 4 bytes, placeholder will be filled in during reconciliation phase
|
||||
break;
|
||||
case 'sizeof':
|
||||
// sizeof can take a second parameter, divisor
|
||||
let divisor = t.parameters.length > 1 ? parseInt(t.parameters[1],10) : 1;
|
||||
b.refs.push({type: 'sizeof', blockName: t.parameters[0], divisor: divisor, offset: b.hex.length / 2});
|
||||
b.hex += "_".repeat(8); // always 4 bytes, placeholder will be filled in during reconciliation phase
|
||||
break;
|
||||
case 'diff':
|
||||
b.refs.push({type: 'diff', blockName: t.parameters[0], blockName2: t.parameters[1], offset: b.hex.length / 2});
|
||||
b.hex += "_".repeat(8); // always 4 bytes, placeholder will be filled in during reconciliation phase
|
||||
break;
|
||||
case 'data':
|
||||
b.hex += t.parameters[0];
|
||||
break;
|
||||
default:
|
||||
reportError(`unknown command ${t.command}`);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
async function load() {
|
||||
for await (const l of reader) {
|
||||
// Error reporting variables
|
||||
currentLine = l;
|
||||
currentLineNumber++;
|
||||
|
||||
const tokens = l.split(/[ \t]+/);
|
||||
for(let t of tokens) {
|
||||
if(t.startsWith('#')) {
|
||||
// comment, ignore all subsequent tokens to EOL
|
||||
break;
|
||||
}
|
||||
if(t == '') {
|
||||
continue;
|
||||
}
|
||||
if(!token(t)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function dwordLeToHex(v: number): string {
|
||||
// hacky but who cares
|
||||
let h = v.toString(16);
|
||||
h = "0".repeat(8-h.length) + h;
|
||||
return h.substring(6,8) + h.substring(4, 6) + h.substring(2, 4) + h.substring(0, 2);
|
||||
}
|
||||
|
||||
function fillBlockPlaceholder(block: HexBlock, offset: number, value: number): void {
|
||||
const hexvalue = dwordLeToHex(value);
|
||||
block.hex = block.hex.substring(0, offset * 2) + hexvalue + block.hex.substring(offset * 2 + 8);
|
||||
}
|
||||
|
||||
function reconciliation(): boolean {
|
||||
let offset = 0;
|
||||
|
||||
// calculate block sizes
|
||||
for(let b of blocks) {
|
||||
b.offset = offset;
|
||||
offset += b.hex.length / 2;
|
||||
}
|
||||
|
||||
// reconcile block offsets and sizes
|
||||
for(let b of blocks) {
|
||||
for(let r of b.refs) {
|
||||
const v = blocks.find(q => q.name == r.blockName);
|
||||
if(!v) {
|
||||
reportError(`Could not find block ${r.blockName} when reconciling ${b.name}`);
|
||||
return false;
|
||||
}
|
||||
switch(r.type) {
|
||||
case 'diff':
|
||||
const v2 = blocks.find(q => q.name == r.blockName2);
|
||||
if(!v2) {
|
||||
reportError(`Could not find block ${r.blockName2} when reconciling ${b.name}`);
|
||||
return false;
|
||||
}
|
||||
fillBlockPlaceholder(b, r.offset, v2.offset - v.offset);
|
||||
break;
|
||||
case 'offset':
|
||||
fillBlockPlaceholder(b, r.offset, v.offset);
|
||||
break;
|
||||
case 'sizeof':
|
||||
fillBlockPlaceholder(b, r.offset, v.hex.length / 2 / r.divisor);
|
||||
break;
|
||||
default:
|
||||
reportError(`Invalid ref ${r.type}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function save(): Uint8Array {
|
||||
let total = blocks.reduce((total: number, item: HexBlock) => Math.max(item.offset + item.hex.length/2, total), 0);
|
||||
|
||||
if(!options.silent) {
|
||||
console.log(`${currentLineNumber} lines read; ${blocks.length} sections to write. Total file size = ${total} bytes.` );
|
||||
}
|
||||
let buffer = new Uint8Array(total);
|
||||
blocks.forEach(item => {
|
||||
let buf = Buffer.from(item.hex, 'hex');
|
||||
buffer.set(buf, item.offset);
|
||||
});
|
||||
|
||||
if(outputFilename) {
|
||||
fs.writeFileSync(outputFilename, buffer);
|
||||
}
|
||||
return buffer;
|
||||
}
|
||||
return buffer;
|
||||
}
|
||||
106
developer/src/kmldmlc/test/fixtures/basic.txt
vendored
106
developer/src/kmldmlc/test/fixtures/basic.txt
vendored
|
|
@ -1,4 +1,4 @@
|
|||
0x0000: # struct COMP_KEYBOARD {
|
||||
block(kmxheader) # struct COMP_KEYBOARD {
|
||||
4b 58 54 53 # KMX_DWORD dwIdentifier; // 0000 Keyman compiled keyboard id
|
||||
|
||||
00 10 00 00 # KMX_DWORD dwFileVersion; // 0004 Version of the file - Keyman 4.0 is 0x0400
|
||||
|
|
@ -26,72 +26,70 @@
|
|||
# };
|
||||
|
||||
|
||||
0x0040: # struct COMP_KEYBOARD_KMXPLUSINFO {
|
||||
48 00 00 00 # KMX_DWORD dpKMXPlus; // 0040 offset of KMXPlus data, <sect> header is first
|
||||
52 01 00 00 # KMX_DWORD dwKMXPlusSize; // 0044 size in bytes of entire KMXPlus data
|
||||
# };
|
||||
block(kmxplusinfo) # struct COMP_KEYBOARD_KMXPLUSINFO {
|
||||
offset(sect) # KMX_DWORD dpKMXPlus; // 0040 offset of KMXPlus data from BOF, <sect> header is first
|
||||
diff(sect,eof) # KMX_DWORD dwKMXPlusSize; // 0044 size in bytes of entire KMXPlus data
|
||||
# };
|
||||
|
||||
0x0048: # struct COMP_KMXPLUS_SECT {
|
||||
block(sect) # struct COMP_KMXPLUS_SECT {
|
||||
73 65 63 74 # KMX_DWORD header.ident; // 0000 Section name
|
||||
30 00 00 00 # KMX_DWORD header.size; // 0004 Section length
|
||||
52 01 00 00 # KMX_DWORD total; // 0008 KMXPlus entire length
|
||||
sizeof(sect) # KMX_DWORD header.size; // 0004 Section length
|
||||
diff(sect,eof) # KMX_DWORD total; // 0008 KMXPlus entire length
|
||||
04 00 00 00 # KMX_DWORD count; // 000C number of section headers
|
||||
|
||||
73 74 72 73 # KMX_DWORD sect; // 0010+ Section identity - strs
|
||||
30 00 00 00 # KMX_DWORD offset; // 0014+ Section offset relative to dpKMXPlus of section
|
||||
diff(sect,strs) # KMX_DWORD offset; // 0014+ Section offset relative to dpKMXPlus of section
|
||||
|
||||
6d 65 74 61 # KMX_DWORD sect; // 0010+ Section identity - meta
|
||||
ea 00 00 00 # KMX_DWORD offset; // 0014+ Section offset relative to dpKMXPlus of section
|
||||
diff(sect,meta) # KMX_DWORD offset; // 0014+ Section offset relative to dpKMXPlus of section
|
||||
|
||||
6c 6f 63 61 # KMX_DWORD sect; // 0010+ Section identity - loca
|
||||
0e 01 00 00 # KMX_DWORD offset; // 0014+ Section offset relative to dpKMXPlus of section
|
||||
diff(sect,loca) # KMX_DWORD offset; // 0014+ Section offset relative to dpKMXPlus of section
|
||||
|
||||
6b 65 79 73 # KMX_DWORD sect; // 0010+ Section identity - keys
|
||||
22 01 00 00 # KMX_DWORD offset; // 0014+ Section offset relative to dpKMXPlus of section
|
||||
diff(sect,keys) # KMX_DWORD offset; // 0014+ Section offset relative to dpKMXPlus of section
|
||||
# };
|
||||
|
||||
0x0078: # struct COMP_KMXPLUS_STRS {
|
||||
73 74 72 73 # KMX_DWORD header.ident; // 0000 Section name - strs
|
||||
ba 00 00 00 # KMX_DWORD header.size; // 0004 Section length
|
||||
09 00 00 00 # KMX_DWORD count; // 0008 count of str entries
|
||||
00 00 00 00 # KMX_DWORD reserved; // 000C padding
|
||||
block(strs) # struct COMP_KMXPLUS_STRS {
|
||||
73 74 72 73 # KMX_DWORD header.ident; // 0000 Section name - strs
|
||||
diff(strs,endstrs) # KMX_DWORD header.size; // 0004 Section length
|
||||
09 00 00 00 # KMX_DWORD count; // 0008 count of str entries
|
||||
00 00 00 00 # KMX_DWORD reserved; // 000C padding
|
||||
|
||||
58 00 00 00 # KMX_DWORD offset; // 0010+ offset from this blob
|
||||
07 00 00 00 # KMX_DWORD length; // 0014+ str length (UTF-16LE units)
|
||||
68 00 00 00 # KMX_DWORD offset; // 0010+ offset from this blob
|
||||
06 00 00 00 # KMX_DWORD length; // 0014+ str length (UTF-16LE units)
|
||||
76 00 00 00 # KMX_DWORD offset; // 0010+ offset from this blob
|
||||
0b 00 00 00 # KMX_DWORD length; // 0014+ str length (UTF-16LE units)
|
||||
8e 00 00 00 # KMX_DWORD offset; // 0010+ offset from this blob
|
||||
06 00 00 00 # KMX_DWORD length; // 0014+ str length (UTF-16LE units)
|
||||
9c 00 00 00 # KMX_DWORD offset; // 0010+ offset from this blob
|
||||
03 00 00 00 # KMX_DWORD length; // 0014+ str length (UTF-16LE units)
|
||||
a4 00 00 00 # KMX_DWORD offset; // 0010+ offset from this blob
|
||||
02 00 00 00 # KMX_DWORD length; // 0014+ str length (UTF-16LE units)
|
||||
aa 00 00 00 # KMX_DWORD offset; // 0010+ offset from this blob
|
||||
02 00 00 00 # KMX_DWORD length; // 0014+ str length (UTF-16LE units)
|
||||
b0 00 00 00 # KMX_DWORD offset; // 0010+ offset from this blob
|
||||
01 00 00 00 # KMX_DWORD length; // 0014+ str length (UTF-16LE units)
|
||||
b4 00 00 00 # KMX_DWORD offset; // 0010+ offset from this blob
|
||||
02 00 00 00 # KMX_DWORD length; // 0014+ str length (UTF-16LE units)
|
||||
# Next sections are string entries
|
||||
# KMX_DWORD offset; // 0010+ offset from this blob
|
||||
# KMX_DWORD length; // 0014+ str length (UTF-16LE units)
|
||||
|
||||
diff(strs,strName) sizeof(strName,2)
|
||||
diff(strs,strAuthor) sizeof(strAuthor,2)
|
||||
diff(strs,strConformsTo) sizeof(strConformsTo,2)
|
||||
diff(strs,strLayout) sizeof(strLayout,2)
|
||||
diff(strs,strNorm) sizeof(strNorm,2)
|
||||
diff(strs,strIndicator) sizeof(strIndicator,2)
|
||||
diff(strs,strLocale) sizeof(strLocale,2)
|
||||
diff(strs,strKey1) sizeof(strKey1,2)
|
||||
diff(strs,strKey2) sizeof(strKey2,2)
|
||||
# };
|
||||
|
||||
# String table
|
||||
# String table -- block(x) is used to store the null u16char at end of each string
|
||||
# without interfering with sizeof() calculation above
|
||||
|
||||
54 00 65 00 73 00 74 00 4b 00 62 00 64 00 00 00 # 0:TestKbd
|
||||
73 00 72 00 6c 00 32 00 39 00 35 00 00 00 # 1:srl295
|
||||
74 00 65 00 63 00 68 00 70 00 72 00 65 00 76 00
|
||||
69 00 65 00 77 00 00 00 # 2:techpreview
|
||||
71 00 77 00 65 00 72 00 74 00 79 00 00 00 # 3:qwerty
|
||||
4e 00 46 00 43 00 00 00 # 4:NFC
|
||||
3d d8 40 de 00 00 # 5:🙀
|
||||
6d 00 74 00 00 00 # 6:mt
|
||||
27 01 00 00 # 7:ħ
|
||||
90 17 b6 17 00 00 # 8:ថា
|
||||
block(strName) 54 00 65 00 73 00 74 00 4b 00 62 00 64 00 block(x) 00 00 # 0:TestKbd
|
||||
block(strAuthor) 73 00 72 00 6c 00 32 00 39 00 35 00 block(x) 00 00 # 1:srl295
|
||||
block(strConformsTo) 74 00 65 00 63 00 68 00 70 00 72 00 65 00 76 00
|
||||
69 00 65 00 77 00 block(x) 00 00 # 2:techpreview
|
||||
block(strLayout) 71 00 77 00 65 00 72 00 74 00 79 00 block(x) 00 00 # 3:qwerty
|
||||
block(strNorm) 4e 00 46 00 43 00 block(x) 00 00 # 4:NFC
|
||||
block(strIndicator) 3d d8 40 de block(x) 00 00 # 5:🙀
|
||||
block(strLocale) 6d 00 74 00 block(x) 00 00 # 7:mt
|
||||
block(strKey1) 27 01 block(x) 00 00 # 8:ħ
|
||||
block(strKey2) 90 17 b6 17 block(x) 00 00 # 9:ថា
|
||||
|
||||
0x132: # struct COMP_KMXPLUS_META {
|
||||
6d 65 74 61 # KMX_DWORD header.ident; // 0000 Section name - meta
|
||||
24 00 00 00 # KMX_DWORD header.size; // 0004 Section length
|
||||
block(endstrs) # end of strs block
|
||||
|
||||
block(meta) # struct COMP_KMXPLUS_META {
|
||||
6d 65 74 61 # KMX_DWORD header.ident; // 0000 Section name - meta
|
||||
sizeof(meta) # KMX_DWORD header.size; // 0004 Section length
|
||||
00 00 00 00 # KMX_DWORD name;
|
||||
01 00 00 00 # KMX_DWORD author;
|
||||
02 00 00 00 # KMX_DWORD conform;
|
||||
|
|
@ -101,17 +99,17 @@
|
|||
00 00 00 00 # KMX_DWORD settings;
|
||||
# };
|
||||
|
||||
0x0156: # struct COMP_KMXPLUS_LOCA {
|
||||
block(loca) # struct COMP_KMXPLUS_LOCA {
|
||||
6c 6f 63 61 # KMX_DWORD header.ident; // 0000 Section name - loca
|
||||
14 00 00 00 # KMX_DWORD header.size; // 0004 Section length
|
||||
sizeof(loca) # KMX_DWORD header.size; // 0004 Section length
|
||||
01 00 00 00 # KMX_DWORD count; // 0008 number of locales
|
||||
00 00 00 00 # KMX_DWORD reserved;
|
||||
06 00 00 00 # KMX_DWORD locale; // 0010+ locale string entry = 'mt'
|
||||
# };
|
||||
|
||||
0x016a: # struct COMP_KMXPLUS_KEYS {
|
||||
block(keys) # struct COMP_KMXPLUS_KEYS {
|
||||
6b 65 79 73 # KMX_DWORD header.ident; // 0000 Section name - keys
|
||||
30 00 00 00 # KMX_DWORD header.size; // 0004 Section length
|
||||
sizeof(keys) # KMX_DWORD header.size; // 0004 Section length
|
||||
02 00 00 00 # KMX_DWORD count; // number of keys
|
||||
00 00 00 00 # KMX_DWORD reserved; // padding
|
||||
# };
|
||||
|
|
@ -120,3 +118,5 @@
|
|||
|
||||
c0 00 00 00 00 00 00 00 07 00 00 00 01 00 00 00 # KMX_DWORD vkey, mod, to, flags;
|
||||
31 00 00 00 00 00 00 00 08 00 00 00 01 00 00 00 # KMX_DWORD vkey, mod, to, flags;
|
||||
|
||||
block(eof) # end of file
|
||||
Loading…
Add table
Reference in a new issue