diff --git a/common/tools/hextobin/hextobin.ts b/common/tools/hextobin/hextobin.ts index 058e199cb8..9a325f6f38 100644 --- a/common/tools/hextobin/hextobin.ts +++ b/common/tools/hextobin/hextobin.ts @@ -9,11 +9,19 @@ let outputFilename: string = ""; program .description( `Will convert the input file which is a hex dump to the output file. Hex dump can contain -blank lines, offsets and comments. File writing will start at zero if no offset given; -any blank spaces in the file will be filled with nul bytes. +blank lines, offsets, commands and comments. File writing will start at zero, padding is +not currently supported. -0x1234: 11 22 33 44 55 aa bb # This is a comment -# offset(0xhex|dec): hex bytes # comment` +Format: + + # This is a comment + block(name) # each part of file must be defined as a block + 11 22 33 44 55 aa bb # inserts hex bytes + offset(block) # inserts 4 byte offset of block from BOF + sizeof(block[,divisor]) # inserts 4 byte size of block, optionally divided by divisor + diff(block1,block2) # inserts 4 byte offset diff between start of block1 and block2 + # block names are required, but if not referenced can be reused (e.g. block(x)) +` ) .arguments('') .arguments('') @@ -34,4 +42,4 @@ function exitDueToUsageError(message: string): never { import hextobin from './index'; -hextobin(inputFilename, outputFilename, {silent: true}); +hextobin(inputFilename, outputFilename, {silent: false}); diff --git a/common/tools/hextobin/index.ts b/common/tools/hextobin/index.ts index 09e70eb29e..c914327945 100644 --- a/common/tools/hextobin/index.ts +++ b/common/tools/hextobin/index.ts @@ -8,70 +8,194 @@ export default async function hextobin(inputFilename: string, outputFilename?: s let reader = rd.createInterface(fs.createReadStream(inputFilename)); - interface HexBlock { - offset: number; - hex: string; + interface HexBlockRef { + type: 'sizeof' | 'offset' | 'diff'; + blockName: string; + blockName2?: string; // used only by diff + divisor?: number; // used only by sizeof + offset: number; // actual byte offset in hex data (i.e. offset in string is * 2) }; - let data: HexBlock[] = []; + interface HexBlock { + name: string; + offset: number; // calculated during reconciliation phase + hex: string; + refs: HexBlockRef[]; + }; - let lineNumber = 0; + let blocks: HexBlock[] = []; - for await (const l of reader) { - lineNumber++; - let tokens = l.split(/[ \t]+/); + let currentLine = ''; + let currentLineNumber = 0; - while(tokens.length && tokens[0] == '') { - tokens.shift(); + if(!await load()) { + return null; + } + + if(!reconciliation()) { + return null; + } + + return save(); + + function reportError(message) { + if(!options.silent) { + console.error(`Invalid input file: ${message} on line #${currentLineNumber}: "${currentLine}"`); } + } - if(tokens.length == 0) { - continue; - } + function currentBlock(): HexBlock { + return blocks.length ? blocks[blocks.length-1] : null; + } - if(tokens[0].endsWith(':')) { - let f = tokens.shift(); - if(!f) { - continue; - } - data.push({offset: parseInt(f), hex:''}); - } - - if(data.length == 0) { - // no offset given, starting at 0 - data.push({offset: 0, hex: ''}); - } - - for(let token of tokens) { - if(token == '') { - continue; - } - if(token.startsWith('#')) { - break; - } + function parseToken(token: string): { command: string, parameters: string[] } { + let m = /^([a-z]+)\((.+)\)$/.exec(token); + if(!m) { if(!token.match(/^[a-fA-F0-9]{2}$/)) { - if(!options.silent) { - console.error(`Invalid input file: expected hex for component "${token}" on line #${lineNumber}: "${l}"`); - } return null; } - data[data.length-1].hex += token; + // hex byte + return { command: 'data', parameters: [token] }; } + + // processing command + return { command: m[1], parameters: m[2].split(',') }; } - let total = data.reduce((total: number, item: HexBlock) => Math.max(item.offset + item.hex.length/2, total), 0); + function token(token: string) { + const t = parseToken(token); + if(!t) { + reportError(`expected command or hex at token "${token}"`); + return false; + } - if(!options.silent) { - console.log(`${lineNumber} lines read; ${data.length} sections to write. Total file size = ${total} bytes.` ); - } - let buffer = new Uint8Array(total); - data.forEach(item => { - let buf = Buffer.from(item.hex, 'hex'); - buffer.set(buf, item.offset); - }); + if(t.command == 'block') { + blocks.push({name: t.parameters[0], offset: 0, hex: '', refs: []}); + return true; + } - if(outputFilename) { - fs.writeFileSync(outputFilename, buffer); + const b = currentBlock(); + if(!b) { + reportError(`expected block() before data`); + return false; + } + + switch(t.command) { + case 'offset': + b.refs.push({type: 'offset', blockName: t.parameters[0], offset: b.hex.length / 2}); + b.hex += "_".repeat(8); // always 4 bytes, placeholder will be filled in during reconciliation phase + break; + case 'sizeof': + // sizeof can take a second parameter, divisor + let divisor = t.parameters.length > 1 ? parseInt(t.parameters[1],10) : 1; + b.refs.push({type: 'sizeof', blockName: t.parameters[0], divisor: divisor, offset: b.hex.length / 2}); + b.hex += "_".repeat(8); // always 4 bytes, placeholder will be filled in during reconciliation phase + break; + case 'diff': + b.refs.push({type: 'diff', blockName: t.parameters[0], blockName2: t.parameters[1], offset: b.hex.length / 2}); + b.hex += "_".repeat(8); // always 4 bytes, placeholder will be filled in during reconciliation phase + break; + case 'data': + b.hex += t.parameters[0]; + break; + default: + reportError(`unknown command ${t.command}`); + return false; + } + return true; + } + + async function load() { + for await (const l of reader) { + // Error reporting variables + currentLine = l; + currentLineNumber++; + + const tokens = l.split(/[ \t]+/); + for(let t of tokens) { + if(t.startsWith('#')) { + // comment, ignore all subsequent tokens to EOL + break; + } + if(t == '') { + continue; + } + if(!token(t)) { + return false; + } + } + } + return true; + } + + function dwordLeToHex(v: number): string { + // hacky but who cares + let h = v.toString(16); + h = "0".repeat(8-h.length) + h; + return h.substring(6,8) + h.substring(4, 6) + h.substring(2, 4) + h.substring(0, 2); + } + + function fillBlockPlaceholder(block: HexBlock, offset: number, value: number): void { + const hexvalue = dwordLeToHex(value); + block.hex = block.hex.substring(0, offset * 2) + hexvalue + block.hex.substring(offset * 2 + 8); + } + + function reconciliation(): boolean { + let offset = 0; + + // calculate block sizes + for(let b of blocks) { + b.offset = offset; + offset += b.hex.length / 2; + } + + // reconcile block offsets and sizes + for(let b of blocks) { + for(let r of b.refs) { + const v = blocks.find(q => q.name == r.blockName); + if(!v) { + reportError(`Could not find block ${r.blockName} when reconciling ${b.name}`); + return false; + } + switch(r.type) { + case 'diff': + const v2 = blocks.find(q => q.name == r.blockName2); + if(!v2) { + reportError(`Could not find block ${r.blockName2} when reconciling ${b.name}`); + return false; + } + fillBlockPlaceholder(b, r.offset, v2.offset - v.offset); + break; + case 'offset': + fillBlockPlaceholder(b, r.offset, v.offset); + break; + case 'sizeof': + fillBlockPlaceholder(b, r.offset, v.hex.length / 2 / r.divisor); + break; + default: + reportError(`Invalid ref ${r.type}`); + return false; + } + } + } + return true; + } + + function save(): Uint8Array { + let total = blocks.reduce((total: number, item: HexBlock) => Math.max(item.offset + item.hex.length/2, total), 0); + + if(!options.silent) { + console.log(`${currentLineNumber} lines read; ${blocks.length} sections to write. Total file size = ${total} bytes.` ); + } + let buffer = new Uint8Array(total); + blocks.forEach(item => { + let buf = Buffer.from(item.hex, 'hex'); + buffer.set(buf, item.offset); + }); + + if(outputFilename) { + fs.writeFileSync(outputFilename, buffer); + } + return buffer; } - return buffer; } \ No newline at end of file diff --git a/developer/src/kmldmlc/test/fixtures/basic.txt b/developer/src/kmldmlc/test/fixtures/basic.txt index 850defcb94..38171949b9 100644 --- a/developer/src/kmldmlc/test/fixtures/basic.txt +++ b/developer/src/kmldmlc/test/fixtures/basic.txt @@ -1,4 +1,4 @@ -0x0000: # struct COMP_KEYBOARD { +block(kmxheader) # struct COMP_KEYBOARD { 4b 58 54 53 # KMX_DWORD dwIdentifier; // 0000 Keyman compiled keyboard id 00 10 00 00 # KMX_DWORD dwFileVersion; // 0004 Version of the file - Keyman 4.0 is 0x0400 @@ -26,72 +26,70 @@ # }; -0x0040: # struct COMP_KEYBOARD_KMXPLUSINFO { - 48 00 00 00 # KMX_DWORD dpKMXPlus; // 0040 offset of KMXPlus data, header is first - 52 01 00 00 # KMX_DWORD dwKMXPlusSize; // 0044 size in bytes of entire KMXPlus data - # }; +block(kmxplusinfo) # struct COMP_KEYBOARD_KMXPLUSINFO { + offset(sect) # KMX_DWORD dpKMXPlus; // 0040 offset of KMXPlus data from BOF, header is first + diff(sect,eof) # KMX_DWORD dwKMXPlusSize; // 0044 size in bytes of entire KMXPlus data + # }; -0x0048: # struct COMP_KMXPLUS_SECT { +block(sect) # struct COMP_KMXPLUS_SECT { 73 65 63 74 # KMX_DWORD header.ident; // 0000 Section name - 30 00 00 00 # KMX_DWORD header.size; // 0004 Section length - 52 01 00 00 # KMX_DWORD total; // 0008 KMXPlus entire length + sizeof(sect) # KMX_DWORD header.size; // 0004 Section length + diff(sect,eof) # KMX_DWORD total; // 0008 KMXPlus entire length 04 00 00 00 # KMX_DWORD count; // 000C number of section headers 73 74 72 73 # KMX_DWORD sect; // 0010+ Section identity - strs - 30 00 00 00 # KMX_DWORD offset; // 0014+ Section offset relative to dpKMXPlus of section + diff(sect,strs) # KMX_DWORD offset; // 0014+ Section offset relative to dpKMXPlus of section 6d 65 74 61 # KMX_DWORD sect; // 0010+ Section identity - meta - ea 00 00 00 # KMX_DWORD offset; // 0014+ Section offset relative to dpKMXPlus of section + diff(sect,meta) # KMX_DWORD offset; // 0014+ Section offset relative to dpKMXPlus of section 6c 6f 63 61 # KMX_DWORD sect; // 0010+ Section identity - loca - 0e 01 00 00 # KMX_DWORD offset; // 0014+ Section offset relative to dpKMXPlus of section + diff(sect,loca) # KMX_DWORD offset; // 0014+ Section offset relative to dpKMXPlus of section 6b 65 79 73 # KMX_DWORD sect; // 0010+ Section identity - keys - 22 01 00 00 # KMX_DWORD offset; // 0014+ Section offset relative to dpKMXPlus of section + diff(sect,keys) # KMX_DWORD offset; // 0014+ Section offset relative to dpKMXPlus of section # }; -0x0078: # struct COMP_KMXPLUS_STRS { - 73 74 72 73 # KMX_DWORD header.ident; // 0000 Section name - strs - ba 00 00 00 # KMX_DWORD header.size; // 0004 Section length - 09 00 00 00 # KMX_DWORD count; // 0008 count of str entries - 00 00 00 00 # KMX_DWORD reserved; // 000C padding +block(strs) # struct COMP_KMXPLUS_STRS { + 73 74 72 73 # KMX_DWORD header.ident; // 0000 Section name - strs + diff(strs,endstrs) # KMX_DWORD header.size; // 0004 Section length + 09 00 00 00 # KMX_DWORD count; // 0008 count of str entries + 00 00 00 00 # KMX_DWORD reserved; // 000C padding - 58 00 00 00 # KMX_DWORD offset; // 0010+ offset from this blob - 07 00 00 00 # KMX_DWORD length; // 0014+ str length (UTF-16LE units) - 68 00 00 00 # KMX_DWORD offset; // 0010+ offset from this blob - 06 00 00 00 # KMX_DWORD length; // 0014+ str length (UTF-16LE units) - 76 00 00 00 # KMX_DWORD offset; // 0010+ offset from this blob - 0b 00 00 00 # KMX_DWORD length; // 0014+ str length (UTF-16LE units) - 8e 00 00 00 # KMX_DWORD offset; // 0010+ offset from this blob - 06 00 00 00 # KMX_DWORD length; // 0014+ str length (UTF-16LE units) - 9c 00 00 00 # KMX_DWORD offset; // 0010+ offset from this blob - 03 00 00 00 # KMX_DWORD length; // 0014+ str length (UTF-16LE units) - a4 00 00 00 # KMX_DWORD offset; // 0010+ offset from this blob - 02 00 00 00 # KMX_DWORD length; // 0014+ str length (UTF-16LE units) - aa 00 00 00 # KMX_DWORD offset; // 0010+ offset from this blob - 02 00 00 00 # KMX_DWORD length; // 0014+ str length (UTF-16LE units) - b0 00 00 00 # KMX_DWORD offset; // 0010+ offset from this blob - 01 00 00 00 # KMX_DWORD length; // 0014+ str length (UTF-16LE units) - b4 00 00 00 # KMX_DWORD offset; // 0010+ offset from this blob - 02 00 00 00 # KMX_DWORD length; // 0014+ str length (UTF-16LE units) + # Next sections are string entries + # KMX_DWORD offset; // 0010+ offset from this blob + # KMX_DWORD length; // 0014+ str length (UTF-16LE units) + + diff(strs,strName) sizeof(strName,2) + diff(strs,strAuthor) sizeof(strAuthor,2) + diff(strs,strConformsTo) sizeof(strConformsTo,2) + diff(strs,strLayout) sizeof(strLayout,2) + diff(strs,strNorm) sizeof(strNorm,2) + diff(strs,strIndicator) sizeof(strIndicator,2) + diff(strs,strLocale) sizeof(strLocale,2) + diff(strs,strKey1) sizeof(strKey1,2) + diff(strs,strKey2) sizeof(strKey2,2) # }; -# String table +# String table -- block(x) is used to store the null u16char at end of each string +# without interfering with sizeof() calculation above - 54 00 65 00 73 00 74 00 4b 00 62 00 64 00 00 00 # 0:TestKbd - 73 00 72 00 6c 00 32 00 39 00 35 00 00 00 # 1:srl295 - 74 00 65 00 63 00 68 00 70 00 72 00 65 00 76 00 - 69 00 65 00 77 00 00 00 # 2:techpreview - 71 00 77 00 65 00 72 00 74 00 79 00 00 00 # 3:qwerty - 4e 00 46 00 43 00 00 00 # 4:NFC - 3d d8 40 de 00 00 # 5:🙀 - 6d 00 74 00 00 00 # 6:mt - 27 01 00 00 # 7:ħ - 90 17 b6 17 00 00 # 8:ថា +block(strName) 54 00 65 00 73 00 74 00 4b 00 62 00 64 00 block(x) 00 00 # 0:TestKbd +block(strAuthor) 73 00 72 00 6c 00 32 00 39 00 35 00 block(x) 00 00 # 1:srl295 +block(strConformsTo) 74 00 65 00 63 00 68 00 70 00 72 00 65 00 76 00 + 69 00 65 00 77 00 block(x) 00 00 # 2:techpreview +block(strLayout) 71 00 77 00 65 00 72 00 74 00 79 00 block(x) 00 00 # 3:qwerty +block(strNorm) 4e 00 46 00 43 00 block(x) 00 00 # 4:NFC +block(strIndicator) 3d d8 40 de block(x) 00 00 # 5:🙀 +block(strLocale) 6d 00 74 00 block(x) 00 00 # 7:mt +block(strKey1) 27 01 block(x) 00 00 # 8:ħ +block(strKey2) 90 17 b6 17 block(x) 00 00 # 9:ថា -0x132: # struct COMP_KMXPLUS_META { - 6d 65 74 61 # KMX_DWORD header.ident; // 0000 Section name - meta - 24 00 00 00 # KMX_DWORD header.size; // 0004 Section length +block(endstrs) # end of strs block + +block(meta) # struct COMP_KMXPLUS_META { + 6d 65 74 61 # KMX_DWORD header.ident; // 0000 Section name - meta + sizeof(meta) # KMX_DWORD header.size; // 0004 Section length 00 00 00 00 # KMX_DWORD name; 01 00 00 00 # KMX_DWORD author; 02 00 00 00 # KMX_DWORD conform; @@ -101,17 +99,17 @@ 00 00 00 00 # KMX_DWORD settings; # }; -0x0156: # struct COMP_KMXPLUS_LOCA { +block(loca) # struct COMP_KMXPLUS_LOCA { 6c 6f 63 61 # KMX_DWORD header.ident; // 0000 Section name - loca - 14 00 00 00 # KMX_DWORD header.size; // 0004 Section length + sizeof(loca) # KMX_DWORD header.size; // 0004 Section length 01 00 00 00 # KMX_DWORD count; // 0008 number of locales 00 00 00 00 # KMX_DWORD reserved; 06 00 00 00 # KMX_DWORD locale; // 0010+ locale string entry = 'mt' # }; -0x016a: # struct COMP_KMXPLUS_KEYS { +block(keys) # struct COMP_KMXPLUS_KEYS { 6b 65 79 73 # KMX_DWORD header.ident; // 0000 Section name - keys - 30 00 00 00 # KMX_DWORD header.size; // 0004 Section length + sizeof(keys) # KMX_DWORD header.size; // 0004 Section length 02 00 00 00 # KMX_DWORD count; // number of keys 00 00 00 00 # KMX_DWORD reserved; // padding # }; @@ -120,3 +118,5 @@ c0 00 00 00 00 00 00 00 07 00 00 00 01 00 00 00 # KMX_DWORD vkey, mod, to, flags; 31 00 00 00 00 00 00 00 08 00 00 00 01 00 00 00 # KMX_DWORD vkey, mod, to, flags; + +block(eof) # end of file \ No newline at end of file