diff --git a/developer/src/kmc/build.sh b/developer/src/kmc/build.sh index 165bad15e7..7cfcfce705 100755 --- a/developer/src/kmc/build.sh +++ b/developer/src/kmc/build.sh @@ -60,7 +60,7 @@ fi if builder_has_action build-fixtures; then # Build basic.kmx and emit its checksum mkdir -p ./build/test/fixtures - node . ./test/fixtures/basic.xml -o ./build/test/fixtures/basic-xml.kmx + node . ./test/fixtures/basic.xml --no-compiler-version --debug --out-file ./build/test/fixtures/basic-xml.kmx printf "${COLOR_GREY}Checksum for basic-xml.kmx: ${COLOR_PURPLE}%s${COLOR_RESET}\n" \ "$(xxd -g 1 -l 12 ./build/test/fixtures/basic-xml.kmx | cut -d' ' -f 10-13)" diff --git a/developer/src/kmc/src/keyman/compiler/compiler-options.ts b/developer/src/kmc/src/keyman/compiler/compiler-options.ts new file mode 100644 index 0000000000..76dd4b5ba4 --- /dev/null +++ b/developer/src/kmc/src/keyman/compiler/compiler-options.ts @@ -0,0 +1,13 @@ + + +export default interface CompilerOptions { + /** + * Add debug information to the .kmx file when compiling + */ + debug: boolean; + + /** + * Add metadata about the compiler version to .kmx file when compiling + */ + addCompilerVersion: boolean; +}; diff --git a/developer/src/kmc/src/keyman/compiler/compiler.ts b/developer/src/kmc/src/keyman/compiler/compiler.ts index 56a9e31504..2565338024 100644 --- a/developer/src/kmc/src/keyman/compiler/compiler.ts +++ b/developer/src/kmc/src/keyman/compiler/compiler.ts @@ -3,6 +3,7 @@ import LDMLKeyboardXMLSourceFile from '../ldml-keyboard/ldml-keyboard-xml'; import LDMLKeyboardXMLSourceFileReader from '../ldml-keyboard/ldml-keyboard-xml-reader'; import { BkspCompiler } from './bksp'; import CompilerCallbacks from './callbacks'; +import CompilerOptions from './compiler-options'; import { KeysCompiler } from './keys'; import { LocaCompiler } from './loca'; import { MetaCompiler } from './meta'; @@ -25,8 +26,16 @@ const SECTION_COMPILERS = [ export default class Compiler { private readonly callbacks: CompilerCallbacks; + // private readonly options: CompilerOptions; // not currently used - constructor (callbacks: CompilerCallbacks) { + constructor (callbacks: CompilerCallbacks, _options?: CompilerOptions) { + /* + this.options = { + debug: false, + addCompilerVersion: true, + ...options + }; + */ this.callbacks = callbacks; } diff --git a/developer/src/kmc/src/keyman/compiler/metadata-compiler.ts b/developer/src/kmc/src/keyman/compiler/metadata-compiler.ts new file mode 100644 index 0000000000..b1d50c6ed1 --- /dev/null +++ b/developer/src/kmc/src/keyman/compiler/metadata-compiler.ts @@ -0,0 +1,55 @@ +import KMXFile, { KEYBOARD } from "../kmx/kmx"; +import { KMXPlusData } from "../kmx/kmx-plus"; +import CompilerOptions from "./compiler-options"; + +const KEYMAN_VERSION = require("@keymanapp/keyman-version").KEYMAN_VERSION; + +export default class KMXPlusMetadataCompiler { + /** + * Look for metadata fields in the KMXPlus data and copy them + * through to the relevant KMX stores + * @param kmxplus const KMXPlusData + */ + public static addKmxMetadata(kmxplus: KMXPlusData, keyboard: KEYBOARD, options: CompilerOptions): void { + // Order of stores is not significant by kmx spec, but kmxplus compiler will + // always store according to dwSystemID binary order for non-zero + // dwSystemID, then by dpName binary order, and finally by dpString binary + // order + + // TSS_NAME = 7 + // TSS_COMPILEDVERSION = 20 + // TSS_KEYBOARDVERSION = 36 + // TSS_TARGETS = 38 + + // TSS_NAME: User friendly name of keyboard + keyboard.stores.push({ + dpName: '&NAME', + dpString: kmxplus.name?.names?.[0]?.value ?? 'unknown', // Empty name should not happen, so ok to use 'unknown' here + dwSystemID: KMXFile.TSS_NAME + }); + + if(options.addCompilerVersion) { + // TSS_COMPILEDVERSION: version of the compiler + keyboard.stores.push({ + dpName: '', + dpString: KEYMAN_VERSION.VERSION_WITH_TAG, + dwSystemID: KMXFile.TSS_COMPILEDVERSION + }); + } + + // TSS_KEYBOARDVERSION: Version of the keyboard, should be semver + keyboard.stores.push({ + dpName: '&KEYBOARDVERSION', + dpString: kmxplus.meta?.version?.value ?? '1.0', // 1.0 is inferred version + dwSystemID: KMXFile.TSS_KEYBOARDVERSION + }); + + // TSS_TARGETS: which platforms are supported + keyboard.stores.push({ + dpName: '&TARGETS', + dpString: 'desktop', // TODO-LDML: support touch layouts in #7238 + dwSystemID: KMXFile.TSS_TARGETS + }); + } + +} \ No newline at end of file diff --git a/developer/src/kmc/src/kmc.ts b/developer/src/kmc/src/kmc.ts index be874ebf78..70d36e86d0 100644 --- a/developer/src/kmc/src/kmc.ts +++ b/developer/src/kmc/src/kmc.ts @@ -7,10 +7,12 @@ import * as fs from 'fs'; import * as path from 'path'; import * as program from 'commander'; +import CompilerOptions from './keyman/compiler/compiler-options'; import Compiler from './keyman/compiler/compiler'; import KMXBuilder from './keyman/kmx/kmx-builder'; import { CompilerMessages } from './keyman/compiler/messages'; import { CompilerEvent } from './keyman/compiler/callbacks'; +import KMXPlusMetadataCompiler from './keyman/compiler/metadata-compiler'; let inputFilename: string; @@ -22,7 +24,9 @@ program .version(KEYMAN_VERSION.VERSION_WITH_TAG) .arguments('') .action((infile:any) => inputFilename = infile) - .option('-o, --outFile ', 'where to save the resulting .kmx file'); + .option('-d, --debug', 'Include debug information in output') + .option('--no-compiler-version', 'Exclude compiler version metadata from output') + .option('-o, --out-file ', 'where to save the resulting .kmx file'); program.parse(process.argv); @@ -51,9 +55,9 @@ class CompilerCallbacks { } } -function compileKeyboard(inputFilename: string): Uint8Array { +function compileKeyboard(inputFilename: string, options: CompilerOptions): Uint8Array { const c = new CompilerCallbacks(); - const k = new Compiler(c); + const k = new Compiler(c, options); let source = k.load(inputFilename); if(!source) { return null; @@ -66,13 +70,22 @@ function compileKeyboard(inputFilename: string): Uint8Array { return null; } + // In order for the KMX file to be loaded by non-KMXPlus components, it is helpful + // to duplicate some of the metadata + KMXPlusMetadataCompiler.addKmxMetadata(kmx.kmxplus, kmx.keyboard, options); + // Use the builder to generate the binary output file - let builder = new KMXBuilder(kmx, true); + let builder = new KMXBuilder(kmx, options.debug); return builder.compile(); } +let options: CompilerOptions = { + debug: program.debug ?? false, + addCompilerVersion: program.compilerVersion ?? true +} + // Compile: -let code = compileKeyboard(inputFilename); +let code = compileKeyboard(inputFilename, options); // Output: diff --git a/developer/src/kmc/test/fixtures/basic.txt b/developer/src/kmc/test/fixtures/basic.txt index f93f3ccea2..02c57e5f17 100644 --- a/developer/src/kmc/test/fixtures/basic.txt +++ b/developer/src/kmc/test/fixtures/basic.txt @@ -21,15 +21,15 @@ block(kmxheader) # struct COMP_KEYBOARD { 00 10 00 00 # KMX_DWORD dwFileVersion; // 0004 Version of the file - Keyman 4.0 is 0x0400 - 26 d2 24 03 # KMX_DWORD dwCheckSum; // 0008 As stored in keyboard + c1 79 c0 dd # KMX_DWORD dwCheckSum; // 0008 As stored in keyboard 00 00 00 00 # KMX_DWORD KeyboardID; // 000C as stored in HKEY_LOCAL_MACHINE//system//currentcontrolset//control//keyboard layouts 01 00 00 00 # KMX_DWORD IsRegistered; // 0010 00 00 00 00 # KMX_DWORD version; // 0014 keyboard version - 00 00 00 00 # KMX_DWORD cxStoreArray; // 0018 in array entries - 00 00 00 00 # KMX_DWORD cxGroupArray; // 001C in array entries + sizeof(stores,12) # KMX_DWORD cxStoreArray; // 0018 in array entries + 00 00 00 00 # KMX_DWORD cxGroupArray; // 001C in array entries - 00 00 00 00 # KMX_DWORD dpStoreArray; // 0020 [LPSTORE] address of first item in store array + offset(stores) # KMX_DWORD dpStoreArray; // 0020 [LPSTORE] address of first item in store array 00 00 00 00 # KMX_DWORD dpGroupArray; // 0024 [LPGROUP] address of first item in group array ff ff ff ff # KMX_DWORD StartGroup[2]; // 0028 index of starting groups [2 of them] @@ -43,12 +43,47 @@ block(kmxheader) # struct COMP_KEYBOARD { 00 00 00 00 # KMX_DWORD dwBitmapSize; // 003C size in bytes of the bitmaps # }; - block(kmxplusinfo) # struct COMP_KEYBOARD_KMXPLUSINFO { offset(sect) # KMX_DWORD dpKMXPlus; // 0040 offset of KMXPlus data from BOF, header is first diff(sect,eof) # KMX_DWORD dwKMXPlusSize; // 0044 size in bytes of entire KMXPlus data # }; +block(stores) # struct COMP_STORE { + 07 00 00 00 # KMX_DWORD dwSystemID; - TSS_NAME + offset(store_name_name) # KMX_DWORD dpName; + offset(store_name_string) # KMX_DWORD dpString; + # }; + # excluding this so we don't have compiled version changes + # 14 00 00 00 # KMX_DWORD dwSystemID; - TSS_COMPILEDVERSION + # offset(store_compiledversion_name) # KMX_DWORD dpName; + # offset(store_compiledversion_string) # KMX_DWORD dpString; + # }; + 24 00 00 00 # KMX_DWORD dwSystemID; - TSS_KEYBOARDVERSION + offset(store_keyboardversion_name) # KMX_DWORD dpName; + offset(store_keyboardversion_string) # KMX_DWORD dpString; + # }; + 26 00 00 00 # KMX_DWORD dwSystemID; - TSS_TARGETS + offset(store_targets_name) # KMX_DWORD dpName; + offset(store_targets_string) # KMX_DWORD dpString; + # }; + +block(store_name_name) + 26 00 4e 00 41 00 4d 00 45 00 00 00 # '&NAME' +block(store_name_string) + 54 00 65 00 73 00 74 00 4b 00 62 00 64 00 00 00 # 'TestKbd' + +block(store_keyboardversion_name) + 26 00 4b 00 45 00 59 00 42 00 4f 00 41 00 52 00 + 44 00 56 00 45 00 52 00 53 00 49 00 4f 00 4e 00 + 00 00 # '&KEYBOARDVERSION' +block(store_keyboardversion_string) + 31 00 2e 00 30 00 2e 00 30 00 00 00 # '1.0.0' + +block(store_targets_name) + 26 00 54 00 41 00 52 00 47 00 45 00 54 00 53 00 00 00 # '&TARGETS' +block(store_targets_string) + 64 00 65 00 73 00 6b 00 74 00 6f 00 70 00 00 00 # 'desktop' + block(sect) # struct COMP_KMXPLUS_SECT { 73 65 63 74 # KMX_DWORD header.ident; // 0000 Section name diff(sect,endsect) # KMX_DWORD header.size; // 0004 Section length @@ -296,7 +331,7 @@ block(strs) # struct COMP_KMXPLUS_STRS { # without interfering with sizeof() calculation above block(strNull) block(x) 00 00 # the zero-length string - block(strVersion) 30 00 block(x) 00 00 # '0' + block(strVersion) 31 00 2e 00 30 00 2e 00 30 00 block(x) 00 00 # '1.0.0' block(strNorm) 4e 00 46 00 43 00 block(x) 00 00 # 'NFC' block(strName) 54 00 65 00 73 00 74 00 4b 00 62 00 64 00 block(x) 00 00 # 'TestKbd' block(strElemTranFrom1) 5E 00 block(x) 00 00 # '^' diff --git a/developer/src/kmc/test/fixtures/basic.xml b/developer/src/kmc/test/fixtures/basic.xml index 9778a39c05..290429a98e 100644 --- a/developer/src/kmc/test/fixtures/basic.xml +++ b/developer/src/kmc/test/fixtures/basic.xml @@ -6,6 +6,8 @@ --> + + diff --git a/developer/src/kmc/test/helpers/index.ts b/developer/src/kmc/test/helpers/index.ts index e028caab8b..e33c63c293 100644 --- a/developer/src/kmc/test/helpers/index.ts +++ b/developer/src/kmc/test/helpers/index.ts @@ -4,9 +4,13 @@ import * as path from 'path'; import * as fs from 'fs'; import { SectionCompiler } from '../../src/keyman/compiler/section-compiler'; -import { Elem, GlobalSections, Section, Strs } from '../../src/keyman/kmx/kmx-plus'; +import KMXPlusFile, { Elem, GlobalSections, Section, Strs } from '../../src/keyman/kmx/kmx-plus'; import LDMLKeyboardXMLSourceFileReader from '../../src/keyman/ldml-keyboard/ldml-keyboard-xml-reader'; -import { CompilerEvent } from '../keyman/compiler/callbacks'; +import { CompilerEvent } from '../../src/keyman/compiler/callbacks'; +import Compiler from '../../src/keyman/compiler/compiler'; +import { assert } from 'chai'; +import KMXPlusMetadataCompiler from '../../src/keyman/compiler/metadata-compiler'; +import CompilerOptions from '../../src/keyman/compiler/compiler-options'; /** * Builds a path to the fixture with the given path components. @@ -50,3 +54,31 @@ export function loadSectionFixture(compilerClass: typeof SectionCompiler, filena return compiler.compile(globalSections); } + +export function compileKeyboard(inputFilename: string, callbacks: CompilerCallbacks, options: CompilerOptions): KMXPlusFile { + const k = new Compiler(callbacks, options); + const source = k.load(inputFilename); + checkMessages(callbacks); + assert.isNotNull(source, 'k.load should not have returned null'); + + const valid = k.validate(source); + checkMessages(callbacks); + assert.isTrue(valid, 'k.validate should not have failed'); + + const kmx = k.compile(source); + checkMessages(callbacks); + assert.isNotNull(kmx, 'k.compile should not have returned null'); + + // In order for the KMX file to be loaded by non-KMXPlus components, it is helpful + // to duplicate some of the metadata + KMXPlusMetadataCompiler.addKmxMetadata(kmx.kmxplus, kmx.keyboard, options); + + return kmx; +} + +export function checkMessages(callbacks: CompilerCallbacks) { + if(callbacks.messages.length > 0) { + console.log(callbacks.messages); + } + assert.isEmpty(callbacks.messages); +} diff --git a/developer/src/kmc/test/test-compiler-e2e.ts b/developer/src/kmc/test/test-compiler-e2e.ts index 145bf2a90e..c59792bfdd 100644 --- a/developer/src/kmc/test/test-compiler-e2e.ts +++ b/developer/src/kmc/test/test-compiler-e2e.ts @@ -1,40 +1,8 @@ import 'mocha'; -import * as fs from 'fs'; -import * as path from 'path'; import {assert} from 'chai'; import hextobin from '@keymanapp/hextobin'; -import Compiler from '../src/keyman/compiler/compiler'; import KMXBuilder from '../src/keyman/kmx/kmx-builder'; -import {CompilerCallbacks, makePathToFixture} from './helpers/index'; - -function checkMessages(callbacks: CompilerCallbacks) { - if(callbacks.messages.length > 0) { - console.log(callbacks.messages); - } - assert.isEmpty(callbacks.messages); -} - -function compileKeyboard(inputFilename: string): Uint8Array { - const callbacks = new CompilerCallbacks(); - const k = new Compiler(callbacks); - const source = k.load(inputFilename); - checkMessages(callbacks); - assert.isNotNull(source, 'k.load should not have returned null'); - - const valid = k.validate(source); - checkMessages(callbacks); - assert.isTrue(valid, 'k.validate should not have failed'); - - const kmx = k.compile(source); - checkMessages(callbacks); - assert.isNotNull(kmx, 'k.compile should not have returned null'); - - // Use the builder to generate the binary output file - const builder = new KMXBuilder(kmx, true); - const result = builder.compile(); - checkMessages(callbacks); - return result; -} +import {checkMessages, compileKeyboard, CompilerCallbacks, makePathToFixture} from './helpers/index'; describe('compiler-tests', function() { this.slow(500); // 0.5 sec -- json schema validation takes a while @@ -44,14 +12,21 @@ describe('compiler-tests', function() { // It should match basic.kmx (built from basic.txt) const inputFilename = makePathToFixture('basic.xml'); - const outputFilename = makePathToFixture('basic.txt'); + const binaryFilename = makePathToFixture('basic.txt'); - // Compile: - const code = compileKeyboard(inputFilename); + // Compile the keyboard + const callbacks = new CompilerCallbacks(); + const kmx = compileKeyboard(inputFilename, callbacks, {debug: true, addCompilerVersion: false}); + assert.isNotNull(kmx); + + // Use the builder to generate the binary output file + const builder = new KMXBuilder(kmx, true); + const code = builder.compile(); + checkMessages(callbacks); assert.isNotNull(code); // Compare output - let expected = await hextobin(outputFilename, undefined, {silent:true}); + let expected = await hextobin(binaryFilename, undefined, {silent:true}); assert.deepEqual(code, expected); }); }); \ No newline at end of file diff --git a/developer/src/kmc/test/test-metadata-compiler.ts b/developer/src/kmc/test/test-metadata-compiler.ts new file mode 100644 index 0000000000..1bb451faae --- /dev/null +++ b/developer/src/kmc/test/test-metadata-compiler.ts @@ -0,0 +1,76 @@ +import 'mocha'; +import {assert} from 'chai'; +import { checkMessages, compileKeyboard, CompilerCallbacks, makePathToFixture } from './helpers'; +import KMXFile from '../src/keyman/kmx/kmx'; + +const KEYMAN_VERSION = require("@keymanapp/keyman-version").KEYMAN_VERSION; + +describe('kmx metadata compiler', function () { + this.slow(500); // 0.5 sec -- json schema validation takes a while + + it('should compile metadata with debug and compiler version', function() { + debugger + const callbacks = new CompilerCallbacks(); + const inputFilename = makePathToFixture('basic.xml'); + + // Compile the keyboard + const kmx = compileKeyboard(inputFilename, callbacks, {debug:true, addCompilerVersion:true}); + checkMessages(callbacks); + assert.isNotNull(kmx); + + // Order of stores is not significant in kmx spec, but kmxplus compiler will + // always store according to dwSystemID binary order for non-zero + // dwSystemID, then by dpName binary order, and finally by dpString binary + // order + + // TSS_NAME = 7 + // TSS_COMPILEDVERSION = 20 + // TSS_KEYBOARDVERSION = 36 + // TSS_TARGETS = 38 + + assert.equal(kmx.keyboard.stores[0].dpName, '&NAME'); + assert.equal(kmx.keyboard.stores[0].dpString, 'TestKbd'); + assert.equal(kmx.keyboard.stores[0].dwSystemID, KMXFile.TSS_NAME); + + assert.equal(kmx.keyboard.stores[1].dpName, ''); + assert.equal(kmx.keyboard.stores[1].dpString, KEYMAN_VERSION.VERSION_WITH_TAG); + assert.equal(kmx.keyboard.stores[1].dwSystemID, KMXFile.TSS_COMPILEDVERSION); + + assert.equal(kmx.keyboard.stores[2].dpName, '&KEYBOARDVERSION'); + assert.equal(kmx.keyboard.stores[2].dpString, '1.0.0'); + assert.equal(kmx.keyboard.stores[2].dwSystemID, KMXFile.TSS_KEYBOARDVERSION); + + assert.equal(kmx.keyboard.stores[3].dpName, '&TARGETS'); + assert.equal(kmx.keyboard.stores[3].dpString, 'desktop'); + assert.equal(kmx.keyboard.stores[3].dwSystemID, KMXFile.TSS_TARGETS); + }); + + it('should compile metadata with no compiler version', function() { + const callbacks = new CompilerCallbacks(); + const inputFilename = makePathToFixture('basic.xml'); + + // Compile the keyboard + const kmx = compileKeyboard(inputFilename, callbacks, {debug:true, addCompilerVersion:false}); + checkMessages(callbacks); + assert.isNotNull(kmx); + + // TSS_NAME = 7 + // -- TSS_COMPILEDVERSION = 20 -- not included with addCompilerVersion = false + // TSS_KEYBOARDVERSION = 36 + // TSS_TARGETS = 38 + + assert.equal(kmx.keyboard.stores[0].dpName, '&NAME'); + assert.equal(kmx.keyboard.stores[0].dpString, 'TestKbd'); + assert.equal(kmx.keyboard.stores[0].dwSystemID, KMXFile.TSS_NAME); + + assert.equal(kmx.keyboard.stores[1].dpName, '&KEYBOARDVERSION'); + assert.equal(kmx.keyboard.stores[1].dpString, '1.0.0'); + assert.equal(kmx.keyboard.stores[1].dwSystemID, KMXFile.TSS_KEYBOARDVERSION); + + assert.equal(kmx.keyboard.stores[2].dpName, '&TARGETS'); + assert.equal(kmx.keyboard.stores[2].dpString, 'desktop'); + assert.equal(kmx.keyboard.stores[2].dwSystemID, KMXFile.TSS_TARGETS); + }); + +}); + diff --git a/developer/src/kmc/test/tsconfig.json b/developer/src/kmc/test/tsconfig.json index 2a126d1ffe..faca8098d5 100644 --- a/developer/src/kmc/test/tsconfig.json +++ b/developer/src/kmc/test/tsconfig.json @@ -16,6 +16,7 @@ "./helpers/index.ts" ], "references": [ + { "path": "../../../../common/web/keyman-version" }, { "path": "../" } ] } \ No newline at end of file