erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-08-07 09:25:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 17

feat(developer): add kmx metadata to kmxplus compiles

Browse source 
Fixes #7273.

Adds 4 kmx system stores to allow existing configuration tools to load
and process KMXPlus files without modification:

* `TSS_NAME` (first `<name>`)
* `TSS_COMPILEDVERSION` (only if `options.addCompilerVersion == true`)
* `TSS_KEYBOARDVERSION` (from `<version number>`)
* `TSS_TARGETS` (currently always 'desktop', will later include more
  targets)

In order for `TSS_COMPILEDVERSION` to be handled correctly, needed to
plumb in the new `CompilerOptions` interface. This could be a separate
PR, but the changes are not huge, so opted to append the changes here.

Updates related tests and fixtures. Refactored some of the e2e helpers
into helpers/index.ts for use with the metadata-compiler tests.
This commit is contained in:
Marc Durdin 2022-09-14 09:26:22 +10:00
parent c820fbc0e6
commit 110f6ffbbd
11 changed files with 263 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
developer/src/kmc/build.sh
View file

@ -60,7 +60,7 @@ fi
if builder_has_action build-fixtures; then
# Build basic.kmx and emit its checksum
mkdir -p ./build/test/fixtures
node . ./test/fixtures/basic.xml -o ./build/test/fixtures/basic-xml.kmx
node . ./test/fixtures/basic.xml --no-compiler-version --debug --out-file ./build/test/fixtures/basic-xml.kmx
printf "${COLOR_GREY}Checksum for basic-xml.kmx: ${COLOR_PURPLE}%s${COLOR_RESET}\n" \
"$(xxd -g 1 -l 12 ./build/test/fixtures/basic-xml.kmx | cut -d' ' -f 10-13)"

13
developer/src/kmc/src/keyman/compiler/compiler-options.ts Normal file
View file

@ -0,0 +1,13 @@
export default interface CompilerOptions {
/**
* Add debug information to the .kmx file when compiling
*/
debug: boolean;
/**
* Add metadata about the compiler version to .kmx file when compiling
*/
addCompilerVersion: boolean;
};

11
developer/src/kmc/src/keyman/compiler/compiler.ts
View file

@ -3,6 +3,7 @@ import LDMLKeyboardXMLSourceFile from '../ldml-keyboard/ldml-keyboard-xml';
import LDMLKeyboardXMLSourceFileReader from '../ldml-keyboard/ldml-keyboard-xml-reader';
import { BkspCompiler } from './bksp';
import CompilerCallbacks from './callbacks';
import CompilerOptions from './compiler-options';
import { KeysCompiler } from './keys';
import { LocaCompiler } from './loca';
import { MetaCompiler } from './meta';
@ -25,8 +26,16 @@ const SECTION_COMPILERS = [
export default class Compiler {
private readonly callbacks: CompilerCallbacks;
// private readonly options: CompilerOptions; // not currently used
constructor (callbacks: CompilerCallbacks) {
constructor (callbacks: CompilerCallbacks, _options?: CompilerOptions) {
/*
this.options = {
debug: false,
addCompilerVersion: true,
...options
};
*/
this.callbacks = callbacks;
}

55
developer/src/kmc/src/keyman/compiler/metadata-compiler.ts Normal file
View file

@ -0,0 +1,55 @@
import KMXFile, { KEYBOARD } from "../kmx/kmx";
import { KMXPlusData } from "../kmx/kmx-plus";
import CompilerOptions from "./compiler-options";
const KEYMAN_VERSION = require("@keymanapp/keyman-version").KEYMAN_VERSION;
export default class KMXPlusMetadataCompiler {
/**
* Look for metadata fields in the KMXPlus data and copy them
* through to the relevant KMX stores
* @param kmxplus const KMXPlusData
*/
public static addKmxMetadata(kmxplus: KMXPlusData, keyboard: KEYBOARD, options: CompilerOptions): void {
// Order of stores is not significant by kmx spec, but kmxplus compiler will
// always store according to dwSystemID binary order for non-zero
// dwSystemID, then by dpName binary order, and finally by dpString binary
// order
// TSS_NAME = 7
// TSS_COMPILEDVERSION = 20
// TSS_KEYBOARDVERSION = 36
// TSS_TARGETS = 38
// TSS_NAME: User friendly name of keyboard
keyboard.stores.push({
dpName: '&NAME',
dpString: kmxplus.name?.names?.[0]?.value ?? 'unknown', // Empty name should not happen, so ok to use 'unknown' here
dwSystemID: KMXFile.TSS_NAME
});
if(options.addCompilerVersion) {
// TSS_COMPILEDVERSION: version of the compiler
keyboard.stores.push({
dpName: '',
dpString: KEYMAN_VERSION.VERSION_WITH_TAG,
dwSystemID: KMXFile.TSS_COMPILEDVERSION
});
}
// TSS_KEYBOARDVERSION: Version of the keyboard, should be semver
keyboard.stores.push({
dpName: '&KEYBOARDVERSION',
dpString: kmxplus.meta?.version?.value ?? '1.0', // 1.0 is inferred version
dwSystemID: KMXFile.TSS_KEYBOARDVERSION
});
// TSS_TARGETS: which platforms are supported
keyboard.stores.push({
dpName: '&TARGETS',
dpString: 'desktop', // TODO-LDML: support touch layouts in #7238
dwSystemID: KMXFile.TSS_TARGETS
});
}
}

23
developer/src/kmc/src/kmc.ts
View file

@ -7,10 +7,12 @@ import * as fs from 'fs';
import * as path from 'path';
import * as program from 'commander';
import CompilerOptions from './keyman/compiler/compiler-options';
import Compiler from './keyman/compiler/compiler';
import KMXBuilder from './keyman/kmx/kmx-builder';
import { CompilerMessages } from './keyman/compiler/messages';
import { CompilerEvent } from './keyman/compiler/callbacks';
import KMXPlusMetadataCompiler from './keyman/compiler/metadata-compiler';
let inputFilename: string;
@ -22,7 +24,9 @@ program
.version(KEYMAN_VERSION.VERSION_WITH_TAG)
.arguments('<infile>')
.action((infile:any) => inputFilename = infile)
.option('-o, --outFile <filename>', 'where to save the resulting .kmx file');
.option('-d, --debug', 'Include debug information in output')
.option('--no-compiler-version', 'Exclude compiler version metadata from output')
.option('-o, --out-file <filename>', 'where to save the resulting .kmx file');
program.parse(process.argv);
@ -51,9 +55,9 @@ class CompilerCallbacks {
}
}
function compileKeyboard(inputFilename: string): Uint8Array {
function compileKeyboard(inputFilename: string, options: CompilerOptions): Uint8Array {
const c = new CompilerCallbacks();
const k = new Compiler(c);
const k = new Compiler(c, options);
let source = k.load(inputFilename);
if(!source) {
return null;
@ -66,13 +70,22 @@ function compileKeyboard(inputFilename: string): Uint8Array {
return null;
}
// In order for the KMX file to be loaded by non-KMXPlus components, it is helpful
// to duplicate some of the metadata
KMXPlusMetadataCompiler.addKmxMetadata(kmx.kmxplus, kmx.keyboard, options);
// Use the builder to generate the binary output file
let builder = new KMXBuilder(kmx, true);
let builder = new KMXBuilder(kmx, options.debug);
return builder.compile();
}
let options: CompilerOptions = {
debug: program.debug ?? false,
addCompilerVersion: program.compilerVersion ?? true
}
// Compile:
let code = compileKeyboard(inputFilename);
let code = compileKeyboard(inputFilename, options);
// Output:

47
developer/src/kmc/test/fixtures/basic.txt vendored
View file

@ -21,15 +21,15 @@ block(kmxheader) # struct COMP_KEYBOARD {
00 10 00 00 # KMX_DWORD dwFileVersion; // 0004 Version of the file - Keyman 4.0 is 0x0400
26 d2 24 03 # KMX_DWORD dwCheckSum; // 0008 As stored in keyboard
c1 79 c0 dd # KMX_DWORD dwCheckSum; // 0008 As stored in keyboard
00 00 00 00 # KMX_DWORD KeyboardID; // 000C as stored in HKEY_LOCAL_MACHINE//system//currentcontrolset//control//keyboard layouts
01 00 00 00 # KMX_DWORD IsRegistered; // 0010
00 00 00 00 # KMX_DWORD version; // 0014 keyboard version
00 00 00 00 # KMX_DWORD cxStoreArray; // 0018 in array entries
00 00 00 00 # KMX_DWORD cxGroupArray; // 001C in array entries
sizeof(stores,12) # KMX_DWORD cxStoreArray; // 0018 in array entries
00 00 00 00 # KMX_DWORD cxGroupArray; // 001C in array entries
00 00 00 00 # KMX_DWORD dpStoreArray; // 0020 [LPSTORE] address of first item in store array
offset(stores) # KMX_DWORD dpStoreArray; // 0020 [LPSTORE] address of first item in store array
00 00 00 00 # KMX_DWORD dpGroupArray; // 0024 [LPGROUP] address of first item in group array
ff ff ff ff # KMX_DWORD StartGroup[2]; // 0028 index of starting groups [2 of them]
@ -43,12 +43,47 @@ block(kmxheader) # struct COMP_KEYBOARD {
00 00 00 00 # KMX_DWORD dwBitmapSize; // 003C size in bytes of the bitmaps
# };
block(kmxplusinfo) # struct COMP_KEYBOARD_KMXPLUSINFO {
offset(sect) # KMX_DWORD dpKMXPlus; // 0040 offset of KMXPlus data from BOF, <sect> header is first
diff(sect,eof) # KMX_DWORD dwKMXPlusSize; // 0044 size in bytes of entire KMXPlus data
# };
block(stores) # struct COMP_STORE {
07 00 00 00 # KMX_DWORD dwSystemID; - TSS_NAME
offset(store_name_name) # KMX_DWORD dpName;
offset(store_name_string) # KMX_DWORD dpString;
# };
# excluding this so we don't have compiled version changes
# 14 00 00 00 # KMX_DWORD dwSystemID; - TSS_COMPILEDVERSION
# offset(store_compiledversion_name) # KMX_DWORD dpName;
# offset(store_compiledversion_string) # KMX_DWORD dpString;
# };
24 00 00 00 # KMX_DWORD dwSystemID; - TSS_KEYBOARDVERSION
offset(store_keyboardversion_name) # KMX_DWORD dpName;
offset(store_keyboardversion_string) # KMX_DWORD dpString;
# };
26 00 00 00 # KMX_DWORD dwSystemID; - TSS_TARGETS
offset(store_targets_name) # KMX_DWORD dpName;
offset(store_targets_string) # KMX_DWORD dpString;
# };
block(store_name_name)
26 00 4e 00 41 00 4d 00 45 00 00 00 # '&NAME'
block(store_name_string)
54 00 65 00 73 00 74 00 4b 00 62 00 64 00 00 00 # 'TestKbd'
block(store_keyboardversion_name)
26 00 4b 00 45 00 59 00 42 00 4f 00 41 00 52 00
44 00 56 00 45 00 52 00 53 00 49 00 4f 00 4e 00
00 00 # '&KEYBOARDVERSION'
block(store_keyboardversion_string)
31 00 2e 00 30 00 2e 00 30 00 00 00 # '1.0.0'
block(store_targets_name)
26 00 54 00 41 00 52 00 47 00 45 00 54 00 53 00 00 00 # '&TARGETS'
block(store_targets_string)
64 00 65 00 73 00 6b 00 74 00 6f 00 70 00 00 00 # 'desktop'
block(sect) # struct COMP_KMXPLUS_SECT {
73 65 63 74 # KMX_DWORD header.ident; // 0000 Section name
diff(sect,endsect) # KMX_DWORD header.size; // 0004 Section length
@ -296,7 +331,7 @@ block(strs) # struct COMP_KMXPLUS_STRS {
# without interfering with sizeof() calculation above
block(strNull) block(x) 00 00 # the zero-length string
block(strVersion) 30 00 block(x) 00 00 # '0'
block(strVersion) 31 00 2e 00 30 00 2e 00 30 00 block(x) 00 00 # '1.0.0'
block(strNorm) 4e 00 46 00 43 00 block(x) 00 00 # 'NFC'
block(strName) 54 00 65 00 73 00 74 00 4b 00 62 00 64 00 block(x) 00 00 # 'TestKbd'
block(strElemTranFrom1) 5E 00 block(x) 00 00 # '^'

2
developer/src/kmc/test/fixtures/basic.xml vendored
View file

@ -6,6 +6,8 @@
-->
<!DOCTYPE keyboard SYSTEM "../../../../../resources/standards-data/ldml-keyboards/techpreview/ldmlKeyboard.dtd">
<keyboard locale="mt" conformsTo="techpreview">
<version number="1.0.0" />
<info author="srl295" indicator="🙀" layout="qwerty" normalization="NFC" />
<names>

36
developer/src/kmc/test/helpers/index.ts
View file

@ -4,9 +4,13 @@
import * as path from 'path';
import * as fs from 'fs';
import { SectionCompiler } from '../../src/keyman/compiler/section-compiler';
import { Elem, GlobalSections, Section, Strs } from '../../src/keyman/kmx/kmx-plus';
import KMXPlusFile, { Elem, GlobalSections, Section, Strs } from '../../src/keyman/kmx/kmx-plus';
import LDMLKeyboardXMLSourceFileReader from '../../src/keyman/ldml-keyboard/ldml-keyboard-xml-reader';
import { CompilerEvent } from '../keyman/compiler/callbacks';
import { CompilerEvent } from '../../src/keyman/compiler/callbacks';
import Compiler from '../../src/keyman/compiler/compiler';
import { assert } from 'chai';
import KMXPlusMetadataCompiler from '../../src/keyman/compiler/metadata-compiler';
import CompilerOptions from '../../src/keyman/compiler/compiler-options';
/**
* Builds a path to the fixture with the given path components.
@ -50,3 +54,31 @@ export function loadSectionFixture(compilerClass: typeof SectionCompiler, filena
return compiler.compile(globalSections);
}
export function compileKeyboard(inputFilename: string, callbacks: CompilerCallbacks, options: CompilerOptions): KMXPlusFile {
const k = new Compiler(callbacks, options);
const source = k.load(inputFilename);
checkMessages(callbacks);
assert.isNotNull(source, 'k.load should not have returned null');
const valid = k.validate(source);
checkMessages(callbacks);
assert.isTrue(valid, 'k.validate should not have failed');
const kmx = k.compile(source);
checkMessages(callbacks);
assert.isNotNull(kmx, 'k.compile should not have returned null');
// In order for the KMX file to be loaded by non-KMXPlus components, it is helpful
// to duplicate some of the metadata
KMXPlusMetadataCompiler.addKmxMetadata(kmx.kmxplus, kmx.keyboard, options);
return kmx;
}
export function checkMessages(callbacks: CompilerCallbacks) {
if(callbacks.messages.length > 0) {
console.log(callbacks.messages);
}
assert.isEmpty(callbacks.messages);
}

49
developer/src/kmc/test/test-compiler-e2e.ts
View file

@ -1,40 +1,8 @@
import 'mocha';
import * as fs from 'fs';
import * as path from 'path';
import {assert} from 'chai';
import hextobin from '@keymanapp/hextobin';
import Compiler from '../src/keyman/compiler/compiler';
import KMXBuilder from '../src/keyman/kmx/kmx-builder';
import {CompilerCallbacks, makePathToFixture} from './helpers/index';
function checkMessages(callbacks: CompilerCallbacks) {
if(callbacks.messages.length > 0) {
console.log(callbacks.messages);
}
assert.isEmpty(callbacks.messages);
}
function compileKeyboard(inputFilename: string): Uint8Array {
const callbacks = new CompilerCallbacks();
const k = new Compiler(callbacks);
const source = k.load(inputFilename);
checkMessages(callbacks);
assert.isNotNull(source, 'k.load should not have returned null');
const valid = k.validate(source);
checkMessages(callbacks);
assert.isTrue(valid, 'k.validate should not have failed');
const kmx = k.compile(source);
checkMessages(callbacks);
assert.isNotNull(kmx, 'k.compile should not have returned null');
// Use the builder to generate the binary output file
const builder = new KMXBuilder(kmx, true);
const result = builder.compile();
checkMessages(callbacks);
return result;
}
import {checkMessages, compileKeyboard, CompilerCallbacks, makePathToFixture} from './helpers/index';
describe('compiler-tests', function() {
this.slow(500); // 0.5 sec -- json schema validation takes a while
@ -44,14 +12,21 @@ describe('compiler-tests', function() {
// It should match basic.kmx (built from basic.txt)
const inputFilename = makePathToFixture('basic.xml');
const outputFilename = makePathToFixture('basic.txt');
const binaryFilename = makePathToFixture('basic.txt');
// Compile:
const code = compileKeyboard(inputFilename);
// Compile the keyboard
const callbacks = new CompilerCallbacks();
const kmx = compileKeyboard(inputFilename, callbacks, {debug: true, addCompilerVersion: false});
assert.isNotNull(kmx);
// Use the builder to generate the binary output file
const builder = new KMXBuilder(kmx, true);
const code = builder.compile();
checkMessages(callbacks);
assert.isNotNull(code);
// Compare output
let expected = await hextobin(outputFilename, undefined, {silent:true});
let expected = await hextobin(binaryFilename, undefined, {silent:true});
assert.deepEqual<Uint8Array>(code, expected);
});
});

76
developer/src/kmc/test/test-metadata-compiler.ts Normal file
View file

@ -0,0 +1,76 @@
import 'mocha';
import {assert} from 'chai';
import { checkMessages, compileKeyboard, CompilerCallbacks, makePathToFixture } from './helpers';
import KMXFile from '../src/keyman/kmx/kmx';
const KEYMAN_VERSION = require("@keymanapp/keyman-version").KEYMAN_VERSION;
describe('kmx metadata compiler', function () {
this.slow(500); // 0.5 sec -- json schema validation takes a while
it('should compile metadata with debug and compiler version', function() {
debugger
const callbacks = new CompilerCallbacks();
const inputFilename = makePathToFixture('basic.xml');
// Compile the keyboard
const kmx = compileKeyboard(inputFilename, callbacks, {debug:true, addCompilerVersion:true});
checkMessages(callbacks);
assert.isNotNull(kmx);
// Order of stores is not significant in kmx spec, but kmxplus compiler will
// always store according to dwSystemID binary order for non-zero
// dwSystemID, then by dpName binary order, and finally by dpString binary
// order
// TSS_NAME = 7
// TSS_COMPILEDVERSION = 20
// TSS_KEYBOARDVERSION = 36
// TSS_TARGETS = 38
assert.equal(kmx.keyboard.stores[0].dpName, '&NAME');
assert.equal(kmx.keyboard.stores[0].dpString, 'TestKbd');
assert.equal(kmx.keyboard.stores[0].dwSystemID, KMXFile.TSS_NAME);
assert.equal(kmx.keyboard.stores[1].dpName, '');
assert.equal(kmx.keyboard.stores[1].dpString, KEYMAN_VERSION.VERSION_WITH_TAG);
assert.equal(kmx.keyboard.stores[1].dwSystemID, KMXFile.TSS_COMPILEDVERSION);
assert.equal(kmx.keyboard.stores[2].dpName, '&KEYBOARDVERSION');
assert.equal(kmx.keyboard.stores[2].dpString, '1.0.0');
assert.equal(kmx.keyboard.stores[2].dwSystemID, KMXFile.TSS_KEYBOARDVERSION);
assert.equal(kmx.keyboard.stores[3].dpName, '&TARGETS');
assert.equal(kmx.keyboard.stores[3].dpString, 'desktop');
assert.equal(kmx.keyboard.stores[3].dwSystemID, KMXFile.TSS_TARGETS);
});
it('should compile metadata with no compiler version', function() {
const callbacks = new CompilerCallbacks();
const inputFilename = makePathToFixture('basic.xml');
// Compile the keyboard
const kmx = compileKeyboard(inputFilename, callbacks, {debug:true, addCompilerVersion:false});
checkMessages(callbacks);
assert.isNotNull(kmx);
// TSS_NAME = 7
// -- TSS_COMPILEDVERSION = 20 -- not included with addCompilerVersion = false
// TSS_KEYBOARDVERSION = 36
// TSS_TARGETS = 38
assert.equal(kmx.keyboard.stores[0].dpName, '&NAME');
assert.equal(kmx.keyboard.stores[0].dpString, 'TestKbd');
assert.equal(kmx.keyboard.stores[0].dwSystemID, KMXFile.TSS_NAME);
assert.equal(kmx.keyboard.stores[1].dpName, '&KEYBOARDVERSION');
assert.equal(kmx.keyboard.stores[1].dpString, '1.0.0');
assert.equal(kmx.keyboard.stores[1].dwSystemID, KMXFile.TSS_KEYBOARDVERSION);
assert.equal(kmx.keyboard.stores[2].dpName, '&TARGETS');
assert.equal(kmx.keyboard.stores[2].dpString, 'desktop');
assert.equal(kmx.keyboard.stores[2].dwSystemID, KMXFile.TSS_TARGETS);
});
});

1
developer/src/kmc/test/tsconfig.json
View file

@ -16,6 +16,7 @@
"./helpers/index.ts"
],
"references": [
{ "path": "../../../../common/web/keyman-version" },
{ "path": "../" }
]
}