feat(developer): add schemas for .kvk and .kvks

Also minor tweaks around reading .xml files.
This commit is contained in:
Marc Durdin 2022-09-26 20:05:52 +10:00
parent c7740a39a2
commit 50db7a2c9b
12 changed files with 515 additions and 105 deletions

122
common/schemas/kvk/kvk.ksy Normal file
View file

@ -0,0 +1,122 @@
meta:
id: kvk
title: Keyman Visual Keyboard
file-extension: kvk
license: MIT
ks-version: 0.9
endian: le
bit-endian: le
doc: |
KVK is the binary file format for Keyman Visual Keyboard
files. KVKS is the equivalent XML source file format
doc-ref:
- https://github.com/keymanapp/keyman/
seq:
- id: header
type: header
- id: keys
type: keys
types:
header:
seq:
- id: identifier
contents: 'KVKF'
doc: Magic file identifier, always KVKF
- id: version
contents: [0, 6, 0, 0]
doc: Version number of KVK file format, always 0x00000600
- id: flag
type: header_flags
- id: associated_keyboard
type: string
- id: ansi_font
type: font
- id: unicode_font
type: font
header_flags:
seq:
- id: display_102
type: b1
doc: kvkh102, Keyboard should display 102nd key
- id: display_underlying
type: b1
doc: kvkhDisplayUnderlying, Keyboard should display underlying characters
- id: use_underlying
type: b1
doc: kvkhUseUnderlying,
- id: altgr
type: b1
doc: kvkhAltGr, Keyboard should treat left/right Ctrl and Alt separately
keys:
seq:
- id: count
type: u4
- id: key
type: key
repeat: expr
repeat-expr: count
key:
seq:
- id: flags
type: key_flags
- id: modifiers
type: key_modifiers
- id: vkey
type: u2
- id: text
type: string
- id: bitmap
type: u4
key_modifiers:
seq:
- id: shift
type: b1
- id: ctrl
type: b1
- id: alt
type: b1
- id: lctrl
type: b1
- id: rctrl
type: b1
- id: lalt
type: b1
- id: ralt
type: b1
- id: padding
type: b1
doc: reserved,
- id: zeropad
contents: [0]
key_flags:
seq:
- id: bitmap
type: b1
- id: unicode
type: b1
- id: padding
type: b6
font:
seq:
- id: name
type: string
- id: size
type: u4
- id: color
type: u4
string:
seq:
- id: len
type: u2
- id: str
type: str
size: len*2 - 2
encoding: utf-16
- id: zero_terminator
contents: [0,0]

View file

@ -0,0 +1,14 @@
# .kvks schema
This schema validates .kvks files, according to the reference implementation
from VisualKeyboardLoaderXML.pas.
## Notes on conversion from xsd to json-schema
Converted using xsd2json. Following structural changes:
* kvk-version base type from km-version to string, copy km-version pattern in
* remove xs:all bracketing
* remove format:double from fontsize, change type to string
* encoding property changed type to array
* kvk-key added _ property for base text value

View file

@ -0,0 +1,155 @@
{
"title": "kvks.xsd",
"$schema": "http://json-schema.org/schema#",
"type": "object",
"properties": {
"visualkeyboard": {
"properties": {
"header": {
"$ref": "#/definitions/kvk-header"
},
"encoding": {
"type": "array",
"items": {
"$ref": "#/definitions/kvk-encoding"
}
}
},
"required": [
"header"
],
"additionalProperties": false,
"type": "object"
}
},
"required": [
"visualkeyboard"
],
"additionalProperties": false,
"definitions": {
"kvk-header": {
"type": "object",
"properties": {
"version": {
"$ref": "#/definitions/kvk-version"
},
"kbdname": {
"type": "string"
},
"flags": {
"$ref": "#/definitions/kvk-header-flags"
},
"layout": {
"type": "string"
}
},
"required": [
"version"
],
"additionalProperties": false
},
"kvk-header-flags": {
"type": "object",
"properties": {
"key102": {
"$ref": "#/definitions/km-empty"
},
"displayunderlying": {
"$ref": "#/definitions/km-empty"
},
"usealtgr": {
"$ref": "#/definitions/km-empty"
},
"useunderlying": {
"$ref": "#/definitions/km-empty"
}
},
"additionalProperties": false
},
"kvk-encoding": {
"type": "object",
"properties": {
"layer": {
"type": "array",
"items": {
"$ref": "#/definitions/kvk-layer"
}
},
"name": {
"$ref": "#/definitions/kvk-encoding-name"
},
"fontname": {
"type": "string"
},
"fontsize": {
"type": "string"
}
},
"required": [
"name"
],
"additionalProperties": false
},
"kvk-layer": {
"type": "object",
"properties": {
"key": {
"type": "array",
"items": {
"$ref": "#/definitions/kvk-key"
}
},
"shift": {
"$ref": "#/definitions/kvk-layer-shift"
}
},
"required": [
"shift"
],
"additionalProperties": false
},
"kvk-key": {
"type": "object",
"properties": {
"bitmap": {
"type": "string"
},
"vkey": {
"type": "string"
},
"_": {
"type": "string"
}
},
"required": [
"vkey"
],
"additionalProperties": false
},
"km-empty": {
"type": "string"
},
"kvk-encoding-name": {
"type": "string",
"enum": [
"ansi",
"unicode"
]
},
"kvk-layer-shift": {
"type": "string",
"pattern": "S?(C|LC|RC)?(A|LA|RA)?"
},
"kvk-version": {
"type": "string",
"pattern": "(\\d+\\.)+(\\d+)",
"enum": [
"10.0"
]
},
"km-version": {
"type": "string",
"pattern": "(\\d+\\.)+(\\d+)"
}
}
}

View file

@ -0,0 +1,92 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
.kvks file schema, version 10.0
Copyright (C) SIL International
Supports KVKS files for Keyman Developer 10+
Expects version 10.0
-->
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
<xs:element name="visualkeyboard">
<xs:complexType>
<xs:all>
<xs:element name="header" type="kvk-header" />
<xs:element minOccurs="0" name="encoding" type="kvk-encoding" />
</xs:all>
</xs:complexType>
</xs:element>
<xs:complexType name="kvk-header">
<xs:all>
<xs:element name="version" type="kvk-version" />
<xs:element minOccurs="0" name="kbdname" type="xs:string" />
<xs:element minOccurs="0" name="flags" type="kvk-header-flags" />
<xs:element minOccurs="0" name="layout" type="xs:string" />
</xs:all>
</xs:complexType>
<xs:complexType name="kvk-header-flags">
<xs:all>
<xs:element minOccurs="0" name="key102" type="km-empty" />
<xs:element minOccurs="0" name="displayunderlying" type="km-empty" />
<xs:element minOccurs="0" name="usealtgr" type="km-empty" />
<xs:element minOccurs="0" name="useunderlying" type="km-empty" />
</xs:all>
</xs:complexType>
<xs:complexType name="kvk-encoding">
<xs:sequence>
<xs:element minOccurs="0" maxOccurs="unbounded" name="layer" type="kvk-layer" />
</xs:sequence>
<xs:attribute name="name" type="kvk-encoding-name" use="required" />
<xs:attribute name="fontname" type="xs:string" />
<xs:attribute name="fontsize" type="xs:double" />
</xs:complexType>
<xs:simpleType name="kvk-encoding-name">
<xs:restriction base="xs:string">
<xs:enumeration value="ansi" />
<xs:enumeration value="unicode" />
</xs:restriction>
</xs:simpleType>
<xs:complexType name="kvk-layer">
<xs:sequence><xs:element minOccurs="0" maxOccurs="unbounded" name="key" type="kvk-key" /></xs:sequence>
<xs:attribute name="shift" use="required" type="kvk-layer-shift" />
</xs:complexType>
<xs:simpleType name="kvk-layer-shift">
<xs:restriction base="xs:string">
<xs:pattern value="S?(C|LC|RC)?(A|LA|RA)?" />
</xs:restriction>
</xs:simpleType>
<xs:complexType name="kvk-key" mixed="true">
<xs:sequence>
<xs:element minOccurs="0" name="bitmap" type="xs:string" />
</xs:sequence>
<xs:attribute name="vkey" type="xs:string" use="required" />
</xs:complexType>
<xs:simpleType name="kvk-version">
<xs:restriction base="km-version">
<xs:enumeration value="10.0" />
</xs:restriction>
</xs:simpleType>
<!-- Some base types -->
<xs:complexType name="km-empty">
<xs:sequence/>
</xs:complexType>
<xs:simpleType name="km-version">
<xs:restriction base="xs:string">
<xs:pattern value="(\d+\.)+(\d+)"/>
</xs:restriction>
</xs:simpleType>
</xs:schema>

View file

@ -36,6 +36,7 @@ else
# We need the schema file at runtime and bundled, so always copy it for all actions except `clean`
mkdir -p "$THIS_SCRIPT_PATH/build/src/"
cp "$KEYMAN_ROOT/resources/standards-data/ldml-keyboards/techpreview/ldml-keyboard.schema.json" "$THIS_SCRIPT_PATH/build/src/"
cp "$KEYMAN_ROOT/common/schemas/kvks/kvks.schema.json" "$THIS_SCRIPT_PATH/build/src/"
fi
#-------------------------------------------------------------------------------------------------------------------

View file

@ -13,8 +13,7 @@
},
"scripts": {
"build": "tsc -b",
"test": "cd test && tsc -b && cd .. && mocha",
"coverage": "cd test && tsc -b && cd .. && c8 mocha",
"test": "cd test && tsc -b && cd .. && c8 mocha",
"prepublishOnly": "npm run build"
},
"author": "Marc Durdin <marc@keyman.com> (https://github.com/mcdurdin)",

View file

@ -6,5 +6,6 @@ export interface CompilerEvent {
export default interface CompilerCallbacks {
loadFile(baseFilename: string, filename: string): Buffer;
loadLdmlKeyboardSchema(): Buffer;
loadKvksJsonSchema(): Buffer;
reportMessage(event: CompilerEvent): void;
};

View file

@ -20,7 +20,7 @@ export class CompilerMessages {
static Error_InvalidNormalization = (o:{form: string}) => m(this.ERROR_InvalidNormalization, `Invalid normalization form '${o.form}`);
static ERROR_InvalidNormalization = SevError | 0x0001;
static Error_InvalidLocale = (o:{tag: string}) => m(this.ERROR_InvalidLocale, `Invalid BCP 47 locale form '${o.tag}`);
static Error_InvalidLocale = (o:{tag: string}) => m(this.ERROR_InvalidLocale, `Invalid BCP 47 locale form '${o.tag}'`);
static ERROR_InvalidLocale = SevError | 0x0002;
static Error_HardwareLayerHasTooManyRows = () => m(this.ERROR_HardwareLayerHasTooManyRows, `'hardware' layer has too many rows`);

View file

@ -2,52 +2,47 @@ import KVKFile, { BUILDER_KVK_FILE, BUILDER_KVK_HEADER_IDENTIFIER, BUILDER_KVK_H
import { VisualKeyboard } from "./visual-keyboard.js";
export default class KvkFileWriter {
private source: VisualKeyboard;
constructor(source: VisualKeyboard) {
this.source = source;
}
/**
* Fills a kvk string from a source string. Returns byte size of s as UTF-16
* zero terminated string for KVK format Note that the format includes both a
* length word and zero termination.
* @param str
* @param value
* @returns number
* Writes the visual keyboard to a binary .kvk format byte array.
* @param source VisualKeyboard
* @returns Uint8Array, the .kvk file
*/
private setString(str: BUILDER_KVK_STRING, value: string): void {
str.len = value.length + 1;
str.str = value;
write(source: VisualKeyboard): Uint8Array {
const binary = this.build(source);
const kvk = new KVKFile();
const file: Uint8Array = new Uint8Array(kvk.KVK_FILE.size(binary));
const data = kvk.KVK_FILE.toBuffer(binary);
file.set(data, 0);
return file;
}
private build() {
let binary: BUILDER_KVK_FILE = {
private build(source: VisualKeyboard) {
const binary: BUILDER_KVK_FILE = {
header: {
identifier: BUILDER_KVK_HEADER_IDENTIFIER,
version: BUILDER_KVK_HEADER_VERSION,
associatedKeyboard: {len:0,str:''},
flags: this.source.header.flags,
flags: source.header.flags,
ansiFont:{
color: this.source.header.ansiFont.color,
size: this.source.header.ansiFont.size,
color: source.header.ansiFont.color,
size: source.header.ansiFont.size,
name: {len:0,str:''}
},
unicodeFont:{
color: this.source.header.unicodeFont.color,
size: this.source.header.unicodeFont.size,
color: source.header.unicodeFont.color,
size: source.header.unicodeFont.size,
name: {len:0,str:''}
},
},
keyCount: this.source.keys.length,
keyCount: source.keys.length,
keys:[]
};
this.setString(binary.header.associatedKeyboard, this.source.header.associatedKeyboard);
this.setString(binary.header.ansiFont.name, this.source.header.ansiFont.name);
this.setString(binary.header.unicodeFont.name, this.source.header.unicodeFont.name);
this.setString(binary.header.associatedKeyboard, source.header.associatedKeyboard);
this.setString(binary.header.ansiFont.name, source.header.ansiFont.name);
this.setString(binary.header.unicodeFont.name, source.header.unicodeFont.name);
for(let sourceKey of this.source.keys) {
for(let sourceKey of source.keys) {
const binaryKey: BUILDER_KVK_KEY = {
flags: sourceKey.flags,
vkey: sourceKey.vkey,
@ -62,12 +57,16 @@ export default class KvkFileWriter {
return binary;
}
compile(): Uint8Array {
const binary = this.build();
const kvk = new KVKFile();
const file: Uint8Array = new Uint8Array(kvk.KVK_FILE.size(binary));
const data = kvk.KVK_FILE.toBuffer(binary);
file.set(data, 0);
return file;
/**
* Fills a kvk string from a source string. Note that the format includes both
* a length word and zero termination.
*
* @param str
* @param value
* @returns number
*/
private setString(str: BUILDER_KVK_STRING, value: string): void {
str.len = value.length + 1;
str.str = value;
}
};

View file

@ -1,92 +1,85 @@
import * as xml2js from 'xml2js';
import KVKSourceFile from './kvks-file.js';
import CompilerCallbacks from '../compiler/callbacks.js';
// import Ajv from 'ajv';
import Ajv from 'ajv';
import { CompilerMessages } from '../compiler/messages.js';
import { boxXmlArray } from '../util/util.js';
import { VisualKeyboard, VisualKeyboardHeaderFlags, VisualKeyboardKey, VisualKeyboardKeyFlags, VisualKeyboardLegalShiftStates, VisualKeyboardShiftState } from './visual-keyboard.js';
import { USVirtualKeyCodes } from '../ldml-keyboard/virtual-key-constants.js';
import { BUILDER_KVK_HEADER_VERSION } from './kvk-file.js';
export default class LDMLKeyboardXMLSourceFileReader {
export default class KVKSFileReader {
private readonly callbacks: CompilerCallbacks;
constructor (callbacks: CompilerCallbacks) {
this.callbacks = callbacks;
}
/**
* xml2js will not place single-entry objects into arrays.
* Easiest way to fix this is to box them ourselves as needed
* @param source KVKSourceFile
*/
private boxArrays(source: KVKSourceFile) {
boxXmlArray(source.visualkeyboard, 'encoding');
for(let encoding of source.visualkeyboard.encoding) {
boxXmlArray(encoding, 'layer');
for(let layer of encoding.layer) {
boxXmlArray(layer, 'key');
}
}
return source;
}
public validate(source: KVKSourceFile): KVKSourceFile {
/* LDML-TODO:
const schema = JSON.parse(this.callbacks.loadKvksSchema().toString('utf8'));
const ajv = new Ajv();
if(!ajv.validate(schema, source)) {
this.callbacks.reportMessage(CompilerMessages.Error_InvalidFile({errorText: ajv.errorsText()}));
return null;
}*/
return source;
}
public loadFile(filename: string) {
const buf = this.callbacks.loadFile(filename, filename);
return this.load(buf);
}
public load(file: Uint8Array): KVKSourceFile {
let source = (() => {
let a: KVKSourceFile;
let parser = new xml2js.Parser({
explicitArray: false,
mergeAttrs: true,
includeWhiteChars: true,
normalize: false,
emptyTag: {} as any
// Why "as any"? xml2js is broken:
// https://github.com/Leonidas-from-XIV/node-xml2js/issues/648 means
// that an old version of `emptyTag` is used which doesn't support
// functions, but DefinitelyTyped is requiring use of function or a
// string. See also notes at
// https://github.com/DefinitelyTyped/DefinitelyTyped/pull/59259#issuecomment-1254405470
// An alternative fix would be to pull xml2js directly from github
// rather than using the version tagged on npmjs.com.
});
parser.parseString(file, (e: unknown, r: unknown) => { a = r as KVKSourceFile });
return a;
})();
return this.validate(this.boxArrays(source));
}
public loadVisualKeyboard(file: Uint8Array): VisualKeyboard {
let source = this.load(file);
public read(file: Uint8Array): VisualKeyboard {
let source = this.internalRead(file);
return this.transform(source);
}
public kvksShiftToKvkShift(shift: string): VisualKeyboardShiftState {
shift = shift.toUpperCase();
public internalRead(file: Uint8Array): KVKSourceFile {
let source: KVKSourceFile;
// TODO-LDML(lowpri): make a map of this?
for(let state of VisualKeyboardLegalShiftStates) {
if(state.name == shift) {
return state.shift;
const parser = new xml2js.Parser({
explicitArray: false,
mergeAttrs: true,
includeWhiteChars: true,
normalize: false,
emptyTag: {} as any
// Why "as any"? xml2js is broken:
// https://github.com/Leonidas-from-XIV/node-xml2js/issues/648 means
// that an old version of `emptyTag` is used which doesn't support
// functions, but DefinitelyTyped is requiring use of function or a
// string. See also notes at
// https://github.com/DefinitelyTyped/DefinitelyTyped/pull/59259#issuecomment-1254405470
// An alternative fix would be to pull xml2js directly from github
// rather than using the version tagged on npmjs.com.
});
parser.parseString(file, (e: unknown, r: unknown) => { source = r as KVKSourceFile });
source = this.boxArrays(source);
this.cleanupUnderscore('visualkeyboard', source.visualkeyboard);
return this.validate(source);
}
/**
* The only element that allows spaces is <key>. Remove
* all other empty whitespace-only values.
* @param root
* @param source
*/
private cleanupUnderscore(root: string, source: any) {
if(root != 'key') {
if(source?.['_']?.trim() === '') {
delete source['_'];
}
}
return 0;
for(let key of Object.keys(source)) {
if(Array.isArray(source[key])) {
for(let item of source[key]) {
if(typeof(item) === 'object') {
this.cleanupUnderscore(key, item);
}
}
} else if(typeof source[key] === 'object') {
this.cleanupUnderscore(key, source[key]);
}
}
}
public validate(source: KVKSourceFile): KVKSourceFile {
const schema = JSON.parse(this.callbacks.loadKvksJsonSchema().toString('utf8'));
const ajv = new Ajv();
if(!ajv.validate(schema, source)) {
console.dir(source, {depth:8});
this.callbacks.reportMessage(CompilerMessages.Error_InvalidFile({errorText: ajv.errorsText()}));
return null;
}
return source;
}
public transform(source: KVKSourceFile): VisualKeyboard {
@ -142,4 +135,33 @@ export default class LDMLKeyboardXMLSourceFileReader {
return result;
}
/**
* xml2js will not place single-entry objects into arrays.
* Easiest way to fix this is to box them ourselves as needed
* @param source KVKSourceFile
*/
private boxArrays(source: KVKSourceFile) {
boxXmlArray(source.visualkeyboard, 'encoding');
for(let encoding of source.visualkeyboard.encoding) {
boxXmlArray(encoding, 'layer');
for(let layer of encoding.layer) {
boxXmlArray(layer, 'key');
}
}
return source;
}
public kvksShiftToKvkShift(shift: string): VisualKeyboardShiftState {
shift = shift.toUpperCase();
// TODO-LDML(lowpri): make a map of this?
for(let state of VisualKeyboardLegalShiftStates) {
if(state.name == shift) {
return state.shift;
}
}
return 0;
}
}

View file

@ -65,6 +65,7 @@ export default class LDMLKeyboardXMLSourceFileReader {
let parser = new xml2js.Parser({
explicitArray: false,
mergeAttrs: true,
includeWhiteChars: false,
emptyTag: {} as any
// Why "as any"? xml2js is broken:
// https://github.com/Leonidas-from-XIV/node-xml2js/issues/648 means

View file

@ -49,6 +49,10 @@ class CompilerCallbacks {
let schemaPath = new URL('ldml-keyboard.schema.json', import.meta.url);
return fs.readFileSync(schemaPath);
}
loadKvksJsonSchema(): Buffer {
let schemaPath = new URL('kvks.schema.json', import.meta.url);
return fs.readFileSync(schemaPath);
}
}
function compileKeyboard(inputFilename: string, options: kmc.CompilerOptions): Uint8Array {