diff --git a/common/schemas/kvk/kvk.ksy b/common/schemas/kvk/kvk.ksy new file mode 100644 index 0000000000..11d8f2faa7 --- /dev/null +++ b/common/schemas/kvk/kvk.ksy @@ -0,0 +1,122 @@ +meta: + id: kvk + title: Keyman Visual Keyboard + file-extension: kvk + license: MIT + ks-version: 0.9 + endian: le + bit-endian: le +doc: | + KVK is the binary file format for Keyman Visual Keyboard + files. KVKS is the equivalent XML source file format +doc-ref: + - https://github.com/keymanapp/keyman/ +seq: + - id: header + type: header + - id: keys + type: keys +types: + header: + seq: + - id: identifier + contents: 'KVKF' + doc: Magic file identifier, always KVKF + - id: version + contents: [0, 6, 0, 0] + doc: Version number of KVK file format, always 0x00000600 + - id: flag + type: header_flags + - id: associated_keyboard + type: string + - id: ansi_font + type: font + - id: unicode_font + type: font + + header_flags: + seq: + - id: display_102 + type: b1 + doc: kvkh102, Keyboard should display 102nd key + - id: display_underlying + type: b1 + doc: kvkhDisplayUnderlying, Keyboard should display underlying characters + - id: use_underlying + type: b1 + doc: kvkhUseUnderlying, + - id: altgr + type: b1 + doc: kvkhAltGr, Keyboard should treat left/right Ctrl and Alt separately + + keys: + seq: + - id: count + type: u4 + - id: key + type: key + repeat: expr + repeat-expr: count + key: + seq: + - id: flags + type: key_flags + - id: modifiers + type: key_modifiers + - id: vkey + type: u2 + - id: text + type: string + - id: bitmap + type: u4 + + key_modifiers: + seq: + - id: shift + type: b1 + - id: ctrl + type: b1 + - id: alt + type: b1 + - id: lctrl + type: b1 + - id: rctrl + type: b1 + - id: lalt + type: b1 + - id: ralt + type: b1 + - id: padding + type: b1 + doc: reserved, + - id: zeropad + contents: [0] + + key_flags: + seq: + - id: bitmap + type: b1 + - id: unicode + type: b1 + - id: padding + type: b6 + + font: + seq: + - id: name + type: string + - id: size + type: u4 + - id: color + type: u4 + + string: + seq: + - id: len + type: u2 + - id: str + type: str + size: len*2 - 2 + encoding: utf-16 + - id: zero_terminator + contents: [0,0] diff --git a/common/schemas/kvks/README.md b/common/schemas/kvks/README.md new file mode 100644 index 0000000000..fb4f2a8a68 --- /dev/null +++ b/common/schemas/kvks/README.md @@ -0,0 +1,14 @@ +# .kvks schema + +This schema validates .kvks files, according to the reference implementation +from VisualKeyboardLoaderXML.pas. + +## Notes on conversion from xsd to json-schema + +Converted using xsd2json. Following structural changes: + +* kvk-version base type from km-version to string, copy km-version pattern in +* remove xs:all bracketing +* remove format:double from fontsize, change type to string +* encoding property changed type to array +* kvk-key added _ property for base text value \ No newline at end of file diff --git a/common/schemas/kvks/kvks.schema.json b/common/schemas/kvks/kvks.schema.json new file mode 100644 index 0000000000..fd58f56e38 --- /dev/null +++ b/common/schemas/kvks/kvks.schema.json @@ -0,0 +1,155 @@ +{ + "title": "kvks.xsd", + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "visualkeyboard": { + "properties": { + "header": { + "$ref": "#/definitions/kvk-header" + }, + "encoding": { + "type": "array", + "items": { + "$ref": "#/definitions/kvk-encoding" + } + } + }, + "required": [ + "header" + ], + "additionalProperties": false, + "type": "object" + } + }, + "required": [ + "visualkeyboard" + ], + "additionalProperties": false, + "definitions": { + "kvk-header": { + "type": "object", + "properties": { + "version": { + "$ref": "#/definitions/kvk-version" + }, + "kbdname": { + "type": "string" + }, + "flags": { + "$ref": "#/definitions/kvk-header-flags" + }, + "layout": { + "type": "string" + } + }, + "required": [ + "version" + ], + "additionalProperties": false + }, + "kvk-header-flags": { + "type": "object", + "properties": { + "key102": { + "$ref": "#/definitions/km-empty" + }, + "displayunderlying": { + "$ref": "#/definitions/km-empty" + }, + "usealtgr": { + "$ref": "#/definitions/km-empty" + }, + "useunderlying": { + "$ref": "#/definitions/km-empty" + } + }, + "additionalProperties": false + }, + "kvk-encoding": { + "type": "object", + "properties": { + "layer": { + "type": "array", + "items": { + "$ref": "#/definitions/kvk-layer" + } + }, + "name": { + "$ref": "#/definitions/kvk-encoding-name" + }, + "fontname": { + "type": "string" + }, + "fontsize": { + "type": "string" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + "kvk-layer": { + "type": "object", + "properties": { + "key": { + "type": "array", + "items": { + "$ref": "#/definitions/kvk-key" + } + }, + "shift": { + "$ref": "#/definitions/kvk-layer-shift" + } + }, + "required": [ + "shift" + ], + "additionalProperties": false + }, + "kvk-key": { + "type": "object", + "properties": { + "bitmap": { + "type": "string" + }, + "vkey": { + "type": "string" + }, + "_": { + "type": "string" + } + }, + "required": [ + "vkey" + ], + "additionalProperties": false + }, + "km-empty": { + "type": "string" + }, + "kvk-encoding-name": { + "type": "string", + "enum": [ + "ansi", + "unicode" + ] + }, + "kvk-layer-shift": { + "type": "string", + "pattern": "S?(C|LC|RC)?(A|LA|RA)?" + }, + "kvk-version": { + "type": "string", + "pattern": "(\\d+\\.)+(\\d+)", + "enum": [ + "10.0" + ] + }, + "km-version": { + "type": "string", + "pattern": "(\\d+\\.)+(\\d+)" + } + } +} \ No newline at end of file diff --git a/common/schemas/kvks/kvks.xsd b/common/schemas/kvks/kvks.xsd new file mode 100644 index 0000000000..17056a5a57 --- /dev/null +++ b/common/schemas/kvks/kvks.xsd @@ -0,0 +1,92 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/developer/src/kmc-keyboard/build.sh b/developer/src/kmc-keyboard/build.sh index 27aeccf27e..938e0e1eca 100755 --- a/developer/src/kmc-keyboard/build.sh +++ b/developer/src/kmc-keyboard/build.sh @@ -36,6 +36,7 @@ else # We need the schema file at runtime and bundled, so always copy it for all actions except `clean` mkdir -p "$THIS_SCRIPT_PATH/build/src/" cp "$KEYMAN_ROOT/resources/standards-data/ldml-keyboards/techpreview/ldml-keyboard.schema.json" "$THIS_SCRIPT_PATH/build/src/" + cp "$KEYMAN_ROOT/common/schemas/kvks/kvks.schema.json" "$THIS_SCRIPT_PATH/build/src/" fi #------------------------------------------------------------------------------------------------------------------- diff --git a/developer/src/kmc-keyboard/package.json b/developer/src/kmc-keyboard/package.json index d5ea89d5bd..864f6967c8 100644 --- a/developer/src/kmc-keyboard/package.json +++ b/developer/src/kmc-keyboard/package.json @@ -13,8 +13,7 @@ }, "scripts": { "build": "tsc -b", - "test": "cd test && tsc -b && cd .. && mocha", - "coverage": "cd test && tsc -b && cd .. && c8 mocha", + "test": "cd test && tsc -b && cd .. && c8 mocha", "prepublishOnly": "npm run build" }, "author": "Marc Durdin (https://github.com/mcdurdin)", diff --git a/developer/src/kmc-keyboard/src/compiler/callbacks.ts b/developer/src/kmc-keyboard/src/compiler/callbacks.ts index 8d32f5e8c4..f6cd673b56 100644 --- a/developer/src/kmc-keyboard/src/compiler/callbacks.ts +++ b/developer/src/kmc-keyboard/src/compiler/callbacks.ts @@ -6,5 +6,6 @@ export interface CompilerEvent { export default interface CompilerCallbacks { loadFile(baseFilename: string, filename: string): Buffer; loadLdmlKeyboardSchema(): Buffer; + loadKvksJsonSchema(): Buffer; reportMessage(event: CompilerEvent): void; }; diff --git a/developer/src/kmc-keyboard/src/compiler/messages.ts b/developer/src/kmc-keyboard/src/compiler/messages.ts index 24c4370767..409b37acab 100644 --- a/developer/src/kmc-keyboard/src/compiler/messages.ts +++ b/developer/src/kmc-keyboard/src/compiler/messages.ts @@ -20,7 +20,7 @@ export class CompilerMessages { static Error_InvalidNormalization = (o:{form: string}) => m(this.ERROR_InvalidNormalization, `Invalid normalization form '${o.form}`); static ERROR_InvalidNormalization = SevError | 0x0001; - static Error_InvalidLocale = (o:{tag: string}) => m(this.ERROR_InvalidLocale, `Invalid BCP 47 locale form '${o.tag}`); + static Error_InvalidLocale = (o:{tag: string}) => m(this.ERROR_InvalidLocale, `Invalid BCP 47 locale form '${o.tag}'`); static ERROR_InvalidLocale = SevError | 0x0002; static Error_HardwareLayerHasTooManyRows = () => m(this.ERROR_HardwareLayerHasTooManyRows, `'hardware' layer has too many rows`); diff --git a/developer/src/kmc-keyboard/src/kvk/kvk-file-writer.ts b/developer/src/kmc-keyboard/src/kvk/kvk-file-writer.ts index 49c0655d4b..78a484284b 100644 --- a/developer/src/kmc-keyboard/src/kvk/kvk-file-writer.ts +++ b/developer/src/kmc-keyboard/src/kvk/kvk-file-writer.ts @@ -2,52 +2,47 @@ import KVKFile, { BUILDER_KVK_FILE, BUILDER_KVK_HEADER_IDENTIFIER, BUILDER_KVK_H import { VisualKeyboard } from "./visual-keyboard.js"; export default class KvkFileWriter { - private source: VisualKeyboard; - - constructor(source: VisualKeyboard) { - this.source = source; - } - /** - * Fills a kvk string from a source string. Returns byte size of s as UTF-16 - * zero terminated string for KVK format Note that the format includes both a - * length word and zero termination. - * @param str - * @param value - * @returns number + * Writes the visual keyboard to a binary .kvk format byte array. + * @param source VisualKeyboard + * @returns Uint8Array, the .kvk file */ - private setString(str: BUILDER_KVK_STRING, value: string): void { - str.len = value.length + 1; - str.str = value; + write(source: VisualKeyboard): Uint8Array { + const binary = this.build(source); + const kvk = new KVKFile(); + const file: Uint8Array = new Uint8Array(kvk.KVK_FILE.size(binary)); + const data = kvk.KVK_FILE.toBuffer(binary); + file.set(data, 0); + return file; } - private build() { - let binary: BUILDER_KVK_FILE = { + private build(source: VisualKeyboard) { + const binary: BUILDER_KVK_FILE = { header: { identifier: BUILDER_KVK_HEADER_IDENTIFIER, version: BUILDER_KVK_HEADER_VERSION, associatedKeyboard: {len:0,str:''}, - flags: this.source.header.flags, + flags: source.header.flags, ansiFont:{ - color: this.source.header.ansiFont.color, - size: this.source.header.ansiFont.size, + color: source.header.ansiFont.color, + size: source.header.ansiFont.size, name: {len:0,str:''} }, unicodeFont:{ - color: this.source.header.unicodeFont.color, - size: this.source.header.unicodeFont.size, + color: source.header.unicodeFont.color, + size: source.header.unicodeFont.size, name: {len:0,str:''} }, }, - keyCount: this.source.keys.length, + keyCount: source.keys.length, keys:[] }; - this.setString(binary.header.associatedKeyboard, this.source.header.associatedKeyboard); - this.setString(binary.header.ansiFont.name, this.source.header.ansiFont.name); - this.setString(binary.header.unicodeFont.name, this.source.header.unicodeFont.name); + this.setString(binary.header.associatedKeyboard, source.header.associatedKeyboard); + this.setString(binary.header.ansiFont.name, source.header.ansiFont.name); + this.setString(binary.header.unicodeFont.name, source.header.unicodeFont.name); - for(let sourceKey of this.source.keys) { + for(let sourceKey of source.keys) { const binaryKey: BUILDER_KVK_KEY = { flags: sourceKey.flags, vkey: sourceKey.vkey, @@ -62,12 +57,16 @@ export default class KvkFileWriter { return binary; } - compile(): Uint8Array { - const binary = this.build(); - const kvk = new KVKFile(); - const file: Uint8Array = new Uint8Array(kvk.KVK_FILE.size(binary)); - const data = kvk.KVK_FILE.toBuffer(binary); - file.set(data, 0); - return file; + /** + * Fills a kvk string from a source string. Note that the format includes both + * a length word and zero termination. + * + * @param str + * @param value + * @returns number + */ + private setString(str: BUILDER_KVK_STRING, value: string): void { + str.len = value.length + 1; + str.str = value; } }; \ No newline at end of file diff --git a/developer/src/kmc-keyboard/src/kvk/kvks-file-reader.ts b/developer/src/kmc-keyboard/src/kvk/kvks-file-reader.ts index 09bcf6b262..3490c709b5 100644 --- a/developer/src/kmc-keyboard/src/kvk/kvks-file-reader.ts +++ b/developer/src/kmc-keyboard/src/kvk/kvks-file-reader.ts @@ -1,92 +1,85 @@ import * as xml2js from 'xml2js'; import KVKSourceFile from './kvks-file.js'; import CompilerCallbacks from '../compiler/callbacks.js'; -// import Ajv from 'ajv'; +import Ajv from 'ajv'; import { CompilerMessages } from '../compiler/messages.js'; import { boxXmlArray } from '../util/util.js'; import { VisualKeyboard, VisualKeyboardHeaderFlags, VisualKeyboardKey, VisualKeyboardKeyFlags, VisualKeyboardLegalShiftStates, VisualKeyboardShiftState } from './visual-keyboard.js'; import { USVirtualKeyCodes } from '../ldml-keyboard/virtual-key-constants.js'; import { BUILDER_KVK_HEADER_VERSION } from './kvk-file.js'; -export default class LDMLKeyboardXMLSourceFileReader { +export default class KVKSFileReader { private readonly callbacks: CompilerCallbacks; constructor (callbacks: CompilerCallbacks) { this.callbacks = callbacks; } - /** - * xml2js will not place single-entry objects into arrays. - * Easiest way to fix this is to box them ourselves as needed - * @param source KVKSourceFile - */ - private boxArrays(source: KVKSourceFile) { - boxXmlArray(source.visualkeyboard, 'encoding'); - for(let encoding of source.visualkeyboard.encoding) { - boxXmlArray(encoding, 'layer'); - for(let layer of encoding.layer) { - boxXmlArray(layer, 'key'); - } - } - return source; - } - - public validate(source: KVKSourceFile): KVKSourceFile { - /* LDML-TODO: - const schema = JSON.parse(this.callbacks.loadKvksSchema().toString('utf8')); - const ajv = new Ajv(); - if(!ajv.validate(schema, source)) { - this.callbacks.reportMessage(CompilerMessages.Error_InvalidFile({errorText: ajv.errorsText()})); - return null; - }*/ - return source; - } - - public loadFile(filename: string) { - const buf = this.callbacks.loadFile(filename, filename); - return this.load(buf); - } - - public load(file: Uint8Array): KVKSourceFile { - let source = (() => { - let a: KVKSourceFile; - let parser = new xml2js.Parser({ - explicitArray: false, - mergeAttrs: true, - includeWhiteChars: true, - normalize: false, - emptyTag: {} as any - // Why "as any"? xml2js is broken: - // https://github.com/Leonidas-from-XIV/node-xml2js/issues/648 means - // that an old version of `emptyTag` is used which doesn't support - // functions, but DefinitelyTyped is requiring use of function or a - // string. See also notes at - // https://github.com/DefinitelyTyped/DefinitelyTyped/pull/59259#issuecomment-1254405470 - // An alternative fix would be to pull xml2js directly from github - // rather than using the version tagged on npmjs.com. - }); - parser.parseString(file, (e: unknown, r: unknown) => { a = r as KVKSourceFile }); - return a; - })(); - - return this.validate(this.boxArrays(source)); - } - - public loadVisualKeyboard(file: Uint8Array): VisualKeyboard { - let source = this.load(file); + public read(file: Uint8Array): VisualKeyboard { + let source = this.internalRead(file); return this.transform(source); } - public kvksShiftToKvkShift(shift: string): VisualKeyboardShiftState { - shift = shift.toUpperCase(); + public internalRead(file: Uint8Array): KVKSourceFile { + let source: KVKSourceFile; - // TODO-LDML(lowpri): make a map of this? - for(let state of VisualKeyboardLegalShiftStates) { - if(state.name == shift) { - return state.shift; + const parser = new xml2js.Parser({ + explicitArray: false, + mergeAttrs: true, + includeWhiteChars: true, + normalize: false, + emptyTag: {} as any + // Why "as any"? xml2js is broken: + // https://github.com/Leonidas-from-XIV/node-xml2js/issues/648 means + // that an old version of `emptyTag` is used which doesn't support + // functions, but DefinitelyTyped is requiring use of function or a + // string. See also notes at + // https://github.com/DefinitelyTyped/DefinitelyTyped/pull/59259#issuecomment-1254405470 + // An alternative fix would be to pull xml2js directly from github + // rather than using the version tagged on npmjs.com. + }); + + parser.parseString(file, (e: unknown, r: unknown) => { source = r as KVKSourceFile }); + source = this.boxArrays(source); + this.cleanupUnderscore('visualkeyboard', source.visualkeyboard); + return this.validate(source); + } + + /** + * The only element that allows spaces is . Remove + * all other empty whitespace-only values. + * @param root + * @param source + */ + private cleanupUnderscore(root: string, source: any) { + if(root != 'key') { + if(source?.['_']?.trim() === '') { + delete source['_']; } } - return 0; + + for(let key of Object.keys(source)) { + if(Array.isArray(source[key])) { + for(let item of source[key]) { + if(typeof(item) === 'object') { + this.cleanupUnderscore(key, item); + } + } + } else if(typeof source[key] === 'object') { + this.cleanupUnderscore(key, source[key]); + } + } + } + + public validate(source: KVKSourceFile): KVKSourceFile { + const schema = JSON.parse(this.callbacks.loadKvksJsonSchema().toString('utf8')); + const ajv = new Ajv(); + if(!ajv.validate(schema, source)) { + console.dir(source, {depth:8}); + this.callbacks.reportMessage(CompilerMessages.Error_InvalidFile({errorText: ajv.errorsText()})); + return null; + } + return source; } public transform(source: KVKSourceFile): VisualKeyboard { @@ -142,4 +135,33 @@ export default class LDMLKeyboardXMLSourceFileReader { return result; } + + /** + * xml2js will not place single-entry objects into arrays. + * Easiest way to fix this is to box them ourselves as needed + * @param source KVKSourceFile + */ + private boxArrays(source: KVKSourceFile) { + boxXmlArray(source.visualkeyboard, 'encoding'); + for(let encoding of source.visualkeyboard.encoding) { + boxXmlArray(encoding, 'layer'); + for(let layer of encoding.layer) { + boxXmlArray(layer, 'key'); + } + } + return source; + } + + + public kvksShiftToKvkShift(shift: string): VisualKeyboardShiftState { + shift = shift.toUpperCase(); + + // TODO-LDML(lowpri): make a map of this? + for(let state of VisualKeyboardLegalShiftStates) { + if(state.name == shift) { + return state.shift; + } + } + return 0; + } } \ No newline at end of file diff --git a/developer/src/kmc-keyboard/src/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/kmc-keyboard/src/ldml-keyboard/ldml-keyboard-xml-reader.ts index 601a1ef997..f820271567 100644 --- a/developer/src/kmc-keyboard/src/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/kmc-keyboard/src/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -65,6 +65,7 @@ export default class LDMLKeyboardXMLSourceFileReader { let parser = new xml2js.Parser({ explicitArray: false, mergeAttrs: true, + includeWhiteChars: false, emptyTag: {} as any // Why "as any"? xml2js is broken: // https://github.com/Leonidas-from-XIV/node-xml2js/issues/648 means diff --git a/developer/src/kmc/src/kmc.ts b/developer/src/kmc/src/kmc.ts index 9c2e8a8cf3..9606e15b82 100644 --- a/developer/src/kmc/src/kmc.ts +++ b/developer/src/kmc/src/kmc.ts @@ -49,6 +49,10 @@ class CompilerCallbacks { let schemaPath = new URL('ldml-keyboard.schema.json', import.meta.url); return fs.readFileSync(schemaPath); } + loadKvksJsonSchema(): Buffer { + let schemaPath = new URL('kvks.schema.json', import.meta.url); + return fs.readFileSync(schemaPath); + } } function compileKeyboard(inputFilename: string, options: kmc.CompilerOptions): Uint8Array {