From b4641efe691c5ad992618d2d7fdee416544494d6 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 10 Oct 2022 10:08:37 +1100 Subject: [PATCH 1/3] feat(developer): generate .js from ldml .xml Compiles a LDML keyboard .xml down to a KeymanWeb .js file. Currently only supports some metadata. Next step is to generate touch layout data, then to embed the binary kmx data. --- .../keyman-touch-layout-file-writer.ts | 61 ++++++++- .../src/compiler/keymanweb-compiler.ts | 117 ++++++++++++++++++ developer/src/kmc-keyboard/src/main.ts | 1 + .../test/fixtures/basic-no-debug.js | 1 + .../src/kmc-keyboard/test/fixtures/basic.js | 22 ++++ .../test/test-keymanweb-compiler.ts | 49 ++++++++ 6 files changed, 250 insertions(+), 1 deletion(-) create mode 100644 developer/src/kmc-keyboard/src/compiler/keymanweb-compiler.ts create mode 100644 developer/src/kmc-keyboard/test/fixtures/basic-no-debug.js create mode 100644 developer/src/kmc-keyboard/test/fixtures/basic.js create mode 100644 developer/src/kmc-keyboard/test/test-keymanweb-compiler.ts diff --git a/common/web/types/src/keyman-touch-layout/keyman-touch-layout-file-writer.ts b/common/web/types/src/keyman-touch-layout/keyman-touch-layout-file-writer.ts index 92d0b87408..5a7a43bb27 100644 --- a/common/web/types/src/keyman-touch-layout/keyman-touch-layout-file-writer.ts +++ b/common/web/types/src/keyman-touch-layout/keyman-touch-layout-file-writer.ts @@ -1,4 +1,4 @@ -import { TouchLayoutFile } from "./keyman-touch-layout-file.js"; +import { TouchLayoutFile, TouchLayoutPlatform, TouchLayoutKey, TouchLayoutSubKey } from "./keyman-touch-layout-file.js"; export interface TouchLayoutFileWriterOptions { formatted?: boolean; @@ -16,4 +16,63 @@ export class TouchLayoutFileWriter { const encoder = new TextEncoder(); return encoder.encode(output); } + + /** + * Compiles the touch layout file into a KeymanWeb-compatible JSON-style + * object string. In the future, this may be optimized to remove unnecessary + * quoting of property names, and remove unused properties, as this string + * is embedded into .js code. + * @param source + * @returns string + */ + compile(source: TouchLayoutFile): string { + // Deep copy the source + source = JSON.parse(JSON.stringify(source)); + + // Fixup pad, width and sp to string types, as that's what KeymanWeb + // currently expects + + const fixupKey = (key: TouchLayoutKey | TouchLayoutSubKey) => { + if(Object.hasOwn(key, 'pad')) (key.pad as any) = key.pad.toString(); + if(Object.hasOwn(key, 'sp')) (key.sp as any) = key.sp.toString(); + if(Object.hasOwn(key, 'width')) (key.width as any) = key.width.toString(); + }; + + const fixupPlatform = (platform: TouchLayoutPlatform) => { + for(let layer of platform.layer) { + for(let row of layer.row) { + for(let key of row.key) { + fixupKey(key); + if(key.sk) { + for(let sk of key.sk) { + fixupKey(sk); + } + } + if(key.multitap) { + for(let sk of key.multitap) { + fixupKey(sk); + } + } + if(key.flick) { + for(let id of Object.keys(key.flick)) { + fixupKey((key.flick as any)[id] as TouchLayoutSubKey); + } + } + } + } + } + }; + + if(source.desktop) { + fixupPlatform(source.desktop); + } + if(source.phone) { + fixupPlatform(source.phone); + } + if(source.tablet) { + fixupPlatform(source.tablet); + } + + return JSON.stringify(source); + } }; \ No newline at end of file diff --git a/developer/src/kmc-keyboard/src/compiler/keymanweb-compiler.ts b/developer/src/kmc-keyboard/src/compiler/keymanweb-compiler.ts new file mode 100644 index 0000000000..caf7ebd849 --- /dev/null +++ b/developer/src/kmc-keyboard/src/compiler/keymanweb-compiler.ts @@ -0,0 +1,117 @@ +import { VisualKeyboard, LDMLKeyboard } from "@keymanapp/common-types"; +import { BUILDER_KVK_HEADER_FLAGS } from "@keymanapp/common-types/src/kvk/kvk-file"; +import * as path from 'path'; +import VisualKeyboardCompiler from "./visual-keyboard-compiler.js"; + +const MINIMUM_KMW_VERSION = '16.0'; + +export interface KeymanWebCompilerOptions { + debug?: boolean; +}; + +export class KeymanWebCompiler { + private readonly options: KeymanWebCompilerOptions; + private readonly nl: string; + private readonly tab: string; + + constructor(options?: KeymanWebCompilerOptions) { + this.options = options; + this.nl = this.options.debug ? "\n" : ''; + this.tab = this.options.debug ? " " : ''; + } + + public compileVisualKeyboard(source: LDMLKeyboard.LDMLKeyboardXMLSourceFile) { + const nl = this.nl, tab = this.tab; + const vkc = new VisualKeyboardCompiler(); + const vk: VisualKeyboard.VisualKeyboard = vkc.compile(source); + + let result = + `{F: '${vk.header.unicodeFont.size}pt ${JSON.stringify(vk.header.unicodeFont.name)}', `+ + `K102: ${vk.header.flags & BUILDER_KVK_HEADER_FLAGS.kvkh102 ? 1 : 0}};${nl}` + // TODO-LDML: escape ' and " in font name correctly + `${tab}this.KV.KLS={${nl}` + + `${tab}${tab}TODO_LDML: ${vk.keys.length}${nl}` + + // TODO-LDML: fill in KLS + `${tab}}`; + + return result; + } + + public compileTouchLayout(source: LDMLKeyboard.LDMLKeyboardXMLSourceFile) { + // const tlc = new TouchLayoutCompiler(); + // const layout = tlc.compile(source); + // TODO-LDML + return ''; + } + + private cleanName(name: string): string { + let result = path.basename(name, '.xml').toLowerCase(); + if(!result.length) { + throw new Error(`Invalid file name ${name}`); + } + result = result.replaceAll(/[^a-z0-9]/g, '_'); + if(result.match(/^[0-9]/)) { + // Can't have a digit as initial + result = '_' + result; + } + return result; + } + + public compile(name: string, source: LDMLKeyboard.LDMLKeyboardXMLSourceFile): string { + const nl = this.nl, tab = this.tab; + + const sName = 'Keyboard_'+this.cleanName(name); + const displayUnderlying = true; // TODO-LDML + const modifierBitmask = 0; // TODO-LDML: define the modifiers used by this keyboard + const vkDictionary = ''; // TODO-LDML: vk dictionary for touch keys + const hasSupplementaryPlaneChars = false; // TODO-LDML + const isRTL = false; // TODO-LDML + + let result = + `if(typeof keyman === 'undefined') {${nl}` + + `${tab}console.error('Keyboard requires KeymanWeb ${MINIMUM_KMW_VERSION} or later');${nl}` + + `} else {${nl}` + + `${tab}KeymanWeb.KR(new ${sName}());${nl}` + + `}${nl}` + + `function ${sName}() {${nl}` + + // `${tab}${this.setupDebug()}${nl}` + ? we may use this for modifierBitmask in future + // `${tab}this._v=(typeof keyman!="undefined"&&typeof keyman.version=="string")?parseInt(keyman.version,10):9;${nl}` + ? we probably don't need this, it's for back-compat + `${tab}this.KI="${sName}";${nl}` + + `${tab}this.KN=${JSON.stringify(source.keyboard.names.name[0])};${nl}` + + `${tab}this.KMINVER=${JSON.stringify(MINIMUM_KMW_VERSION)};${nl}` + + `${tab}this.KV=${this.compileVisualKeyboard(source)};${nl}` + + `${tab}this.KDU=${displayUnderlying ? '1' : '0'};${nl}` + + `${tab}this.KH="";${nl}` + // TODO-LDML: help text not supported + `${tab}this.KM=0;${nl}` + // TODO-LDML: mnemonic layout not supported for LDML keyboards + `${tab}this.KBVER=${JSON.stringify(source.keyboard.version.number)};${nl}` + + `${tab}this.KMBM=${modifierBitmask};${nl}`; + + if(isRTL) { + result += `${tab}this.KRTL=1;${nl}`; + } + + if(hasSupplementaryPlaneChars) { + result += `${tab}this.KS=1;${nl}`; + } + + if(vkDictionary != '') { + result += `${tab}this.KVKD=${JSON.stringify(vkDictionary)};${nl}`; + } + + let layoutFile = this.compileTouchLayout(source); + if(layoutFile != '') { + result += `${tab}this.KVKL=${layoutFile};${nl}`; + } + // TODO-LDML: KCSS not supported + + // TODO-LDML: embed binary keyboard for loading into Core + + // A LDML keyboard has a no-op for its gs() (begin Unicode) function, + // because the functionality is embedded in Keyman Core + result += `${tab}this.gs=function(t,e){${nl}`+ + `${tab}${tab}return 0;${nl}`+ // TODO-LDML: we will start by embedding call into Keyman Core here + `${tab}};${nl}`; + + result += `}${nl}`; + return result; + } +} \ No newline at end of file diff --git a/developer/src/kmc-keyboard/src/main.ts b/developer/src/kmc-keyboard/src/main.ts index 81441cd709..cbeb006e9b 100644 --- a/developer/src/kmc-keyboard/src/main.ts +++ b/developer/src/kmc-keyboard/src/main.ts @@ -1,5 +1,6 @@ export { default as Compiler } from './compiler/compiler.js'; +export { KeymanWebCompiler } from './compiler/keymanweb-compiler.js'; export { default as VisualKeyboardCompiler } from './compiler/visual-keyboard-compiler.js'; export { CompilerEvent, default as CompilerCallbacks } from './compiler/callbacks.js'; export { default as CompilerOptions } from './compiler/compiler-options.js'; diff --git a/developer/src/kmc-keyboard/test/fixtures/basic-no-debug.js b/developer/src/kmc-keyboard/test/fixtures/basic-no-debug.js new file mode 100644 index 0000000000..f6b8fcbec8 --- /dev/null +++ b/developer/src/kmc-keyboard/test/fixtures/basic-no-debug.js @@ -0,0 +1 @@ +if(typeof keyman === 'undefined') {console.error('Keyboard requires KeymanWeb 16.0 or later');} else {KeymanWeb.KR(new Keyboard_basic());}function Keyboard_basic() {this.KI="Keyboard_basic";this.KN={"value":"TestKbd"};this.KMINVER="16.0";this.KV={F: '10pt "Arial"', K102: 0};this.KV.KLS={TODO_LDML: 2};this.KDU=1;this.KH="";this.KM=0;this.KBVER="1.0.0";this.KMBM=0;this.gs=function(t,e){return 0;};} \ No newline at end of file diff --git a/developer/src/kmc-keyboard/test/fixtures/basic.js b/developer/src/kmc-keyboard/test/fixtures/basic.js new file mode 100644 index 0000000000..1a01673afe --- /dev/null +++ b/developer/src/kmc-keyboard/test/fixtures/basic.js @@ -0,0 +1,22 @@ +if(typeof keyman === 'undefined') { + console.error('Keyboard requires KeymanWeb 16.0 or later'); +} else { + KeymanWeb.KR(new Keyboard_basic()); +} +function Keyboard_basic() { + this.KI="Keyboard_basic"; + this.KN={"value":"TestKbd"}; + this.KMINVER="16.0"; + this.KV={F: '10pt "Arial"', K102: 0}; + this.KV.KLS={ + TODO_LDML: 2 + }; + this.KDU=1; + this.KH=""; + this.KM=0; + this.KBVER="1.0.0"; + this.KMBM=0; + this.gs=function(t,e){ + return 0; + }; +} diff --git a/developer/src/kmc-keyboard/test/test-keymanweb-compiler.ts b/developer/src/kmc-keyboard/test/test-keymanweb-compiler.ts new file mode 100644 index 0000000000..d40cab2ecc --- /dev/null +++ b/developer/src/kmc-keyboard/test/test-keymanweb-compiler.ts @@ -0,0 +1,49 @@ +import 'mocha'; +import { assert } from 'chai'; +import { checkMessages, compilerTestCallbacks, makePathToFixture } from './helpers/index.js'; +import { KeymanWebCompiler } from '../src/compiler/keymanweb-compiler.js'; +import Compiler from '../src/compiler/compiler.js'; +import * as fs from 'fs'; + +describe('KeymanWebCompiler', function() { + + it('should build a .js file', async function() { + // Let's build basic.xml + // It should generate content identical to basic.js + const inputFilename = makePathToFixture('basic.xml'); + const outputFilename = makePathToFixture('basic.js'); + const outputFilenameNoDebug = makePathToFixture('basic-no-debug.js'); + + // Load input data; we'll use the LDML keyboard compiler loader to save us + // effort here + const k = new Compiler(compilerTestCallbacks, {debug: true, addCompilerVersion: false}); + const source = k.load(inputFilename); + checkMessages(); + assert.isNotNull(source, 'k.load should not have returned null'); + + // Sanity check ... this is also checked in other tests + const valid = k.validate(source); + checkMessages(); + assert.isTrue(valid, 'k.validate should not have failed'); + + // Actual test: compile to javascript + const jsCompiler = new KeymanWebCompiler({debug: true}); + const output = jsCompiler.compile('basic.xml', source); + assert.isNotNull(output); + + // Does the emitted js match? + const outputFixture = fs.readFileSync(outputFilename, 'utf-8'); + assert.strictEqual(output, outputFixture); + + // Second test: compile to javascript without debug formatting + const jsCompilerNoDebug = new KeymanWebCompiler({debug: false}); + const outputNoDebug = jsCompilerNoDebug.compile('basic.xml', source); + assert.isNotNull(outputNoDebug); + + // Does the emitted js match? + const outputFixtureNoDebug = fs.readFileSync(outputFilenameNoDebug, 'utf-8'); + assert.strictEqual(outputNoDebug, outputFixtureNoDebug); + + // TODO(lowpri): consider using Typescript parser to generate AST for further validation + }); +}); \ No newline at end of file From faa37f24a912915f55bfa08fd59ed1c72b41c813 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 10 Oct 2022 10:55:35 +1100 Subject: [PATCH 2/3] chore(developer): handle EOL diff in tests --- developer/src/kmc-keyboard/test/test-keymanweb-compiler.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/developer/src/kmc-keyboard/test/test-keymanweb-compiler.ts b/developer/src/kmc-keyboard/test/test-keymanweb-compiler.ts index d40cab2ecc..c4449b44c1 100644 --- a/developer/src/kmc-keyboard/test/test-keymanweb-compiler.ts +++ b/developer/src/kmc-keyboard/test/test-keymanweb-compiler.ts @@ -32,7 +32,7 @@ describe('KeymanWebCompiler', function() { assert.isNotNull(output); // Does the emitted js match? - const outputFixture = fs.readFileSync(outputFilename, 'utf-8'); + const outputFixture = fs.readFileSync(outputFilename, 'utf-8').replaceAll(/\r\n/g, '\n'); assert.strictEqual(output, outputFixture); // Second test: compile to javascript without debug formatting @@ -41,7 +41,7 @@ describe('KeymanWebCompiler', function() { assert.isNotNull(outputNoDebug); // Does the emitted js match? - const outputFixtureNoDebug = fs.readFileSync(outputFilenameNoDebug, 'utf-8'); + const outputFixtureNoDebug = fs.readFileSync(outputFilenameNoDebug, 'utf-8').replaceAll(/\r\n/g, '\n'); assert.strictEqual(outputNoDebug, outputFixtureNoDebug); // TODO(lowpri): consider using Typescript parser to generate AST for further validation From 47da67a742e0d9292fffe0972c24eb56cbf8b8ff Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 10 Oct 2022 11:51:55 +1100 Subject: [PATCH 3/3] chore(developer): fixup import --- developer/src/kmc-keyboard/src/compiler/keymanweb-compiler.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/developer/src/kmc-keyboard/src/compiler/keymanweb-compiler.ts b/developer/src/kmc-keyboard/src/compiler/keymanweb-compiler.ts index caf7ebd849..8ccb68993c 100644 --- a/developer/src/kmc-keyboard/src/compiler/keymanweb-compiler.ts +++ b/developer/src/kmc-keyboard/src/compiler/keymanweb-compiler.ts @@ -1,5 +1,5 @@ import { VisualKeyboard, LDMLKeyboard } from "@keymanapp/common-types"; -import { BUILDER_KVK_HEADER_FLAGS } from "@keymanapp/common-types/src/kvk/kvk-file"; + import * as path from 'path'; import VisualKeyboardCompiler from "./visual-keyboard-compiler.js"; @@ -27,7 +27,7 @@ export class KeymanWebCompiler { let result = `{F: '${vk.header.unicodeFont.size}pt ${JSON.stringify(vk.header.unicodeFont.name)}', `+ - `K102: ${vk.header.flags & BUILDER_KVK_HEADER_FLAGS.kvkh102 ? 1 : 0}};${nl}` + // TODO-LDML: escape ' and " in font name correctly + `K102: ${vk.header.flags & VisualKeyboard.VisualKeyboardHeaderFlags.kvkh102 ? 1 : 0}};${nl}` + // TODO-LDML: escape ' and " in font name correctly `${tab}this.KV.KLS={${nl}` + `${tab}${tab}TODO_LDML: ${vk.keys.length}${nl}` + // TODO-LDML: fill in KLS