Merge pull request #7422 from keymanapp/feat/developer/epic-ldml/7238-generate-keymanweb-js

feat(developer): generate .js from ldml .xml 🙀

common/web/types/src/keyman-touch-layout/keyman-touch-layout-file-writer.ts

@@ -1,4 +1,4 @@
-import { TouchLayoutFile } from "./keyman-touch-layout-file.js";
+import { TouchLayoutFile, TouchLayoutPlatform, TouchLayoutKey, TouchLayoutSubKey } from "./keyman-touch-layout-file.js";
 
 export interface TouchLayoutFileWriterOptions {
   formatted?: boolean;
@@ -16,4 +16,63 @@ export class TouchLayoutFileWriter {
     const encoder = new TextEncoder();
     return encoder.encode(output);
   }
+
+  /**
+   * Compiles the touch layout file into a KeymanWeb-compatible JSON-style
+   * object string. In the future, this may be optimized to remove unnecessary
+   * quoting of property names, and remove unused properties, as this string
+   * is embedded into .js code.
+   * @param source
+   * @returns string
+   */
+  compile(source: TouchLayoutFile): string {
+    // Deep copy the source
+    source = JSON.parse(JSON.stringify(source));
+
+    // Fixup pad, width and sp to string types, as that's what KeymanWeb
+    // currently expects
+
+    const fixupKey = (key: TouchLayoutKey | TouchLayoutSubKey) => {
+      if(Object.hasOwn(key, 'pad')) (key.pad as any) = key.pad.toString();
+      if(Object.hasOwn(key, 'sp')) (key.sp as any) = key.sp.toString();
+      if(Object.hasOwn(key, 'width')) (key.width as any) = key.width.toString();
+    };
+
+    const fixupPlatform = (platform: TouchLayoutPlatform) => {
+      for(let layer of platform.layer) {
+        for(let row of layer.row) {
+          for(let key of row.key) {
+            fixupKey(key);
+            if(key.sk) {
+              for(let sk of key.sk) {
+                fixupKey(sk);
+              }
+            }
+            if(key.multitap) {
+              for(let sk of key.multitap) {
+                fixupKey(sk);
+              }
+            }
+            if(key.flick) {
+              for(let id of Object.keys(key.flick)) {
+                fixupKey((key.flick as any)[id] as TouchLayoutSubKey);
+              }
+            }
+          }
+        }
+      }
+    };
+
+    if(source.desktop) {
+      fixupPlatform(source.desktop);
+    }
+    if(source.phone) {
+      fixupPlatform(source.phone);
+    }
+    if(source.tablet) {
+      fixupPlatform(source.tablet);
+    }
+
+    return JSON.stringify(source);
+  }
 };

developer/src/kmc-keyboard/src/compiler/keymanweb-compiler.ts

@@ -0,0 +1,117 @@
+import { VisualKeyboard, LDMLKeyboard } from "@keymanapp/common-types";
+
+import * as path from 'path';
+import VisualKeyboardCompiler from "./visual-keyboard-compiler.js";
+
+const MINIMUM_KMW_VERSION = '16.0';
+
+export interface KeymanWebCompilerOptions {
+  debug?: boolean;
+};
+
+export class KeymanWebCompiler {
+  private readonly options: KeymanWebCompilerOptions;
+  private readonly nl: string;
+  private readonly tab: string;
+
+  constructor(options?: KeymanWebCompilerOptions) {
+    this.options = options;
+    this.nl = this.options.debug ? "\n" : '';
+    this.tab = this.options.debug ? "  " : '';
+  }
+
+  public compileVisualKeyboard(source: LDMLKeyboard.LDMLKeyboardXMLSourceFile) {
+    const nl = this.nl, tab = this.tab;
+    const vkc = new VisualKeyboardCompiler();
+    const vk: VisualKeyboard.VisualKeyboard = vkc.compile(source);
+
+    let result =
+      `{F: '${vk.header.unicodeFont.size}pt ${JSON.stringify(vk.header.unicodeFont.name)}', `+
+      `K102: ${vk.header.flags & VisualKeyboard.VisualKeyboardHeaderFlags.kvkh102 ? 1 : 0}};${nl}` + // TODO-LDML: escape ' and " in font name correctly
+      `${tab}this.KV.KLS={${nl}` +
+      `${tab}${tab}TODO_LDML: ${vk.keys.length}${nl}` +
+      // TODO-LDML: fill in KLS
+      `${tab}}`;
+
+    return result;
+  }
+
+  public compileTouchLayout(source: LDMLKeyboard.LDMLKeyboardXMLSourceFile) {
+    // const tlc = new TouchLayoutCompiler();
+    // const layout = tlc.compile(source);
+    // TODO-LDML
+    return '';
+  }
+
+  private cleanName(name: string): string {
+    let result = path.basename(name, '.xml').toLowerCase();
+    if(!result.length) {
+      throw new Error(`Invalid file name ${name}`);
+    }
+    result = result.replaceAll(/[^a-z0-9]/g, '_');
+    if(result.match(/^[0-9]/)) {
+      // Can't have a digit as initial
+      result = '_' + result;
+    }
+    return result;
+  }
+
+  public compile(name: string, source: LDMLKeyboard.LDMLKeyboardXMLSourceFile): string {
+    const nl = this.nl, tab = this.tab;
+
+    const sName = 'Keyboard_'+this.cleanName(name);
+    const displayUnderlying = true; // TODO-LDML
+    const modifierBitmask = 0; // TODO-LDML: define the modifiers used by this keyboard
+    const vkDictionary = '';  // TODO-LDML: vk dictionary for touch keys
+    const hasSupplementaryPlaneChars = false; // TODO-LDML
+    const isRTL = false; // TODO-LDML
+
+    let result =
+      `if(typeof keyman === 'undefined') {${nl}` +
+      `${tab}console.error('Keyboard requires KeymanWeb ${MINIMUM_KMW_VERSION} or later');${nl}` +
+      `} else {${nl}` +
+      `${tab}KeymanWeb.KR(new ${sName}());${nl}` +
+      `}${nl}` +
+      `function ${sName}() {${nl}` +
+      // `${tab}${this.setupDebug()}${nl}` + ? we may use this for modifierBitmask in future
+      // `${tab}this._v=(typeof keyman!="undefined"&&typeof keyman.version=="string")?parseInt(keyman.version,10):9;${nl}` + ? we probably don't need this, it's for back-compat
+      `${tab}this.KI="${sName}";${nl}` +
+      `${tab}this.KN=${JSON.stringify(source.keyboard.names.name[0])};${nl}` +
+      `${tab}this.KMINVER=${JSON.stringify(MINIMUM_KMW_VERSION)};${nl}` +
+      `${tab}this.KV=${this.compileVisualKeyboard(source)};${nl}` +
+      `${tab}this.KDU=${displayUnderlying ? '1' : '0'};${nl}` +
+      `${tab}this.KH="";${nl}` +  // TODO-LDML: help text not supported
+      `${tab}this.KM=0;${nl}` +  // TODO-LDML: mnemonic layout not supported for LDML keyboards
+      `${tab}this.KBVER=${JSON.stringify(source.keyboard.version.number)};${nl}` +
+      `${tab}this.KMBM=${modifierBitmask};${nl}`;
+
+    if(isRTL) {
+      result += `${tab}this.KRTL=1;${nl}`;
+    }
+
+    if(hasSupplementaryPlaneChars) {
+      result += `${tab}this.KS=1;${nl}`;
+    }
+
+    if(vkDictionary != '') {
+      result += `${tab}this.KVKD=${JSON.stringify(vkDictionary)};${nl}`;
+    }
+
+    let layoutFile = this.compileTouchLayout(source);
+    if(layoutFile != '') {
+      result += `${tab}this.KVKL=${layoutFile};${nl}`;
+    }
+    // TODO-LDML: KCSS not supported
+
+    // TODO-LDML: embed binary keyboard for loading into Core
+
+    // A LDML keyboard has a no-op for its gs() (begin Unicode) function,
+    // because the functionality is embedded in Keyman Core
+    result += `${tab}this.gs=function(t,e){${nl}`+
+              `${tab}${tab}return 0;${nl}`+ // TODO-LDML: we will start by embedding call into Keyman Core here
+              `${tab}};${nl}`;
+
+    result += `}${nl}`;
+    return result;
+  }
+}

developer/src/kmc-keyboard/src/main.ts

@@ -1,5 +1,6 @@
 
 export { default as Compiler } from './compiler/compiler.js';
+export { KeymanWebCompiler } from './compiler/keymanweb-compiler.js';
 export { default as VisualKeyboardCompiler } from './compiler/visual-keyboard-compiler.js';
 export { CompilerEvent, default as CompilerCallbacks } from './compiler/callbacks.js';
 export { default as CompilerOptions } from './compiler/compiler-options.js';

developer/src/kmc-keyboard/test/fixtures/basic-no-debug.js

@@ -0,0 +1 @@
+if(typeof keyman === 'undefined') {console.error('Keyboard requires KeymanWeb 16.0 or later');} else {KeymanWeb.KR(new Keyboard_basic());}function Keyboard_basic() {this.KI="Keyboard_basic";this.KN={"value":"TestKbd"};this.KMINVER="16.0";this.KV={F: '10pt "Arial"', K102: 0};this.KV.KLS={TODO_LDML: 2};this.KDU=1;this.KH="";this.KM=0;this.KBVER="1.0.0";this.KMBM=0;this.gs=function(t,e){return 0;};}

developer/src/kmc-keyboard/test/fixtures/basic.js

@@ -0,0 +1,22 @@
+if(typeof keyman === 'undefined') {
+  console.error('Keyboard requires KeymanWeb 16.0 or later');
+} else {
+  KeymanWeb.KR(new Keyboard_basic());
+}
+function Keyboard_basic() {
+  this.KI="Keyboard_basic";
+  this.KN={"value":"TestKbd"};
+  this.KMINVER="16.0";
+  this.KV={F: '10pt "Arial"', K102: 0};
+  this.KV.KLS={
+    TODO_LDML: 2
+  };
+  this.KDU=1;
+  this.KH="";
+  this.KM=0;
+  this.KBVER="1.0.0";
+  this.KMBM=0;
+  this.gs=function(t,e){
+    return 0;
+  };
+}

developer/src/kmc-keyboard/test/test-keymanweb-compiler.ts

@@ -0,0 +1,49 @@
+import 'mocha';
+import { assert } from 'chai';
+import { checkMessages, compilerTestCallbacks, makePathToFixture } from './helpers/index.js';
+import { KeymanWebCompiler } from '../src/compiler/keymanweb-compiler.js';
+import Compiler from '../src/compiler/compiler.js';
+import * as fs from 'fs';
+
+describe('KeymanWebCompiler', function() {
+
+  it('should build a .js file', async function() {
+    // Let's build basic.xml
+    // It should generate content identical to basic.js
+    const inputFilename = makePathToFixture('basic.xml');
+    const outputFilename = makePathToFixture('basic.js');
+    const outputFilenameNoDebug = makePathToFixture('basic-no-debug.js');
+
+    // Load input data; we'll use the LDML keyboard compiler loader to save us
+    // effort here
+    const k = new Compiler(compilerTestCallbacks, {debug: true, addCompilerVersion: false});
+    const source = k.load(inputFilename);
+    checkMessages();
+    assert.isNotNull(source, 'k.load should not have returned null');
+
+    // Sanity check ... this is also checked in other tests
+    const valid = k.validate(source);
+    checkMessages();
+    assert.isTrue(valid, 'k.validate should not have failed');
+
+    // Actual test: compile to javascript
+    const jsCompiler = new KeymanWebCompiler({debug: true});
+    const output = jsCompiler.compile('basic.xml', source);
+    assert.isNotNull(output);
+
+    // Does the emitted js match?
+    const outputFixture = fs.readFileSync(outputFilename, 'utf-8').replaceAll(/\r\n/g, '\n');
+    assert.strictEqual(output, outputFixture);
+
+    // Second test: compile to javascript without debug formatting
+    const jsCompilerNoDebug = new KeymanWebCompiler({debug: false});
+    const outputNoDebug = jsCompilerNoDebug.compile('basic.xml', source);
+    assert.isNotNull(outputNoDebug);
+
+    // Does the emitted js match?
+    const outputFixtureNoDebug = fs.readFileSync(outputFilenameNoDebug, 'utf-8').replaceAll(/\r\n/g, '\n');
+    assert.strictEqual(outputNoDebug, outputFixtureNoDebug);
+
+    // TODO(lowpri): consider using Typescript parser to generate AST for further validation
+  });
+});