Merge pull request #7154 from keymanapp/feat/developer/kmldmlc-meta-tests-and-cleanup

feat(developer): add unit tests for meta compiler 🙀

developer/src/kmldmlc/src/keyman/compiler/callbacks.ts

@@ -1,4 +1,4 @@
 export default interface CompilerCallbacks {
   loadFile(baseFilename: string, filename: string): Buffer;
-  reportMessage(severity: number, message: string): void;
+  reportMessage(code: number, message: string): void;
 };

developer/src/kmldmlc/src/keyman/compiler/errors.ts

@@ -0,0 +1,26 @@
+
+export enum CompilerErrorSeverity {
+  Info =          0x000000,
+  Hint =          0x100000,
+  Warn =          0x200000,
+  Error =         0x300000,
+  Fatal =         0x400000,
+  Severity_Mask = 0xF00000, // includes reserved bits
+  Error_Mask =    0x0FFFFF,
+};
+
+export enum CompilerErrors {
+  ERROR_InvalidNormalization = CompilerErrorSeverity.Error | 0x0001 // `Invalid normalization form '${}'`
+};
+
+export function getErrorSeverityName(code: number): string {
+  let severity = code & CompilerErrorSeverity.Severity_Mask;
+  switch(severity) {
+    case CompilerErrorSeverity.Info: return 'INFO';
+    case CompilerErrorSeverity.Hint: return 'HINT';
+    case CompilerErrorSeverity.Warn: return 'WARN';
+    case CompilerErrorSeverity.Error: return 'ERROR';
+    case CompilerErrorSeverity.Fatal: return 'FATAL';
+    default: return 'UNKNOWN';
+  }
+}

developer/src/kmldmlc/src/keyman/compiler/keys.ts

@@ -14,8 +14,8 @@ export class KeysCompiler extends SectionCompiler {
   public compile(): Keys {
     // Use LayerMap + keys to generate compiled keys for hardware
 
-    if(this.source.keyboard.layerMaps?.[0]?.form == 'hardware') {
-      for(let layer of this.source.keyboard.layerMaps[0].layerMap) {
+    if(this.keyboard.layerMaps?.[0]?.form == 'hardware') {
+      for(let layer of this.keyboard.layerMaps[0].layerMap) {
         let sect = this.compileHardwareLayer(layer);
         return sect;
       }
@@ -49,7 +49,7 @@ export class KeysCompiler extends SectionCompiler {
           break;
         }
 
-        let keydef = this.source.keyboard.keys?.key?.find(x => x.id == key);
+        let keydef = this.keyboard.keys?.key?.find(x => x.id == key);
         if(!keydef) {
           this.callbacks.reportMessage(0,
             `Key ${key} in position #${x+1} on row #${y+1} of layer ${layer.id}, form 'hardware' not found in key bag`);

developer/src/kmldmlc/src/keyman/compiler/loca.ts

@@ -10,7 +10,7 @@ export class LocaCompiler extends SectionCompiler {
 
   public compile(): Loca {
     let result = new Loca();
-    result.locales.push(this.source.keyboard.locale);
+    result.locales.push(this.keyboard.locale);
     return result;
   }
 }

developer/src/kmldmlc/src/keyman/compiler/meta.ts

@@ -1,5 +1,7 @@
 import { constants } from "@keymanapp/ldml-keyboard-constants";
-import { Meta } from "../kmx/kmx-plus";
+import { KeyboardSettings, Meta, Meta_NormalizationForm } from "../kmx/kmx-plus";
+import { isValidEnumValue } from "../util/util";
+import { CompilerErrors } from "./errors";
 import { SectionCompiler } from "./section-compiler";
 
 export class MetaCompiler extends SectionCompiler {
@@ -9,19 +11,31 @@ export class MetaCompiler extends SectionCompiler {
   }
 
   public validate(): boolean {
-    //
-    return true;
+    let valid = true;
+
+    const normalization = this.keyboard.info?.normalization;
+    if(normalization !== undefined) {
+      if(!isValidEnumValue(Meta_NormalizationForm, normalization)) {
+        this.callbacks.reportMessage(CompilerErrors.ERROR_InvalidNormalization, `Invalid normalization form '${normalization}'`);
+        valid = false;
+      }
+    }
+
+    return valid;
   }
 
   public compile(): Meta {
     let result = new Meta();
-    result.name = this.source.keyboard.names?.name?.[0]?.value;
-    result.author = this.source.keyboard.info?.author;
-    result.conform = this.source.keyboard.conformsTo;
-    result.layout = this.source.keyboard.info?.layout;
-    result.normalization = this.source.keyboard.info?.normalization;
-    result.indicator = this.source.keyboard.info?.indicator;
-    result.settings = 0;
+    result.name = this.keyboard.names?.name?.[0]?.value;
+    result.author = this.keyboard.info?.author;
+    result.conform = this.keyboard.conformsTo;
+    result.layout = this.keyboard.info?.layout;
+    result.normalization = this.keyboard.info?.normalization as Meta_NormalizationForm;
+    result.indicator = this.keyboard.info?.indicator;
+    result.settings =
+      (this.keyboard.settings?.fallback == "omit" ? KeyboardSettings.fallback : 0) |
+      (this.keyboard.settings?.transformFailure == "omit" ? KeyboardSettings.transformFailure : 0) |
+      (this.keyboard.settings?.transformPartial == "hide" ? KeyboardSettings.transformPartial : 0);
     return result;
   }
 }

developer/src/kmldmlc/src/keyman/compiler/section-compiler.ts

@@ -1,14 +1,14 @@
 import { Section } from "../kmx/kmx-plus";
-import LDMLKeyboardXMLSourceFile from "../ldml-keyboard/ldml-keyboard-xml";
+import LDMLKeyboardXMLSourceFile, { LKKeyboard } from "../ldml-keyboard/ldml-keyboard-xml";
 import CompilerCallbacks from "./callbacks";
 import { SectionIdent } from '@keymanapp/ldml-keyboard-constants';
 
 export class SectionCompiler {
-  protected readonly source: LDMLKeyboardXMLSourceFile;
+  protected readonly keyboard: LKKeyboard;
   protected readonly callbacks: CompilerCallbacks;
 
   constructor(source: LDMLKeyboardXMLSourceFile, callbacks: CompilerCallbacks) {
-    this.source = source;
+    this.keyboard = source.keyboard;
     this.callbacks = callbacks;
   }
 

developer/src/kmldmlc/src/keyman/kmx/kmx-plus.ts

@@ -16,12 +16,14 @@ export enum KeyboardSettings {
   transformPartial = 1<<2,
 };
 
+export enum Meta_NormalizationForm { NFC='NFC', NFD='NFD', other='other' };
+
 export class Meta extends Section {
   name: string;
   author: string;
   conform: string;
   layout: string;
-  normalization: string;
+  normalization: Meta_NormalizationForm;
   indicator: string;
   settings: KeyboardSettings;
 };

developer/src/kmldmlc/src/keyman/ldml-keyboard/ldml-keyboard-xml.ts

@@ -20,6 +20,7 @@ export interface LKKeyboard {
 
   info?: LKInfo;
   names?: LKNames;
+  settings?: LKSettings;
   keys?: LKKeys;
   layerMaps?: LKLayerMaps[];
 };
@@ -35,6 +36,12 @@ export interface LKNames {
   name: LKName[];
 };
 
+export interface LKSettings {
+  fallback: "omit";
+  transformFailure: "omit";
+  transformPartial: "hide";
+};
+
 export interface LKName {
   value?: string;
 };

developer/src/kmldmlc/src/keyman/util/util.ts

@@ -0,0 +1,7 @@
+
+/**
+ * Verifies that value is an item in the enumeration.
+ */
+export function isValidEnumValue<T extends {[key: number]: string | number}>(enu: T, value: string) {
+  return (Object.values(enu) as string[]).includes(value);
+}

developer/src/kmldmlc/src/kmldmlc.ts

@@ -9,6 +9,7 @@ import * as program from 'commander';
 
 import Compiler from './keyman/compiler/compiler';
 import KMXBuilder from './keyman/kmx/kmx-builder';
+import { getErrorSeverityName } from './keyman/compiler/errors';
 
 let inputFilename: string;
 
@@ -41,8 +42,8 @@ class CompilerCallbacks {
     // TODO: translate filename based on the baseFilename
     return fs.readFileSync(filename);
   }
-  reportMessage(severity: number, message: string): void {
-    console.log(message);
+  reportMessage(code: number, message: string): void {
+    console.log(getErrorSeverityName(code) + ' ' + code.toString(16) + ': ' + message);
   }
 }
 

developer/src/kmldmlc/test/fixtures/sections/meta/invalid-normalization.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!DOCTYPE keyboard SYSTEM "../../../../../../../resources/standards-data/ldml-keyboards/techpreview/ldmlKeyboard.dtd">
+<keyboard locale="mt" conformsTo="techpreview">
+  <info normalization="NFQ" />
+
+  <names>
+    <name value="meta-minimal" />
+  </names>
+
+  <keys />
+</keyboard>

developer/src/kmldmlc/test/fixtures/sections/meta/maximal.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!DOCTYPE keyboard SYSTEM "../../../../../../../resources/standards-data/ldml-keyboards/techpreview/ldmlKeyboard.dtd">
+<keyboard locale="mt" conformsTo="techpreview">
+  <info author="The Keyman Team" indicator="QW" layout="QWIRKY" normalization="NFC" />
+
+  <names>
+    <name value="meta-maximal" />
+  </names>
+
+  <settings fallback="omit" transformFailure="omit" transformPartial="hide" />
+
+  <keys />
+</keyboard>

developer/src/kmldmlc/test/fixtures/sections/meta/minimal.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!DOCTYPE keyboard SYSTEM "../../../../../../../resources/standards-data/ldml-keyboards/techpreview/ldmlKeyboard.dtd">
+<keyboard locale="mt" conformsTo="techpreview">
+  <names>
+    <name value="meta-minimal" />
+  </names>
+
+  <keys />
+</keyboard>

developer/src/kmldmlc/test/helpers/index.ts

@@ -2,6 +2,10 @@
  * Helpers and utilities for the Mocha tests.
  */
 import * as path from 'path';
+import * as fs from 'fs';
+import { SectionCompiler } from '../../src/keyman/compiler/section-compiler';
+import { Section } from '../../src/keyman/kmx/kmx-plus';
+import LDMLKeyboardXMLSourceFileReader from '../../src/keyman/ldml-keyboard/ldml-keyboard-xml-reader';
 
 /**
  * Builds a path to the fixture with the given path components.
@@ -14,3 +18,24 @@ export function makePathToFixture(...components: string[]): string {
   return path.join(__dirname, '..', '..', '..', 'test', 'fixtures', ...components);
 }
 
+
+export class CompilerCallbacks {
+  messages: { code: number, message: string}[] = [];
+  loadFile(baseFilename: string, filename:string): Buffer {
+    // TODO: translate filename based on the baseFilename
+    return fs.readFileSync(filename);
+  }
+  reportMessage(code: number, message: string): void {
+    this.messages.push({code, message});
+  }
+}
+
+export function loadSectionFixture(compilerClass: typeof SectionCompiler, filename: string, callbacks: CompilerCallbacks): Section {
+  const inputFilename = makePathToFixture(filename);
+  const source = (new LDMLKeyboardXMLSourceFileReader(callbacks)).loadFile(inputFilename);
+  const compiler = new compilerClass(source, callbacks);
+  if(!compiler.validate()) {
+    return null;
+  }
+  return compiler.compile();
+}

developer/src/kmldmlc/test/test-compiler-e2e.ts

@@ -5,21 +5,11 @@ import {assert} from 'chai';
 import hextobin from '@keymanapp/hextobin';
 import Compiler from '../src/keyman/compiler/compiler';
 import KMXBuilder from '../src/keyman/kmx/kmx-builder';
-import {makePathToFixture} from './helpers/index';
-
-class CompilerCallbacks {
-  loadFile(baseFilename: string, filename:string): Buffer {
-    // TODO: translate filename based on the baseFilename
-    return fs.readFileSync(filename);
-  }
-  reportMessage(severity: number, message: string): void {
-    console.log(message);
-  }
-}
+import {CompilerCallbacks, makePathToFixture} from './helpers/index';
 
 function compileKeyboard(inputFilename: string): Uint8Array {
-  const c = new CompilerCallbacks();
-  const k = new Compiler(c);
+  const callbacks = new CompilerCallbacks();
+  const k = new Compiler(callbacks);
   let source = k.load(inputFilename);
   if(!source) {
     return null;
@@ -34,7 +24,9 @@ function compileKeyboard(inputFilename: string): Uint8Array {
 
   // Use the builder to generate the binary output file
   let builder = new KMXBuilder(kmx, true);
-  return builder.compile();
+  let result = builder.compile();
+  assert(callbacks.messages.length == 0);
+  return result;
 }
 
 describe('compiler-tests', function() {

developer/src/kmldmlc/test/test-meta.ts

@@ -0,0 +1,45 @@
+import 'mocha';
+import {assert} from 'chai';
+import { MetaCompiler } from '../src/keyman/compiler/meta';
+import { CompilerCallbacks, loadSectionFixture } from './helpers';
+import { KeyboardSettings, Meta } from '../src/keyman/kmx/kmx-plus';
+import { CompilerErrors } from '../src/keyman/compiler/errors';
+
+describe('meta', function () {
+  it('should compile minimal metadata', function() {
+    const callbacks = new CompilerCallbacks();
+    let meta = loadSectionFixture(MetaCompiler, 'sections/meta/minimal.xml', callbacks) as Meta;
+    assert.equal(callbacks.messages.length, 0);
+
+    assert.equal(meta.name, 'meta-minimal');
+    assert.isUndefined(meta.author);        // TODO-LDML: default author string "unknown"?
+    assert.equal(meta.conform, 'techpreview');
+    assert.isUndefined(meta.layout);        // TODO-LDML: assumed layout?
+    assert.isUndefined(meta.normalization); // TODO-LDML: assumed normalization?
+    assert.isUndefined(meta.indicator);     // TODO-LDML: synthesize an indicator?
+    assert.equal(meta.settings, KeyboardSettings.none);
+  });
+
+  it('should compile maximal metadata', function() {
+    const callbacks = new CompilerCallbacks();
+    let meta = loadSectionFixture(MetaCompiler, 'sections/meta/maximal.xml', callbacks) as Meta;
+    assert.equal(callbacks.messages.length, 0);
+
+    assert.equal(meta.name, 'meta-maximal');
+    assert.equal(meta.author, 'The Keyman Team');
+    assert.equal(meta.conform, 'techpreview');
+    assert.equal(meta.layout, 'QWIRKY');
+    assert.equal(meta.normalization, 'NFC');
+    assert.equal(meta.indicator, 'QW');
+    assert.equal(meta.settings, KeyboardSettings.fallback | KeyboardSettings.transformFailure | KeyboardSettings.transformPartial);
+  });
+
+  it('should reject invalid normalization', function() {
+    const callbacks = new CompilerCallbacks();
+    let meta = loadSectionFixture(MetaCompiler, 'sections/meta/invalid-normalization.xml', callbacks) as Meta;
+    assert.isNull(meta);
+    assert.equal(callbacks.messages.length, 1);
+    assert.deepEqual(callbacks.messages[0], {code: CompilerErrors.ERROR_InvalidNormalization, message: "Invalid normalization form 'NFQ'"});
+  })
+});
+