From 995d731ccbbc178aebc67164416acc53d53e0c0c Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Tue, 30 Aug 2022 13:52:36 +1000 Subject: [PATCH 1/2] feat(developer): add unit tests for meta compiler First, a little bit of refactoring to make it cleaner to do unit test for each section. Then add some global error values. Throw in a helper function or two. Dust off the edges of the CompilerCallbacks object. Duct tape together validation of the normalization attribute. And ... eventually we get a nice clean-like unit test for the meta compiler, with three pretty little fixtures and as-neat-as-you-like-it assertions. --- .../kmldmlc/src/keyman/compiler/callbacks.ts | 2 +- .../src/kmldmlc/src/keyman/compiler/errors.ts | 26 +++++++++++ .../src/kmldmlc/src/keyman/compiler/keys.ts | 6 +-- .../src/kmldmlc/src/keyman/compiler/loca.ts | 2 +- .../src/kmldmlc/src/keyman/compiler/meta.ts | 34 +++++++++----- .../src/keyman/compiler/section-compiler.ts | 6 +-- .../src/kmldmlc/src/keyman/kmx/kmx-plus.ts | 4 +- .../keyman/ldml-keyboard/ldml-keyboard-xml.ts | 7 +++ developer/src/kmldmlc/src/keyman/util/util.ts | 7 +++ developer/src/kmldmlc/src/kmldmlc.ts | 5 ++- .../sections/meta/invalid-normalization.xml | 12 +++++ .../test/fixtures/sections/meta/maximal.xml | 14 ++++++ .../test/fixtures/sections/meta/minimal.xml | 10 +++++ developer/src/kmldmlc/test/helpers/index.ts | 25 +++++++++++ .../src/kmldmlc/test/test-compiler-e2e.ts | 20 +++------ developer/src/kmldmlc/test/test-meta.ts | 45 +++++++++++++++++++ 16 files changed, 190 insertions(+), 35 deletions(-) create mode 100644 developer/src/kmldmlc/src/keyman/compiler/errors.ts create mode 100644 developer/src/kmldmlc/src/keyman/util/util.ts create mode 100644 developer/src/kmldmlc/test/fixtures/sections/meta/invalid-normalization.xml create mode 100644 developer/src/kmldmlc/test/fixtures/sections/meta/maximal.xml create mode 100644 developer/src/kmldmlc/test/fixtures/sections/meta/minimal.xml create mode 100644 developer/src/kmldmlc/test/test-meta.ts diff --git a/developer/src/kmldmlc/src/keyman/compiler/callbacks.ts b/developer/src/kmldmlc/src/keyman/compiler/callbacks.ts index 7191da8c8a..509a0b2090 100644 --- a/developer/src/kmldmlc/src/keyman/compiler/callbacks.ts +++ b/developer/src/kmldmlc/src/keyman/compiler/callbacks.ts @@ -1,4 +1,4 @@ export default interface CompilerCallbacks { loadFile(baseFilename: string, filename: string): Buffer; - reportMessage(severity: number, message: string): void; + reportMessage(code: number, message: string): void; }; diff --git a/developer/src/kmldmlc/src/keyman/compiler/errors.ts b/developer/src/kmldmlc/src/keyman/compiler/errors.ts new file mode 100644 index 0000000000..33e7ef8e6a --- /dev/null +++ b/developer/src/kmldmlc/src/keyman/compiler/errors.ts @@ -0,0 +1,26 @@ + +export enum CompilerErrorSeverity { + Info = 0x000000, + Hint = 0x100000, + Warn = 0x200000, + Error = 0x300000, + Fatal = 0x400000, + Severity_Mask = 0xF00000, // includes reserved bits + Error_Mask = 0x0FFFFF, +}; + +export enum CompilerErrors { + ERROR_InvalidNormalization = CompilerErrorSeverity.Error | 0x0001 // `Invalid normalization form '${}'` +}; + +export function getErrorSeverityName(code: number): string { + let severity = code & CompilerErrorSeverity.Severity_Mask; + switch(severity) { + case CompilerErrorSeverity.Info: return 'INFO'; + case CompilerErrorSeverity.Hint: return 'HINT'; + case CompilerErrorSeverity.Warn: return 'WARN'; + case CompilerErrorSeverity.Error: return 'ERROR'; + case CompilerErrorSeverity.Fatal: return 'FATAL'; + default: return 'UNKNOWN'; + } +} \ No newline at end of file diff --git a/developer/src/kmldmlc/src/keyman/compiler/keys.ts b/developer/src/kmldmlc/src/keyman/compiler/keys.ts index c48898fa61..2108edd813 100644 --- a/developer/src/kmldmlc/src/keyman/compiler/keys.ts +++ b/developer/src/kmldmlc/src/keyman/compiler/keys.ts @@ -14,8 +14,8 @@ export class KeysCompiler extends SectionCompiler { public compile(): Keys { // Use LayerMap + keys to generate compiled keys for hardware - if(this.source.keyboard.layerMaps?.[0]?.form == 'hardware') { - for(let layer of this.source.keyboard.layerMaps[0].layerMap) { + if(this.keyboard.layerMaps?.[0]?.form == 'hardware') { + for(let layer of this.keyboard.layerMaps[0].layerMap) { let sect = this.compileHardwareLayer(layer); return sect; } @@ -49,7 +49,7 @@ export class KeysCompiler extends SectionCompiler { break; } - let keydef = this.source.keyboard.keys?.key?.find(x => x.id == key); + let keydef = this.keyboard.keys?.key?.find(x => x.id == key); if(!keydef) { this.callbacks.reportMessage(0, `Key ${key} in position #${x+1} on row #${y+1} of layer ${layer.id}, form 'hardware' not found in key bag`); diff --git a/developer/src/kmldmlc/src/keyman/compiler/loca.ts b/developer/src/kmldmlc/src/keyman/compiler/loca.ts index bca07e5934..ec26425fab 100644 --- a/developer/src/kmldmlc/src/keyman/compiler/loca.ts +++ b/developer/src/kmldmlc/src/keyman/compiler/loca.ts @@ -10,7 +10,7 @@ export class LocaCompiler extends SectionCompiler { public compile(): Loca { let result = new Loca(); - result.locales.push(this.source.keyboard.locale); + result.locales.push(this.keyboard.locale); return result; } } diff --git a/developer/src/kmldmlc/src/keyman/compiler/meta.ts b/developer/src/kmldmlc/src/keyman/compiler/meta.ts index 465a593e50..21a4e71ef6 100644 --- a/developer/src/kmldmlc/src/keyman/compiler/meta.ts +++ b/developer/src/kmldmlc/src/keyman/compiler/meta.ts @@ -1,5 +1,7 @@ import { constants } from "@keymanapp/ldml-keyboard-constants"; -import { Meta } from "../kmx/kmx-plus"; +import { KeyboardSettings, Meta, Meta_NormalizationForm } from "../kmx/kmx-plus"; +import { isValidEnumValue } from "../util/util"; +import { CompilerErrors } from "./errors"; import { SectionCompiler } from "./section-compiler"; export class MetaCompiler extends SectionCompiler { @@ -9,19 +11,31 @@ export class MetaCompiler extends SectionCompiler { } public validate(): boolean { - // - return true; + let valid = true; + + const normalization = this.keyboard.info?.normalization; + if(normalization !== undefined) { + if(!isValidEnumValue(Meta_NormalizationForm, normalization)) { + this.callbacks.reportMessage(CompilerErrors.ERROR_InvalidNormalization, `Invalid normalization form '${normalization}'`); + valid = false; + } + } + + return valid; } public compile(): Meta { let result = new Meta(); - result.name = this.source.keyboard.names?.name?.[0]?.value; - result.author = this.source.keyboard.info?.author; - result.conform = this.source.keyboard.conformsTo; - result.layout = this.source.keyboard.info?.layout; - result.normalization = this.source.keyboard.info?.normalization; - result.indicator = this.source.keyboard.info?.indicator; - result.settings = 0; + result.name = this.keyboard.names?.name?.[0]?.value; + result.author = this.keyboard.info?.author; + result.conform = this.keyboard.conformsTo; + result.layout = this.keyboard.info?.layout; + result.normalization = this.keyboard.info?.normalization as Meta_NormalizationForm; + result.indicator = this.keyboard.info?.indicator; + result.settings = + (this.keyboard.settings?.fallback == "omit" ? KeyboardSettings.fallback : 0) | + (this.keyboard.settings?.transformFailure == "omit" ? KeyboardSettings.transformFailure : 0) | + (this.keyboard.settings?.transformPartial == "hide" ? KeyboardSettings.transformPartial : 0); return result; } } \ No newline at end of file diff --git a/developer/src/kmldmlc/src/keyman/compiler/section-compiler.ts b/developer/src/kmldmlc/src/keyman/compiler/section-compiler.ts index e09673003f..fb263c4dab 100644 --- a/developer/src/kmldmlc/src/keyman/compiler/section-compiler.ts +++ b/developer/src/kmldmlc/src/keyman/compiler/section-compiler.ts @@ -1,14 +1,14 @@ import { Section } from "../kmx/kmx-plus"; -import LDMLKeyboardXMLSourceFile from "../ldml-keyboard/ldml-keyboard-xml"; +import LDMLKeyboardXMLSourceFile, { LKKeyboard } from "../ldml-keyboard/ldml-keyboard-xml"; import CompilerCallbacks from "./callbacks"; import { SectionIdent } from '@keymanapp/ldml-keyboard-constants'; export class SectionCompiler { - protected readonly source: LDMLKeyboardXMLSourceFile; + protected readonly keyboard: LKKeyboard; protected readonly callbacks: CompilerCallbacks; constructor(source: LDMLKeyboardXMLSourceFile, callbacks: CompilerCallbacks) { - this.source = source; + this.keyboard = source.keyboard; this.callbacks = callbacks; } diff --git a/developer/src/kmldmlc/src/keyman/kmx/kmx-plus.ts b/developer/src/kmldmlc/src/keyman/kmx/kmx-plus.ts index 3b3f897359..dad532b651 100644 --- a/developer/src/kmldmlc/src/keyman/kmx/kmx-plus.ts +++ b/developer/src/kmldmlc/src/keyman/kmx/kmx-plus.ts @@ -16,12 +16,14 @@ export enum KeyboardSettings { transformPartial = 1<<2, }; +export enum Meta_NormalizationForm { NFC='NFC', NFD='NFD', other='other' }; + export class Meta extends Section { name: string; author: string; conform: string; layout: string; - normalization: string; + normalization: Meta_NormalizationForm; indicator: string; settings: KeyboardSettings; }; diff --git a/developer/src/kmldmlc/src/keyman/ldml-keyboard/ldml-keyboard-xml.ts b/developer/src/kmldmlc/src/keyman/ldml-keyboard/ldml-keyboard-xml.ts index 501bedcb77..df559466ac 100644 --- a/developer/src/kmldmlc/src/keyman/ldml-keyboard/ldml-keyboard-xml.ts +++ b/developer/src/kmldmlc/src/keyman/ldml-keyboard/ldml-keyboard-xml.ts @@ -20,6 +20,7 @@ export interface LKKeyboard { info?: LKInfo; names?: LKNames; + settings?: LKSettings; keys?: LKKeys; layerMaps?: LKLayerMaps[]; }; @@ -35,6 +36,12 @@ export interface LKNames { name: LKName[]; }; +export interface LKSettings { + fallback: "omit"; + transformFailure: "omit"; + transformPartial: "hide"; +}; + export interface LKName { value?: string; }; diff --git a/developer/src/kmldmlc/src/keyman/util/util.ts b/developer/src/kmldmlc/src/keyman/util/util.ts new file mode 100644 index 0000000000..d663fc490b --- /dev/null +++ b/developer/src/kmldmlc/src/keyman/util/util.ts @@ -0,0 +1,7 @@ + +/** + * Verifies that value is an item in the enumeration. + */ +export function isValidEnumValue(enu: T, value: string) { + return (Object.values(enu) as string[]).includes(value); +} diff --git a/developer/src/kmldmlc/src/kmldmlc.ts b/developer/src/kmldmlc/src/kmldmlc.ts index bffd397bfe..1aa8a0f377 100644 --- a/developer/src/kmldmlc/src/kmldmlc.ts +++ b/developer/src/kmldmlc/src/kmldmlc.ts @@ -9,6 +9,7 @@ import * as program from 'commander'; import Compiler from './keyman/compiler/compiler'; import KMXBuilder from './keyman/kmx/kmx-builder'; +import { getErrorSeverityName } from './keyman/compiler/errors'; let inputFilename: string; @@ -41,8 +42,8 @@ class CompilerCallbacks { // TODO: translate filename based on the baseFilename return fs.readFileSync(filename); } - reportMessage(severity: number, message: string): void { - console.log(message); + reportMessage(code: number, message: string): void { + console.log(getErrorSeverityName(code) + ' ' + code.toString(16) + ': ' + message); } } diff --git a/developer/src/kmldmlc/test/fixtures/sections/meta/invalid-normalization.xml b/developer/src/kmldmlc/test/fixtures/sections/meta/invalid-normalization.xml new file mode 100644 index 0000000000..d313be6f03 --- /dev/null +++ b/developer/src/kmldmlc/test/fixtures/sections/meta/invalid-normalization.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/developer/src/kmldmlc/test/fixtures/sections/meta/maximal.xml b/developer/src/kmldmlc/test/fixtures/sections/meta/maximal.xml new file mode 100644 index 0000000000..47caf09ba1 --- /dev/null +++ b/developer/src/kmldmlc/test/fixtures/sections/meta/maximal.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/developer/src/kmldmlc/test/fixtures/sections/meta/minimal.xml b/developer/src/kmldmlc/test/fixtures/sections/meta/minimal.xml new file mode 100644 index 0000000000..696a206db0 --- /dev/null +++ b/developer/src/kmldmlc/test/fixtures/sections/meta/minimal.xml @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/developer/src/kmldmlc/test/helpers/index.ts b/developer/src/kmldmlc/test/helpers/index.ts index 84ceb137bc..46c25d2e11 100644 --- a/developer/src/kmldmlc/test/helpers/index.ts +++ b/developer/src/kmldmlc/test/helpers/index.ts @@ -2,6 +2,10 @@ * Helpers and utilities for the Mocha tests. */ import * as path from 'path'; +import * as fs from 'fs'; +import { SectionCompiler } from '../../src/keyman/compiler/section-compiler'; +import { Section } from '../../src/keyman/kmx/kmx-plus'; +import LDMLKeyboardXMLSourceFileReader from '../../src/keyman/ldml-keyboard/ldml-keyboard-xml-reader'; /** * Builds a path to the fixture with the given path components. @@ -14,3 +18,24 @@ export function makePathToFixture(...components: string[]): string { return path.join(__dirname, '..', '..', '..', 'test', 'fixtures', ...components); } + +export class CompilerCallbacks { + messages: { code: number, message: string}[] = []; + loadFile(baseFilename: string, filename:string): Buffer { + // TODO: translate filename based on the baseFilename + return fs.readFileSync(filename); + } + reportMessage(code: number, message: string): void { + this.messages.push({code, message}); + } +} + +export function loadSectionFixture(compilerClass: typeof SectionCompiler, filename: string, callbacks: CompilerCallbacks): Section { + const inputFilename = makePathToFixture(filename); + const source = (new LDMLKeyboardXMLSourceFileReader(callbacks)).loadFile(inputFilename); + const compiler = new compilerClass(source, callbacks); + if(!compiler.validate()) { + return null; + } + return compiler.compile(); +} diff --git a/developer/src/kmldmlc/test/test-compiler-e2e.ts b/developer/src/kmldmlc/test/test-compiler-e2e.ts index ce8f4cba63..81280ff667 100644 --- a/developer/src/kmldmlc/test/test-compiler-e2e.ts +++ b/developer/src/kmldmlc/test/test-compiler-e2e.ts @@ -5,21 +5,11 @@ import {assert} from 'chai'; import hextobin from '@keymanapp/hextobin'; import Compiler from '../src/keyman/compiler/compiler'; import KMXBuilder from '../src/keyman/kmx/kmx-builder'; -import {makePathToFixture} from './helpers/index'; - -class CompilerCallbacks { - loadFile(baseFilename: string, filename:string): Buffer { - // TODO: translate filename based on the baseFilename - return fs.readFileSync(filename); - } - reportMessage(severity: number, message: string): void { - console.log(message); - } -} +import {CompilerCallbacks, makePathToFixture} from './helpers/index'; function compileKeyboard(inputFilename: string): Uint8Array { - const c = new CompilerCallbacks(); - const k = new Compiler(c); + const callbacks = new CompilerCallbacks(); + const k = new Compiler(callbacks); let source = k.load(inputFilename); if(!source) { return null; @@ -34,7 +24,9 @@ function compileKeyboard(inputFilename: string): Uint8Array { // Use the builder to generate the binary output file let builder = new KMXBuilder(kmx, true); - return builder.compile(); + let result = builder.compile(); + assert(callbacks.messages.length == 0); + return result; } describe('compiler-tests', function() { diff --git a/developer/src/kmldmlc/test/test-meta.ts b/developer/src/kmldmlc/test/test-meta.ts new file mode 100644 index 0000000000..c3b008162e --- /dev/null +++ b/developer/src/kmldmlc/test/test-meta.ts @@ -0,0 +1,45 @@ +import 'mocha'; +import {assert} from 'chai'; +import { MetaCompiler } from '../src/keyman/compiler/meta'; +import { CompilerCallbacks, loadSectionFixture } from './helpers'; +import { KeyboardSettings, Meta } from '../src/keyman/kmx/kmx-plus'; +import { CompilerErrors } from '../src/keyman/compiler/errors'; + +describe('meta', function () { + it('should compile minimal metadata', function() { + const callbacks = new CompilerCallbacks(); + let meta = loadSectionFixture(MetaCompiler, 'sections/meta/minimal.xml', callbacks) as Meta; + assert.equal(callbacks.messages.length, 0); + + assert.equal(meta.name, 'meta-minimal'); + assert.isUndefined(meta.author); // TODO: default author string "unknown"? + assert.equal(meta.conform, 'techpreview'); + assert.isUndefined(meta.layout); // TODO: assumed layout? + assert.isUndefined(meta.normalization); // TODO: assumed normalization? + assert.isUndefined(meta.indicator); // TODO: synthesize an indicator? + assert.equal(meta.settings, KeyboardSettings.none); + }); + + it('should compile maximal metadata', function() { + const callbacks = new CompilerCallbacks(); + let meta = loadSectionFixture(MetaCompiler, 'sections/meta/maximal.xml', callbacks) as Meta; + assert.equal(callbacks.messages.length, 0); + + assert.equal(meta.name, 'meta-maximal'); + assert.equal(meta.author, 'The Keyman Team'); + assert.equal(meta.conform, 'techpreview'); + assert.equal(meta.layout, 'QWIRKY'); + assert.equal(meta.normalization, 'NFC'); + assert.equal(meta.indicator, 'QW'); + assert.equal(meta.settings, KeyboardSettings.fallback | KeyboardSettings.transformFailure | KeyboardSettings.transformPartial); + }); + + it('should reject invalid normalization', function() { + const callbacks = new CompilerCallbacks(); + let meta = loadSectionFixture(MetaCompiler, 'sections/meta/invalid-normalization.xml', callbacks) as Meta; + assert.isNull(meta); + assert.equal(callbacks.messages.length, 1); + assert.deepEqual(callbacks.messages[0], {code: CompilerErrors.ERROR_InvalidNormalization, message: "Invalid normalization form 'NFQ'"}); + }) +}); + From 8fc7921ed451141bda656205cc12cdcc20567cd7 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Tue, 30 Aug 2022 13:57:16 +1000 Subject: [PATCH 2/2] chore(developer): cleanup todo comments --- developer/src/kmldmlc/test/test-meta.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/developer/src/kmldmlc/test/test-meta.ts b/developer/src/kmldmlc/test/test-meta.ts index c3b008162e..c56e1b25dc 100644 --- a/developer/src/kmldmlc/test/test-meta.ts +++ b/developer/src/kmldmlc/test/test-meta.ts @@ -12,11 +12,11 @@ describe('meta', function () { assert.equal(callbacks.messages.length, 0); assert.equal(meta.name, 'meta-minimal'); - assert.isUndefined(meta.author); // TODO: default author string "unknown"? + assert.isUndefined(meta.author); // TODO-LDML: default author string "unknown"? assert.equal(meta.conform, 'techpreview'); - assert.isUndefined(meta.layout); // TODO: assumed layout? - assert.isUndefined(meta.normalization); // TODO: assumed normalization? - assert.isUndefined(meta.indicator); // TODO: synthesize an indicator? + assert.isUndefined(meta.layout); // TODO-LDML: assumed layout? + assert.isUndefined(meta.normalization); // TODO-LDML: assumed normalization? + assert.isUndefined(meta.indicator); // TODO-LDML: synthesize an indicator? assert.equal(meta.settings, KeyboardSettings.none); });