From 9a31e866dfcdf0f58ceeab87f4e13d92e5f88d94 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 10 Feb 2023 12:49:28 -0600 Subject: [PATCH] =?UTF-8?q?feat(common):=20ldml=20read=20keyboard=20testda?= =?UTF-8?q?ta=20=F0=9F=99=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - early scaffolding - with some basic tests #7535 --- .../ldml-keyboard-testdata-xml.ts | 74 +++++++++++ .../ldml-keyboard/ldml-keyboard-xml-reader.ts | 118 +++++++++++++++++- common/web/types/src/util/common-events.ts | 5 + common/web/types/test/fixtures/test-fr.xml | 22 ++++ common/web/types/test/helpers/index.ts | 4 + .../test/helpers/reader-callback-test.ts | 82 +++++++++++- .../test-ldml-keyboard-testdata-reader.ts | 38 ++++++ 7 files changed, 340 insertions(+), 3 deletions(-) create mode 100644 common/web/types/src/ldml-keyboard/ldml-keyboard-testdata-xml.ts create mode 100644 common/web/types/test/fixtures/test-fr.xml create mode 100644 common/web/types/test/ldml-keyboard/test-ldml-keyboard-testdata-reader.ts diff --git a/common/web/types/src/ldml-keyboard/ldml-keyboard-testdata-xml.ts b/common/web/types/src/ldml-keyboard/ldml-keyboard-testdata-xml.ts new file mode 100644 index 0000000000..c0a5e9dd6d --- /dev/null +++ b/common/web/types/src/ldml-keyboard/ldml-keyboard-testdata-xml.ts @@ -0,0 +1,74 @@ +// +// Conforms to techpreview +// +// The interfaces in this file are designed with reference to the mapped +// structures produced by xml2js when passed a LDML keyboard test data .xml file. +// +// Using prefix LKT for LDML Keyboard Test +// + +export interface LDMLKeyboardTestDataXMLSourceFile { + /** + * -- the root element. + */ + keyboardTest: LKTKeyboardTest; +} + +export interface LKTKeyboardTest { + conformsTo?: string; + info?: LKTInfo; + repertoire?: LKTRepertoire[]; + tests?: LKTTests[]; +}; + +export interface LKTInfo { + author?: string; + keyboard?: string; + name?: string; +}; + +export interface LKTRepertoire { + name?: string; + chars?: string; + type?: string; +}; + +export interface LKTTests { + name?: string; + test?: LKTTest[]; +}; + +export interface LKTTest { + name?: string; + startContext?: LKTStartContext; + actions?: LKTAction[]; // differs from XML, to represent order of actions +}; + +export interface LKTStartContext { + to?: string; +}; + +export interface LKTCheck { + result?: string; +}; + +export interface LKTEmit { + to?: string; +}; + +export interface LKTKeystroke { + key?: string; + flick?: string; + longPress?: string; + tapCount?: string; +}; + +/** + * Test Actions. + * The expectation is that each LKTAction object will have exactly one non-falsy field. + */ +export interface LKTAction { + check?: LKTCheck; + emit?: LKTEmit; + keystroke?: LKTKeystroke; +}; diff --git a/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts b/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts index cec46a602a..fde66f52a0 100644 --- a/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -5,6 +5,13 @@ import { boxXmlArray } from '../util/util.js'; import { CompilerCallbacks } from '../util/compiler-interfaces.js'; import { constants } from '@keymanapp/ldml-keyboard-constants'; import { CommonTypesMessages } from '../util/common-events.js'; +import { LDMLKeyboardTestDataXMLSourceFile } from './ldml-keyboard-testdata-xml.js'; + +interface NameAndProps { + '$'?: any; // content + '#name'?: string; // element name + '$$'?: any; // children +}; export default class LDMLKeyboardXMLSourceFileReader { callbacks: CompilerCallbacks; @@ -186,7 +193,7 @@ export default class LDMLKeyboardXMLSourceFileReader { /** * @returns true if valid, false if invalid */ - public validate(source: LDMLKeyboardXMLSourceFile, schemaSource: Buffer): boolean { + public validate(source: LDMLKeyboardXMLSourceFile | LDMLKeyboardTestDataXMLSourceFile, schemaSource: Buffer): boolean { const schema = JSON.parse(schemaSource.toString('utf8')); const ajv = new Ajv(); if(!ajv.validate(schema, source)) { @@ -220,7 +227,7 @@ export default class LDMLKeyboardXMLSourceFileReader { // An alternative fix would be to pull xml2js directly from github // rather than using the version tagged on npmjs.com. }); - parser.parseString(file, (e: unknown, r: unknown) => { a = r as LDMLKeyboardXMLSourceFile }); + parser.parseString(file, (e: unknown, r: unknown) => { a = r as LDMLKeyboardXMLSourceFile }); // TODO-LDML: isn't 'e' the error? return a; })(); return source; @@ -241,4 +248,111 @@ export default class LDMLKeyboardXMLSourceFileReader { return null; } } + + loadTestDataUnboxed(file: Uint8Array): any { + let source = (() => { + let a: any; + let parser = new xml2js.Parser({ + // explicitArray: false, + preserveChildrenOrder:true, // needed for test data + explicitChildren: true, // needed for test data + // mergeAttrs: true, + // includeWhiteChars: false, + // emptyTag: {} as any + // Why "as any"? xml2js is broken: + // https://github.com/Leonidas-from-XIV/node-xml2js/issues/648 means + // that an old version of `emptyTag` is used which doesn't support + // functions, but DefinitelyTyped is requiring use of function or a + // string. See also notes at + // https://github.com/DefinitelyTyped/DefinitelyTyped/pull/59259#issuecomment-1254405470 + // An alternative fix would be to pull xml2js directly from github + // rather than using the version tagged on npmjs.com. + }); + parser.parseString(file, (e: unknown, r: unknown) => { a = r as any }); // TODO-LDML: isn't 'e' the error? + return a; // Why 'any'? Because we need to box up the $'s into proper properties. + })(); + return source; + } + + /** + * Filter the obj array for a subtag + * @param source array of source objs + * @param subtag subtag to filter on + * @returns + */ + findSubtagArray(source: NameAndProps[], subtag: string): NameAndProps[] { + return source?.filter(o => o['#name'] === subtag); + } + + /** + * Get exactly one element + * @param source + * @param subtag + * @returns + */ + findSubtag(source: NameAndProps[], subtag: string): NameAndProps | null { + const r = this.findSubtagArray(source, subtag); + if (!r || r.length === 0) { + return null; + } else if (r.length === 1) { + return r[0]; + } else { + this.callbacks.reportMessage(CommonTypesMessages.Error_TestDataUnexpectedArray({subtag})); + return null; // ERROR + } + } + + /** + * The default test data stuffer. + * Just gets $ (the attrs) as the body. + * Override to use something more complex, such as including child nodes. + */ + static readonly defaultMapper = ((o : NameAndProps) => o?.$); + + /** + * + * @param obj target object + * @param source array of $/#name strings + * @param subtag name to extract + * @param mapper custom mapper function + */ + stuffBoxes(obj: any, source: NameAndProps[], subtag: string, asArray?: boolean, mapper?: (v: NameAndProps) => any) { + if (!mapper) { + mapper = LDMLKeyboardXMLSourceFileReader.defaultMapper; + } + if (asArray) { + obj[subtag] = this.findSubtagArray(source, subtag)?.map(mapper); // extract contents only + } else { + obj[subtag] = mapper(this.findSubtag(source, subtag)); // run the mapper once + } + } + + boxTestDataArrays(raw: any) : LDMLKeyboardTestDataXMLSourceFile | null { + if (!raw) return null; + const a : LDMLKeyboardTestDataXMLSourceFile = { + keyboardTest: { + conformsTo: raw?.keyboardTest?.$?.conformsTo, + } + }; + + const $$ = raw?.keyboardTest?.$$; + this.stuffBoxes(a.keyboardTest, $$, 'info'); + this.stuffBoxes(a.keyboardTest, $$, 'repertoire', true); + + // TODO-LDML: all the things + + return a; + } + + /** + * @param file test file + * @returns source on success, otherwise null + */ + public loadTestData(file: Uint8Array): LDMLKeyboardTestDataXMLSourceFile | null { + if (!file) { + return null; + } + const source = this.loadTestDataUnboxed(file); + return this.boxTestDataArrays(source); + } } diff --git a/common/web/types/src/util/common-events.ts b/common/web/types/src/util/common-events.ts index 02319afee9..65743d686d 100644 --- a/common/web/types/src/util/common-events.ts +++ b/common/web/types/src/util/common-events.ts @@ -38,4 +38,9 @@ export class CommonTypesMessages { m(this.ERROR_ImportMergeFail, `Problem importing ${o.path}: not sure how to handle non-array ${o.subtag}.${o.subsubtag}`); static ERROR_ImportMergeFail = SevError | 0x0006; + + static Error_TestDataUnexpectedArray = (o: {subtag: string}) => + m(this.ERROR_TestDataUnexpectedArray, + `Problem reading test data: expected single ${o.subtag} element, found multiple`); + static ERROR_TestDataUnexpectedArray = SevError | 0x0007; }; diff --git a/common/web/types/test/fixtures/test-fr.xml b/common/web/types/test/fixtures/test-fr.xml new file mode 100644 index 0000000000..c97825385a --- /dev/null +++ b/common/web/types/test/fixtures/test-fr.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/common/web/types/test/helpers/index.ts b/common/web/types/test/helpers/index.ts index 3bc4e743aa..0a11470db9 100644 --- a/common/web/types/test/helpers/index.ts +++ b/common/web/types/test/helpers/index.ts @@ -29,3 +29,7 @@ export function loadFile(baseFilename: string, filename: string | URL): Buffer { export function loadLdmlKeyboardSchema(): Buffer { return fs.readFileSync(new URL(path.join('..', '..', 'src', 'ldml-keyboard.schema.json'), import.meta.url)); } + +export function loadLdmlKeyboardTestDataSchema(): Buffer { + return fs.readFileSync(new URL(path.join('..', '..', 'src', 'ldml-keyboardtest.schema.json'), import.meta.url)); +} diff --git a/common/web/types/test/helpers/reader-callback-test.ts b/common/web/types/test/helpers/reader-callback-test.ts index 73198f26f7..cf415832e4 100644 --- a/common/web/types/test/helpers/reader-callback-test.ts +++ b/common/web/types/test/helpers/reader-callback-test.ts @@ -1,9 +1,10 @@ import 'mocha'; import {assert} from 'chai'; -import {loadLdmlKeyboardSchema, loadFile, makePathToFixture} from '../helpers/index.js'; +import {loadLdmlKeyboardSchema, loadFile, makePathToFixture, loadLdmlKeyboardTestDataSchema} from '../helpers/index.js'; import LDMLKeyboardXMLSourceFileReader from '../../src/ldml-keyboard/ldml-keyboard-xml-reader.js'; import { CompilerCallbacks, CompilerEvent } from '../../src/util/compiler-interfaces.js'; import { LDMLKeyboardXMLSourceFile } from '../../src/ldml-keyboard/ldml-keyboard-xml.js'; +import { LDMLKeyboardTestDataXMLSourceFile } from '../ldml-keyboard/ldml-keyboard-testdata-xml.js'; // TODO-LDML: this is largely a port from developer/src/kmc-keyboard/test/helpers/index.ts @@ -66,6 +67,34 @@ export interface CompilationCase { throws?: RegExp; } + +export interface TestDataCase { + /** + * If true, loading (validation) should fail + */ + loadfail?: boolean; + /** + * path to xml, such as 'sections/layr/invalid-case.xml' + */ + subpath: string; + /** + * expected error messages. If falsy, expected to succeed. All must be present to pass. + */ + errors?: CompilerEvent[]; + /** + * expected warning messages. All must be present to pass. + */ + warnings?: CompilerEvent[]; + /** + * optional callback with the section + */ + callback?: (data: Buffer, source: LDMLKeyboardTestDataXMLSourceFile, subpath: string, callbacks: TestCompilerCallbacks ) => void; + /** + * if present, expect compiler to throw (use .* to match all) + */ + throws?: RegExp; +} + /** * Run a bunch of cases * @param cases cases to run @@ -115,3 +144,54 @@ export function testReaderCases(cases : CompilationCase[]) { }); } } + + +/** + * Run a bunch of cases + * @param cases cases to run + * @param compiler argument to loadSectionFixture() + * @param callbacks argument to loadSectionFixture() + */ +export function testTestdataReaderCases(cases : TestDataCase[]) { + // we need our own callbacks rather than using the global so messages don't get mixed + const callbacks = new TestCompilerCallbacks(); + const reader = new LDMLKeyboardXMLSourceFileReader(callbacks); + for (let testcase of cases) { + const expectFailure = testcase.throws || !!(testcase.errors); // if true, we expect this to fail + const testHeading = expectFailure ? `should fail to load: ${testcase.subpath}`: + `should load: ${testcase.subpath}`; + it(testHeading, function () { + callbacks.clear(); + + const data = loadFile(testcase.subpath, makePathToFixture(testcase.subpath)); + assert.ok(data, `reading ${testcase.subpath}`); + const source = reader.loadTestData(data); + if (!testcase.loadfail) { + assert.ok(source, `loading ${testcase.subpath}`); + } else { + assert.notOk(source, `loading ${testcase.subpath} (expected failure)`); + } + // special case for an expected exception + if (testcase.throws) { + assert.throws(() => reader.validate(source, loadLdmlKeyboardTestDataSchema()), testcase.throws); + } else { + assert.doesNotThrow(() => reader.validate(source, loadLdmlKeyboardTestDataSchema()), `validating ${testcase.subpath}`); + // if we expected errors or warnings, show them + if (testcase.errors) { + assert.includeDeepMembers(callbacks.messages, testcase.errors, 'expected errors to be included'); + } + if (testcase.warnings) { + assert.includeDeepMembers(callbacks.messages, testcase.warnings, 'expected warnings to be included'); + } else if (!expectFailure) { + // no warnings, so expect zero messages + assert.strictEqual(callbacks.messages.length, 0, 'expected zero messages'); + } + + // run the user-supplied callback if any + if (testcase.callback) { + testcase.callback(data, source, testcase.subpath, callbacks); + } + } + }); + } +} diff --git a/common/web/types/test/ldml-keyboard/test-ldml-keyboard-testdata-reader.ts b/common/web/types/test/ldml-keyboard/test-ldml-keyboard-testdata-reader.ts new file mode 100644 index 0000000000..55280648e1 --- /dev/null +++ b/common/web/types/test/ldml-keyboard/test-ldml-keyboard-testdata-reader.ts @@ -0,0 +1,38 @@ +import { constants } from '@keymanapp/ldml-keyboard-constants'; +import { assert } from 'chai'; +import 'mocha'; +import { testTestdataReaderCases } from '../helpers/reader-callback-test.js'; + +describe('ldml keyboard xml reader tests', function () { + this.slow(500); // 0.5 sec -- json schema validation takes a while + + testTestdataReaderCases([ + { + subpath: 'test-fr.xml', + callback: (data, source) => { + // TODO-LDML: for dev, dump it out + console.dir({source}, {depth: Infinity}); + assert.ok(source); + assert.ok(source.keyboardTest); + assert.equal(source.keyboardTest.conformsTo, constants.cldr_version_latest); + + assert.deepEqual(source.keyboardTest.info, { + keyboard: 'fr-t-k0-azerty.xml', + author: 'Team Keyboard', + name: 'fr-test' + }); + + assert.sameDeepMembers(source.keyboardTest.repertoire, [ + { + name: 'simple-repertoire', + chars: '[a b c d e \\u{22}]', + type: 'simple' + }, + { name: 'chars-repertoire', chars: '[á é ó]', type: 'gesture' } + ]); + + // TODO-LDML: all the things + }, + }, + ]); +});