From 6bebefda10bf3fd1a22b42f301a45d4036cdc172 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Thu, 7 Dec 2023 11:05:31 +0700 Subject: [PATCH 01/14] feat(developer): move kmc-kmn to KeymanCompiler interface Part of #9473. KmnCompiler now implements KeymanCompiler, including the returned artifacts. Also establishes the common types for the compiler interfaces and consolidates and renames various API surfaces for kmc-kmn. --- common/web/types/src/main.ts | 7 ++ .../web/types/src/util/compiler-interfaces.ts | 39 +++++++- .../src/util/get-osk-from-kmn-file.ts | 12 +-- .../src/kmc-kmn/src/compiler/compiler.ts | 90 ++++++++++--------- .../src/kmw-compiler/compiler-globals.ts | 6 +- .../kmc-kmn/src/kmw-compiler/kmw-compiler.ts | 4 +- .../src/kmc-kmn/test/kmw/test-kmw-compiler.ts | 17 ++-- developer/src/kmc-kmn/test/test-compiler.ts | 45 +++++++--- developer/src/kmc-kmn/test/test-features.ts | 8 +- developer/src/kmc-kmn/test/test-messages.ts | 4 +- developer/src/kmc-kmn/test/test-wasm-uset.ts | 10 +-- .../src/kmc-ldml/src/compiler/compiler.ts | 2 +- .../commands/buildClasses/BuildKmnKeyboard.ts | 27 +++--- .../src/kmc/src/util/TestKeymanSentry.ts | 2 +- 14 files changed, 172 insertions(+), 101 deletions(-) diff --git a/common/web/types/src/main.ts b/common/web/types/src/main.ts index 06671a13d7..4a1afc1bf9 100644 --- a/common/web/types/src/main.ts +++ b/common/web/types/src/main.ts @@ -27,6 +27,13 @@ export { defaultCompilerOptions, CompilerBaseOptions, CompilerCallbacks, Compile compilerExceptionToString, compilerErrorFormatCode, compilerLogLevelToSeverity, CompilerLogLevel, compilerEventFormat, ALL_COMPILER_LOG_LEVELS, ALL_COMPILER_LOG_FORMATS, CompilerLogFormat, + + KeymanCompilerArtifact, + KeymanCompilerArtifactOptional, + KeymanCompilerArtifacts, + KeymanCompilerResult, + KeymanCompiler + } from './util/compiler-interfaces.js'; export { CommonTypesMessages } from './util/common-events.js'; diff --git a/common/web/types/src/util/compiler-interfaces.ts b/common/web/types/src/util/compiler-interfaces.ts index b27a170ab3..fc6a33fa1a 100644 --- a/common/web/types/src/util/compiler-interfaces.ts +++ b/common/web/types/src/util/compiler-interfaces.ts @@ -253,6 +253,40 @@ export interface CompilerCallbackOptions { compilerWarningsAsErrors?: boolean; }; +export interface KeymanCompilerArtifact { + data: Uint8Array; + filename: string; +}; + +export type KeymanCompilerArtifactOptional = KeymanCompilerArtifact | undefined; + +export interface KeymanCompilerArtifacts { + readonly [type:string]: KeymanCompilerArtifactOptional; +}; + +export interface KeymanCompilerResult { + artifacts: KeymanCompilerArtifacts; +}; + +export interface KeymanCompiler { + init(callbacks: CompilerCallbacks, options: CompilerOptions): Promise; + /** + * Run the compiler, and save the result in memory arrays. Note that while + * `outputFilename` is provided here, the output file is not written to in + * this function. + * @param inputFilename + * @param outputFilename The intended output filename, optional, if missing, + * calculated from inputFilename + * @param data + */ + run(inputFilename:string, outputFilename?:string /*, data?: any*/): Promise; + /** + * Writes the compiled output files to disk + * @param artifacts + */ + write(artifacts: KeymanCompilerArtifacts): Promise; +}; + /** * Abstract interface for callbacks, to abstract out file i/o */ @@ -369,10 +403,7 @@ export interface CompilerBaseOptions { * Format of output for log to console */ logFormat?: CompilerLogFormat; - /** - * Optional output file for activities that generate output - */ - outFile?: string; + outFile?: string; //TODO:REMOVE /** * Colorize log output, default is detected from console */ diff --git a/developer/src/kmc-analyze/src/util/get-osk-from-kmn-file.ts b/developer/src/kmc-analyze/src/util/get-osk-from-kmn-file.ts index 2df34e9dae..0e2ec28995 100644 --- a/developer/src/kmc-analyze/src/util/get-osk-from-kmn-file.ts +++ b/developer/src/kmc-analyze/src/util/get-osk-from-kmn-file.ts @@ -9,15 +9,15 @@ export async function getOskFromKmnFile(callbacks: CompilerCallbacks, filename: let touchLayoutFilename: string; const kmnCompiler = new KmnCompiler(); - if(!await kmnCompiler.init(callbacks)) { + if(!await kmnCompiler.init(callbacks, { + shouldAddCompilerVersion: false, + saveDebug: false, + })) { // kmnCompiler will report errors return null; } - let result = kmnCompiler.runCompiler(filename, { - shouldAddCompilerVersion: false, - saveDebug: false, - }); + let result = await kmnCompiler.run(filename, null); if(!result) { // kmnCompiler will report any errors @@ -29,7 +29,7 @@ export async function getOskFromKmnFile(callbacks: CompilerCallbacks, filename: } const reader = new KmxFileReader(); - const keyboard: KMX.KEYBOARD = reader.read(result.kmx.data); + const keyboard: KMX.KEYBOARD = reader.read(result.artifacts.kmx.data); const touchLayoutStore = keyboard.stores.find(store => store.dwSystemID == KMX.KMXFile.TSS_LAYOUTFILE); if(touchLayoutStore) { diff --git a/developer/src/kmc-kmn/src/compiler/compiler.ts b/developer/src/kmc-kmn/src/compiler/compiler.ts index 317cfc757c..a4033ad89c 100644 --- a/developer/src/kmc-kmn/src/compiler/compiler.ts +++ b/developer/src/kmc-kmn/src/compiler/compiler.ts @@ -6,17 +6,12 @@ TODO: implement additional interfaces: */ // TODO: rename wasm-host? -import { UnicodeSetParser, UnicodeSet, Osk, VisualKeyboard, KvkFileReader } from '@keymanapp/common-types'; +import { UnicodeSetParser, UnicodeSet, Osk, VisualKeyboard, KvkFileReader, KeymanCompiler, KeymanCompilerArtifacts, KeymanCompilerArtifactOptional, KeymanCompilerResult, KeymanCompilerArtifact } from '@keymanapp/common-types'; import { CompilerCallbacks, CompilerEvent, CompilerOptions, KeymanFileTypes, KvkFileWriter, KvksFileReader } from '@keymanapp/common-types'; import loadWasmHost from '../import/kmcmplib/wasm-host.js'; import { CompilerMessages, mapErrorFromKmcmplib } from './kmn-compiler-messages.js'; import { WriteCompiledKeyboard } from '../kmw-compiler/kmw-compiler.js'; -export interface CompilerResultFile { - filename: string; - data: Uint8Array; -}; - // // Matches kmcmplibapi.h definitions // @@ -46,7 +41,7 @@ export const COMPILETARGETS__MASK = 0x03; /** * Data in CompilerResultExtra comes from kmcmplib */ -export interface CompilerResultExtra { +export interface KmnCompilerResultExtra { /** * A bitmask, consisting of COMPILETARGETS_KMX and/or COMPILETARGETS_JS */ @@ -61,11 +56,15 @@ export interface CompilerResultExtra { // Internal in-memory result from a successful compilation // -export interface CompilerResult { - kmx?: CompilerResultFile; - kvk?: CompilerResultFile; - js?: CompilerResultFile; - extra: CompilerResultExtra; +export interface KmnCompilerArtifacts extends KeymanCompilerArtifacts { + kmx?: KeymanCompilerArtifactOptional; + kvk?: KeymanCompilerArtifactOptional; + js?: KeymanCompilerArtifactOptional; +}; + +export interface KmnCompilerResult extends KeymanCompilerResult { + artifacts: KmnCompilerArtifacts; + extra: KmnCompilerResultExtra; displayMap?: Osk.PuaMap; }; @@ -97,18 +96,20 @@ interface MallocAndFree { let Module: any; -export class KmnCompiler implements UnicodeSetParser { +export class KmnCompiler implements KeymanCompiler, UnicodeSetParser { callbackID: string; // a unique numeric id added to globals with prefixed names callbacks: CompilerCallbacks; wasmExports: MallocAndFree; + options: KmnCompilerOptions; constructor() { this.callbackID = callbackPrefix + callbackProcIdentifier.toString(); callbackProcIdentifier++; } - public async init(callbacks: CompilerCallbacks): Promise { + public async init(callbacks: CompilerCallbacks, options: KmnCompilerOptions): Promise { this.callbacks = callbacks; + this.options = options; if(!Module) { try { Module = await loadWasmHost(); @@ -140,20 +141,22 @@ export class KmnCompiler implements UnicodeSetParser { return true; } - public run(infile: string, options?: KmnCompilerOptions): boolean { - let result = this.runCompiler(infile, options); - if(result) { - if(result.kmx) { - this.callbacks.fs.writeFileSync(result.kmx.filename, result.kmx.data); - } - if(result.kvk) { - this.callbacks.fs.writeFileSync(result.kvk.filename, result.kvk.data); - } - if(result.js) { - this.callbacks.fs.writeFileSync(result.js.filename, result.js.data); - } + public async write(artifacts: KmnCompilerArtifacts): Promise { + if(!artifacts) { + throw Error('artifacts must be defined'); } - return !!result; + + if(artifacts.kmx) { + this.callbacks.fs.writeFileSync(artifacts.kmx.filename, artifacts.kmx.data); + } + if(artifacts.kvk) { + this.callbacks.fs.writeFileSync(artifacts.kvk.filename, artifacts.kvk.data); + } + if(artifacts.js) { + this.callbacks.fs.writeFileSync(artifacts.js.filename, artifacts.js.data); + } + + return true; } private compilerMessageCallback = (line: number, code: number, msg: string): number => { @@ -196,9 +199,10 @@ export class KmnCompiler implements UnicodeSetParser { return 1; } - private copyWasmResult(wasm_result: any): CompilerResult { - let result: CompilerResult = { + private copyWasmResult(wasm_result: any): KmnCompilerResult { + let result: KmnCompilerResult = { // We cannot Object.assign or {...} on a wasm-defined object, so... + artifacts: {}, extra: { targets: wasm_result.extra.targets, displayMapFilename: wasm_result.extra.displayMapFilename, @@ -234,15 +238,15 @@ export class KmnCompiler implements UnicodeSetParser { return new Uint8Array(new Uint8Array(Module.HEAP8.buffer, offset, size)); } - public runCompiler(infile: string, options: KmnCompilerOptions): CompilerResult { + public async run(infile: string, outfile: string): Promise { if(!this.verifyInitialized()) { /* c8 ignore next 2 */ return null; } - options = {...baseOptions, ...options}; + const options = {...baseOptions, ...this.options}; - options.outFile = options.outFile ?? infile.replace(/\.kmn$/i, '.kmx'); + outfile = outfile ?? infile.replace(/\.kmn$/i, '.kmx'); (globalThis as any)[this.callbackID] = { message: this.compilerMessageCallback, @@ -264,11 +268,11 @@ export class KmnCompiler implements UnicodeSetParser { return null; } - const result: CompilerResult = this.copyWasmResult(wasm_result); + const result: KmnCompilerResult = this.copyWasmResult(wasm_result); if(result.extra.targets & COMPILETARGETS_KMX) { - result.kmx = { - filename: options.outFile, + result.artifacts.kmx = { + filename: outfile, data: this.copyWasmBuffer(wasm_result.kmx, wasm_result.kmxSize) }; } @@ -286,8 +290,8 @@ export class KmnCompiler implements UnicodeSetParser { } if(result.extra.kvksFilename) { - result.kvk = this.runKvkCompiler(result.extra.kvksFilename, infile, options.outFile, result.displayMap); - if(!result.kvk) { + result.artifacts.kvk = this.runKvkCompiler(result.extra.kvksFilename, infile, outfile, result.displayMap); + if(!result.artifacts.kvk) { return null; } } @@ -308,12 +312,12 @@ export class KmnCompiler implements UnicodeSetParser { if(!wasm_result.result) { return null; } - const kmw_result: CompilerResult = this.copyWasmResult(wasm_result); + const kmw_result: KmnCompilerResult = this.copyWasmResult(wasm_result); kmw_result.displayMap = result.displayMap; // we can safely re-use the kmx compile displayMap const web_kmx = this.copyWasmBuffer(wasm_result.kmx, wasm_result.kmxSize); - result.js = this.runWebCompiler(infile, options.outFile, web_kmx, result.kvk?.data, kmw_result, options); - if(!result.js) { + result.artifacts.js = this.runWebCompiler(infile, outfile, web_kmx, result.artifacts.kvk?.data, kmw_result, options); + if(!result.artifacts.js) { return null; } } @@ -338,9 +342,9 @@ export class KmnCompiler implements UnicodeSetParser { kmxFilename: string, web_kmx: Uint8Array, kvk: Uint8Array, - kmxResult: CompilerResult, + kmxResult: KmnCompilerResult, options: CompilerOptions - ): CompilerResultFile { + ): KeymanCompilerArtifact { const data = WriteCompiledKeyboard(this.callbacks, kmnFilename, web_kmx, kvk, kmxResult, options.saveDebug); if(!data) { return null; @@ -348,7 +352,7 @@ export class KmnCompiler implements UnicodeSetParser { return { filename: this.callbacks.path.join(this.callbacks.path.dirname(kmxFilename), - this.keyboardIdFromKmnFilename(kmnFilename) + KeymanFileTypes.Binary.WebKeyboard), + this.keyboardIdFromKmnFilename(kmnFilename) + KeymanFileTypes.Binary.WebKeyboard), data: new TextEncoder().encode(data) }; } diff --git a/developer/src/kmc-kmn/src/kmw-compiler/compiler-globals.ts b/developer/src/kmc-kmn/src/kmw-compiler/compiler-globals.ts index 92cba914c5..3b09a18f47 100644 --- a/developer/src/kmc-kmn/src/kmw-compiler/compiler-globals.ts +++ b/developer/src/kmc-kmn/src/kmw-compiler/compiler-globals.ts @@ -1,10 +1,10 @@ import { KMX, CompilerCallbacks, CompilerOptions } from "@keymanapp/common-types"; -import { CompilerResult } from "../compiler/compiler.js"; +import { KmnCompilerResult } from "../compiler/compiler.js"; export let FTabStop: string; export let nl: string; export let FCompilerWarningsAsErrors = false; -export let kmxResult: CompilerResult; +export let kmxResult: KmnCompilerResult; export let fk: KMX.KEYBOARD; export let FMnemonic: boolean; export let options: CompilerOptions; @@ -19,7 +19,7 @@ export function setupGlobals( _options: CompilerOptions, _tab: string, _nl: string, - _kmxResult: CompilerResult, + _kmxResult: KmnCompilerResult, _keyboard: KMX.KEYBOARD, _kmnfile: string ) { diff --git a/developer/src/kmc-kmn/src/kmw-compiler/kmw-compiler.ts b/developer/src/kmc-kmn/src/kmw-compiler/kmw-compiler.ts index dd891cd7cb..974e4fd485 100644 --- a/developer/src/kmc-kmn/src/kmw-compiler/kmw-compiler.ts +++ b/developer/src/kmc-kmn/src/kmw-compiler/kmw-compiler.ts @@ -5,7 +5,7 @@ import { JavaScript_ContextMatch, JavaScript_KeyAsString, JavaScript_Name, JavaS import { KmwCompilerMessages } from "./kmw-compiler-messages.js"; import { ValidateLayoutFile } from "./validate-layout-file.js"; import { VisualKeyboardFromFile } from "./visual-keyboard-compiler.js"; -import { CompilerResult, STORETYPE_DEBUG, STORETYPE_OPTION, STORETYPE_RESERVED } from "../compiler/compiler.js"; +import { KmnCompilerResult, STORETYPE_DEBUG, STORETYPE_OPTION, STORETYPE_RESERVED } from "../compiler/compiler.js"; function requote(s: string): string { return "'" + s.replaceAll(/(['\\])/g, "\\$1") + "'"; @@ -41,7 +41,7 @@ export function WriteCompiledKeyboard( kmnfile: string, keyboardData: Uint8Array, kvkData: Uint8Array, - kmxResult: CompilerResult, + kmxResult: KmnCompilerResult, FDebug: boolean = false ): string { let opts: CompilerOptions = { diff --git a/developer/src/kmc-kmn/test/kmw/test-kmw-compiler.ts b/developer/src/kmc-kmn/test/kmw/test-kmw-compiler.ts index a7eecda019..6732351576 100644 --- a/developer/src/kmc-kmn/test/kmw/test-kmw-compiler.ts +++ b/developer/src/kmc-kmn/test/kmw/test-kmw-compiler.ts @@ -5,7 +5,7 @@ import { dirname } from 'path'; import { fileURLToPath } from 'url'; import fs from 'fs'; import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers'; -import { CompilerResult, KmnCompiler } from '../../src/compiler/compiler.js'; +import { KmnCompilerResult, KmnCompiler } from '../../src/compiler/compiler.js'; import { ETLResult, extractTouchLayout as parseWebTestResult } from './util.js'; import { KeymanFileTypes } from '@keymanapp/common-types'; @@ -27,7 +27,10 @@ describe('KeymanWeb Compiler', function() { const kmnCompiler: KmnCompiler = new KmnCompiler(); this.beforeAll(async function() { - assert.isTrue(await kmnCompiler.init(callbacks)); + assert.isTrue(await kmnCompiler.init(callbacks, { + shouldAddCompilerVersion: false, + saveDebug: true, + })); }); this.afterEach(function() { @@ -79,18 +82,16 @@ describe('KeymanWeb Compiler', function() { }); -function run_test_keyboard(kmnCompiler: KmnCompiler, id: string): { result: CompilerResult, actualCode: string, actual: ETLResult, expectedCode: string, expected: ETLResult } { +async function run_test_keyboard(kmnCompiler: KmnCompiler, id: string): + Promise<{ result: KmnCompilerResult, actualCode: string, actual: ETLResult, expectedCode: string, expected: ETLResult }> { const filenames = generateTestFilenames(id); - let result = kmnCompiler.runCompiler(filenames.source, { - shouldAddCompilerVersion: false, - saveDebug: true, - }); + let result = await kmnCompiler.run(filenames.source, null); assert.isNotNull(result); let value = { result, - actualCode: new TextDecoder().decode(result.js.data), + actualCode: new TextDecoder().decode(result.artifacts.js.data), expectedCode: fs.readFileSync(filenames.fixture, 'utf8'), expected: null, actual: null, diff --git a/developer/src/kmc-kmn/test/test-compiler.ts b/developer/src/kmc-kmn/test/test-compiler.ts index 0b776266ce..d103625e9b 100644 --- a/developer/src/kmc-kmn/test/test-compiler.ts +++ b/developer/src/kmc-kmn/test/test-compiler.ts @@ -15,7 +15,7 @@ describe('Compiler class', function() { const compiler = new KmnCompiler(); const callbacks : any = null; // ERROR try { - await compiler.init(callbacks) + await compiler.init(callbacks, null) assert.fail('Expected exception'); } catch(e) { assert.ok(e); @@ -26,21 +26,27 @@ describe('Compiler class', function() { it('should start', async function() { const compiler = new KmnCompiler(); const callbacks = new TestCompilerCallbacks(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, null)); assert(compiler.verifyInitialized()); }); it('should compile a basic keyboard', async function() { const compiler = new KmnCompiler(); const callbacks = new TestCompilerCallbacks(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, {saveDebug: true, shouldAddCompilerVersion: false})); assert(compiler.verifyInitialized()); const fixtureName = baselineDir + 'k_000___null_keyboard.kmx'; const infile = baselineDir + 'k_000___null_keyboard.kmn'; const outFile = __dirname + '/k_000___null_keyboard.kmx'; - assert(compiler.run(infile, {saveDebug: true, outFile, shouldAddCompilerVersion: false})); + if(fs.existsSync(outFile)) { + fs.rmSync(outFile); + } + + const result = await compiler.run(infile, outFile); + assert.isNotNull(result); + assert.isTrue(await compiler.write(result.artifacts)); assert(fs.existsSync(outFile)); const outfileData = fs.readFileSync(outFile); @@ -54,7 +60,7 @@ describe('Compiler class', function() { it('should build all baseline fixtures', async function() { const compiler = new KmnCompiler(); const callbacks = new TestCompilerCallbacks(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, {saveDebug: true, shouldAddCompilerVersion: false})); assert(compiler.verifyInitialized()); const files = fs.readdirSync(baselineDir); @@ -64,7 +70,13 @@ describe('Compiler class', function() { const infile = baselineDir + file.replace(/x$/, 'n'); const outFile = __dirname + '/' + file; - assert(compiler.run(infile, {saveDebug: true, outFile, shouldAddCompilerVersion: false})); + if(fs.existsSync(outFile)) { + fs.rmSync(outFile); + } + + const result = await compiler.run(infile, outFile); + assert.isNotNull(result); + assert.isTrue(await compiler.write(result.artifacts)); assert(fs.existsSync(outFile)); const outfileData = fs.readFileSync(outFile); @@ -78,7 +90,10 @@ describe('Compiler class', function() { it('should compile a keyboard with visual keyboard', async function() { const compiler = new KmnCompiler(); const callbacks = new TestCompilerCallbacks(); - assert.isTrue(await compiler.init(callbacks)); + assert.isTrue(await compiler.init(callbacks, { + saveDebug: true, + shouldAddCompilerVersion: false, + })); assert.isTrue(compiler.verifyInitialized()); const fixtureDir = keyboardsDir + 'caps_lock_layer_3620/' @@ -89,11 +104,17 @@ describe('Compiler class', function() { const resultingKmxfile = __dirname + '/caps_lock_layer_3620.kmx'; const resultingKvkfile = __dirname + '/caps_lock_layer_3620.kvk'; - assert.isTrue(compiler.run(infile, { - saveDebug: true, - shouldAddCompilerVersion: false, - outFile: resultingKmxfile, - })); + if(fs.existsSync(resultingKmxfile)) { + fs.rmSync(resultingKmxfile); + } + + if(fs.existsSync(resultingKvkfile)) { + fs.rmSync(resultingKvkfile); + } + + const result = await compiler.run(infile, resultingKmxfile); + assert.isNotNull(result); + assert.isTrue(await compiler.write(result.artifacts)); assert.isTrue(fs.existsSync(resultingKmxfile)); assert.isTrue(fs.existsSync(resultingKvkfile)); diff --git a/developer/src/kmc-kmn/test/test-features.ts b/developer/src/kmc-kmn/test/test-features.ts index 81c9aa24ea..7f4be10951 100644 --- a/developer/src/kmc-kmn/test/test-features.ts +++ b/developer/src/kmc-kmn/test/test-features.ts @@ -12,7 +12,7 @@ describe('Keyboard compiler features', async function() { this.beforeAll(async function() { compiler = new KmnCompiler(); callbacks = new TestCompilerCallbacks(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, {saveDebug: true})); assert(compiler.verifyInitialized()); }); @@ -29,15 +29,15 @@ describe('Keyboard compiler features', async function() { ]; for(const v of versions) { - it(`should build a version ${v[0]} keyboard`, function() { + it(`should build a version ${v[0]} keyboard`, async function() { const fixtureName = makePathToFixture('features', `version_${v[1]}.kmn`); - const result = compiler.runCompiler(fixtureName, {outFile: `version_${v[1]}.kmx`, saveDebug: true}); + const result = await compiler.run(fixtureName, `version_${v[1]}.kmx`); if(result === null) callbacks.printMessages(); assert.isNotNull(result); const reader = new KmxFileReader(); - const keyboard = reader.read(result.kmx.data); + const keyboard = reader.read(result.artifacts.kmx.data); assert.equal(keyboard.fileVersion, v[2]); }); } diff --git a/developer/src/kmc-kmn/test/test-messages.ts b/developer/src/kmc-kmn/test/test-messages.ts index 51e6894daa..92f8936f32 100644 --- a/developer/src/kmc-kmn/test/test-messages.ts +++ b/developer/src/kmc-kmn/test/test-messages.ts @@ -23,13 +23,13 @@ describe('CompilerMessages', function () { callbacks.clear(); const compiler = new KmnCompiler(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, {saveDebug: true, shouldAddCompilerVersion: false})); assert(compiler.verifyInitialized()); const kmnPath = makePathToFixture(...fixture); // Note: throwing away compile results (just to memory) - compiler.runCompiler(kmnPath, {saveDebug: true, shouldAddCompilerVersion: false}); + await compiler.run(kmnPath, null); if(messageId) { assert.isTrue(callbacks.hasMessage(messageId), `messageId ${messageId.toString(16)} not generated, instead got: `+JSON.stringify(callbacks.messages,null,2)); diff --git a/developer/src/kmc-kmn/test/test-wasm-uset.ts b/developer/src/kmc-kmn/test/test-wasm-uset.ts index e2bf5fcf1d..13288e10ca 100644 --- a/developer/src/kmc-kmn/test/test-wasm-uset.ts +++ b/developer/src/kmc-kmn/test/test-wasm-uset.ts @@ -25,14 +25,14 @@ describe('Compiler UnicodeSet function', function() { it('should start', async function() { const compiler = new KmnCompiler(); const callbacks = new TestCompilerCallbacks(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, null)); assert(compiler.verifyInitialized()); }); it('should compile a basic uset', async function() { const compiler = new KmnCompiler(); const callbacks = new TestCompilerCallbacks(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, null)); assert(compiler.verifyInitialized()); const pat = "[abc]"; @@ -50,7 +50,7 @@ describe('Compiler UnicodeSet function', function() { it('should compile a more complex uset', async function() { const compiler = new KmnCompiler(); const callbacks = new TestCompilerCallbacks(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, null)); assert(compiler.verifyInitialized()); const pat = "[[🙀A-C]-[CB]]"; @@ -70,7 +70,7 @@ describe('Compiler UnicodeSet function', function() { it('should compile an even more complex uset', async function() { const compiler = new KmnCompiler(); const callbacks = new TestCompilerCallbacks(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, null)); assert(compiler.verifyInitialized()); const pat = "[\\u{10FFFD}\\u{2019}\\u{22}\\u{a}\\u{ead}\\u{1F640}]"; @@ -97,7 +97,7 @@ describe('Compiler UnicodeSet function', function() { it('should fail in various ways', async function() { const compiler = new KmnCompiler(); const callbacks = new TestCompilerCallbacks(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, null)); assert(compiler.verifyInitialized()); // map from string to failing error const failures = { diff --git a/developer/src/kmc-ldml/src/compiler/compiler.ts b/developer/src/kmc-ldml/src/compiler/compiler.ts index b6f51f553b..b7dcbf9dfb 100644 --- a/developer/src/kmc-ldml/src/compiler/compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/compiler.ts @@ -59,7 +59,7 @@ export class LdmlKeyboardCompiler { if (this.usetparser === undefined) { // initialize const compiler = new KmnCompiler(); - const ok = await compiler.init(this.callbacks); + const ok = await compiler.init(this.callbacks, null); if (ok) { this.usetparser = compiler; } else { diff --git a/developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts b/developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts index 61dd2f97a6..519d251b61 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts @@ -11,16 +11,12 @@ export class BuildKmnKeyboard extends BuildActivity { public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.KeymanKeyboard; } public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.Keyboard; } public get description(): string { return 'Build a Keyman keyboard'; } - public async build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { - let compiler = new KmnCompiler(); - if(!await compiler.init(callbacks)) { - return false; - } - + public async build(infile: string, /*TODO: outfile?: string,*/ callbacks: CompilerCallbacks, options: CompilerOptions): Promise { // We need to resolve paths to absolute paths before calling kmc-kmn - if(options.outFile) { - options.outFile = getPosixAbsolutePath(options.outFile); - const folderName = path.dirname(options.outFile); + let outfile = options.outFile;//TODO: remove here + if(outfile) { + outfile = getPosixAbsolutePath(outfile); + const folderName = path.dirname(outfile); try { fs.mkdirSync(folderName, {recursive: true}); } catch(e) { @@ -29,7 +25,18 @@ export class BuildKmnKeyboard extends BuildActivity { } } infile = getPosixAbsolutePath(infile); - return compiler.run(infile, options); + + const compiler = new KmnCompiler(); + if(!await compiler.init(callbacks, options)) { + return false; + } + + const result = await compiler.run(infile, outfile); + if(!result) { + return false; + } + + return await compiler.write(result.artifacts); } } diff --git a/developer/src/kmc/src/util/TestKeymanSentry.ts b/developer/src/kmc/src/util/TestKeymanSentry.ts index 75da80e53d..b1c4aa40ee 100644 --- a/developer/src/kmc/src/util/TestKeymanSentry.ts +++ b/developer/src/kmc/src/util/TestKeymanSentry.ts @@ -26,7 +26,7 @@ export class TestKeymanSentry { if(cli.includes('kmcmplib')) { const compiler = new KmnCompiler(); const callbacks = new NodeCompilerCallbacks({}); - if(!await compiler.init(callbacks)) { + if(!await compiler.init(callbacks, null)) { throw new Error('Failed to instantiate WASM compiler'); } try { From d18d4833a96d7237db3f81f3a3361617f89aff04 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Thu, 7 Dec 2023 11:41:27 +0700 Subject: [PATCH 02/14] chore(developer): tidyup of KmnCompiler members --- developer/src/kmc-kmn/src/compiler/compiler.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/developer/src/kmc-kmn/src/compiler/compiler.ts b/developer/src/kmc-kmn/src/compiler/compiler.ts index a4033ad89c..6878fc1b47 100644 --- a/developer/src/kmc-kmn/src/compiler/compiler.ts +++ b/developer/src/kmc-kmn/src/compiler/compiler.ts @@ -97,10 +97,10 @@ let Module: any; export class KmnCompiler implements KeymanCompiler, UnicodeSetParser { - callbackID: string; // a unique numeric id added to globals with prefixed names - callbacks: CompilerCallbacks; - wasmExports: MallocAndFree; - options: KmnCompilerOptions; + private readonly callbackID: string; // a unique numeric id added to globals with prefixed names + private callbacks: CompilerCallbacks; + private wasmExports: MallocAndFree; + private options: KmnCompilerOptions; constructor() { this.callbackID = callbackPrefix + callbackProcIdentifier.toString(); @@ -109,7 +109,7 @@ export class KmnCompiler implements KeymanCompiler, UnicodeSetParser { public async init(callbacks: CompilerCallbacks, options: KmnCompilerOptions): Promise { this.callbacks = callbacks; - this.options = options; + this.options = {...options}; if(!Module) { try { Module = await loadWasmHost(); From d3ee8c98020056085e8552c5e1b942a22ef611c4 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Thu, 7 Dec 2023 11:42:10 +0700 Subject: [PATCH 03/14] feat(developer): LdmlKeyboardCompiler now implements KeymanCompiler Relates to #9473. Refactors the public API of LdmlKeyboardCompiler to meet KeymanCompiler, including asyncing a bunch of functions, and moving file write responsibilities out of kmc and into kmc-ldml. --- .../src/kmc-ldml/src/compiler/compiler.ts | 94 ++++++++++++++++-- developer/src/kmc-ldml/test/helpers/index.ts | 13 ++- .../src/kmc-ldml/test/test-compiler-e2e.ts | 27 +++--- .../kmc-ldml/test/test-keymanweb-compiler.ts | 3 +- .../src/kmc-ldml/test/test-testdata-e2e.ts | 4 +- .../buildClasses/BuildLdmlKeyboard.ts | 96 ++++--------------- .../kmc/src/commands/buildTestData/index.ts | 13 ++- 7 files changed, 140 insertions(+), 110 deletions(-) diff --git a/developer/src/kmc-ldml/src/compiler/compiler.ts b/developer/src/kmc-ldml/src/compiler/compiler.ts index b7dcbf9dfb..9cd19bf19e 100644 --- a/developer/src/kmc-ldml/src/compiler/compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/compiler.ts @@ -1,4 +1,4 @@ -import { LDMLKeyboardXMLSourceFileReader, LDMLKeyboard, KMXPlus, CompilerCallbacks, LDMLKeyboardTestDataXMLSourceFile, UnicodeSetParser } from '@keymanapp/common-types'; +import { LDMLKeyboardXMLSourceFileReader, LDMLKeyboard, KMXPlus, CompilerCallbacks, LDMLKeyboardTestDataXMLSourceFile, UnicodeSetParser, KeymanCompiler, KeymanCompilerResult, KeymanCompilerArtifacts, defaultCompilerOptions, KMXBuilder, KvkFileWriter, KeymanCompilerArtifactOptional } from '@keymanapp/common-types'; import { LdmlCompilerOptions } from './ldml-compiler-options.js'; import { CompilerMessages } from './messages.js'; import { BkspCompiler, TranCompiler } from './tran.js'; @@ -16,6 +16,9 @@ import KMXPlusFile = KMXPlus.KMXPlusFile; import DependencySections = KMXPlus.DependencySections; import { SectionIdent, constants } from '@keymanapp/ldml-keyboard-constants'; import { KmnCompiler } from '@keymanapp/kmc-kmn'; +import { KMXPlusMetadataCompiler } from './metadata-compiler.js'; +import { LdmlKeyboardVisualKeyboardCompiler } from './visual-keyboard-compiler.js'; +import { LdmlKeyboardKeymanWebCompiler } from './keymanweb-compiler.js'; export const SECTION_COMPILERS = [ // These are in dependency order. @@ -37,18 +40,93 @@ export const SECTION_COMPILERS = [ TranCompiler, ]; -export class LdmlKeyboardCompiler { - private readonly callbacks: CompilerCallbacks; - private readonly options: LdmlCompilerOptions; +export interface LdmlKeyboardCompilerArtifacts extends KeymanCompilerArtifacts { + kmx?: KeymanCompilerArtifactOptional; + kvk?: KeymanCompilerArtifactOptional; + js?: KeymanCompilerArtifactOptional; +}; + +export interface LdmlKeyboardCompilerResult extends KeymanCompilerResult { + artifacts: LdmlKeyboardCompilerArtifacts; +}; + +export class LdmlKeyboardCompiler implements KeymanCompiler { + private callbacks: CompilerCallbacks; + private options: LdmlCompilerOptions; // uset parser private usetparser?: UnicodeSetParser = undefined; - constructor (callbacks: CompilerCallbacks, options: LdmlCompilerOptions) { - this.options = { - ...options - }; + async init(callbacks: CompilerCallbacks, options: LdmlCompilerOptions): Promise { + this.options = {...options}; this.callbacks = callbacks; + return true; + } + + async run(inputFilename: string, outputFilename?: string): Promise { + + let compilerOptions: LdmlCompilerOptions = { + ...defaultCompilerOptions, + ...this.options, + }; + + let source = this.load(inputFilename); + if (!source) { + return null; + } + let kmx = await this.compile(source); + if (!kmx) { + return null; + } + + // In order for the KMX file to be loaded by non-KMXPlus components, it is helpful + // to duplicate some of the metadata + KMXPlusMetadataCompiler.addKmxMetadata(kmx.kmxplus, kmx.keyboard, compilerOptions); + + // Use the builder to generate the binary output file + const builder = new KMXBuilder(kmx, compilerOptions.saveDebug); + const kmx_binary = builder.compile(); + + const vkcompiler = new LdmlKeyboardVisualKeyboardCompiler(this.callbacks); + const vk = vkcompiler.compile(source); + const writer = new KvkFileWriter(); + const kvk_binary = writer.write(vk); + + // Note: we could have a step of generating source files here + // KvksFileWriter()... + // const tlcompiler = new kmc.TouchLayoutCompiler(); + // const tl = tlcompiler.compile(source); + // const tlwriter = new TouchLayoutFileWriter(); + const kmwcompiler = new LdmlKeyboardKeymanWebCompiler(this.callbacks, compilerOptions); + const kmw_string = kmwcompiler.compile(inputFilename, source); + const encoder = new TextEncoder(); + const kmw_binary = encoder.encode(kmw_string); + + outputFilename = outputFilename ?? inputFilename.replace(/\.xml$/, '.kmx'); + + return { + artifacts: { + kmx: { data: kmx_binary, filename: outputFilename }, + kvk: { data: kvk_binary, filename: outputFilename.replace(/\.kmx$/, '.kvk') }, + js: { data: kmw_binary, filename: outputFilename.replace(/\.kmx$/, '.js') }, + } + }; + } + + async write(artifacts: LdmlKeyboardCompilerArtifacts): Promise { + if(artifacts.kmx) { + this.callbacks.fs.writeFileSync(artifacts.kmx.filename, artifacts.kmx.data); + } + + if(artifacts.kvk) { + this.callbacks.fs.writeFileSync(artifacts.kvk.filename, artifacts.kvk.data); + } + + if(artifacts.js) { + this.callbacks.fs.writeFileSync(artifacts.js.filename, artifacts.js.data); + } + + return true; } /** diff --git a/developer/src/kmc-ldml/test/helpers/index.ts b/developer/src/kmc-ldml/test/helpers/index.ts index 8f6d73ef43..53406873a4 100644 --- a/developer/src/kmc-ldml/test/helpers/index.ts +++ b/developer/src/kmc-ldml/test/helpers/index.ts @@ -112,14 +112,16 @@ async function loadDepsFor(sections: DependencySections, parentCompiler: Section } } -export function loadTestdata(inputFilename: string, options: LdmlCompilerOptions) : LDMLKeyboardTestDataXMLSourceFile { - const k = new LdmlKeyboardCompiler(compilerTestCallbacks, options); +export async function loadTestData(inputFilename: string, options: LdmlCompilerOptions) : Promise { + const k = new LdmlKeyboardCompiler(); + assert.isTrue(await k.init(compilerTestCallbacks, options)); const source = k.loadTestData(inputFilename); return source; } export async function compileKeyboard(inputFilename: string, options: LdmlCompilerOptions, validateMessages?: CompilerEvent[], expectFailValidate?: boolean, compileMessages?: CompilerEvent[]): Promise { - const k = new LdmlKeyboardCompiler(compilerTestCallbacks, options); + const k = new LdmlKeyboardCompiler(); + assert.isTrue(await k.init(compilerTestCallbacks, options)); const source = k.load(inputFilename); checkMessages(); assert.isNotNull(source, 'k.load should not have returned null'); @@ -151,7 +153,8 @@ export async function compileKeyboard(inputFilename: string, options: LdmlCompil } export async function compileVisualKeyboard(inputFilename: string, options: LdmlCompilerOptions): Promise { - const k = new LdmlKeyboardCompiler(compilerTestCallbacks, options); + const k = new LdmlKeyboardCompiler(); + assert.isTrue(await k.init(compilerTestCallbacks, options)); const source = k.load(inputFilename); checkMessages(); assert.isNotNull(source, 'k.load should not have returned null'); @@ -254,7 +257,7 @@ async function getTestUnicodeSetParser(callbacks: CompilerCallbacks): Promise(code, expected); }); - it('should handle non existent files', () => { + it('should handle non existent files', async () => { const filename = 'DOES_NOT_EXIST.xml'; - const k = new LdmlKeyboardCompiler(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); + const k = new LdmlKeyboardCompiler(); + await k.init(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); const source = k.load(filename); assert.notOk(source, `Trying to load(${filename})`); }); - it('should handle unparseable files', () => { + it('should handle unparseable files', async () => { const filename = makePathToFixture('basic-kvk.txt'); // not an .xml file - const k = new LdmlKeyboardCompiler(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); + const k = new LdmlKeyboardCompiler(); + await k.init(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); const source = k.load(filename); assert.notOk(source, `Trying to load(${filename})`); }); - it('should handle not-valid files', () => { + it('should handle not-valid files', async () => { const filename = makePathToFixture('test-fr.xml'); // not a keyboard .xml file - const k = new LdmlKeyboardCompiler(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); + const k = new LdmlKeyboardCompiler(); + await k.init(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); const source = k.load(filename); assert.notOk(source, `Trying to load(${filename})`); }); - it('should handle non existent test files', () => { + it('should handle non existent test files', async () => { const filename = 'DOES_NOT_EXIST.xml'; - const k = new LdmlKeyboardCompiler(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); + const k = new LdmlKeyboardCompiler(); + await k.init(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); const source = k.loadTestData(filename); assert.notOk(source, `Trying to loadTestData(${filename})`); }); - it('should handle unparseable test files', () => { + it('should handle unparseable test files', async () => { const filename = makePathToFixture('basic-kvk.txt'); // not an .xml file - const k = new LdmlKeyboardCompiler(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); + const k = new LdmlKeyboardCompiler(); + await k.init(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); const source = k.load(filename); assert.notOk(source, `Trying to loadTestData(${filename})`); }); diff --git a/developer/src/kmc-ldml/test/test-keymanweb-compiler.ts b/developer/src/kmc-ldml/test/test-keymanweb-compiler.ts index e1a570ea12..9a14b38c33 100644 --- a/developer/src/kmc-ldml/test/test-keymanweb-compiler.ts +++ b/developer/src/kmc-ldml/test/test-keymanweb-compiler.ts @@ -16,7 +16,8 @@ describe('LdmlKeyboardKeymanWebCompiler', function() { // Load input data; we'll use the LDML keyboard compiler loader to save us // effort here - const k = new LdmlKeyboardCompiler(compilerTestCallbacks, {...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false}); + const k = new LdmlKeyboardCompiler(); + await k.init(compilerTestCallbacks, {...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false}); const source = k.load(inputFilename); checkMessages(); assert.isNotNull(source, 'k.load should not have returned null'); diff --git a/developer/src/kmc-ldml/test/test-testdata-e2e.ts b/developer/src/kmc-ldml/test/test-testdata-e2e.ts index eea3631759..cf23352328 100644 --- a/developer/src/kmc-ldml/test/test-testdata-e2e.ts +++ b/developer/src/kmc-ldml/test/test-testdata-e2e.ts @@ -1,7 +1,7 @@ import { readFileSync } from 'fs'; import 'mocha'; import {assert} from 'chai'; -import {compilerTestOptions, loadTestdata, makePathToFixture} from './helpers/index.js'; +import {compilerTestOptions, loadTestData, makePathToFixture} from './helpers/index.js'; describe('testdata-tests', function() { this.slow(500); // 0.5 sec -- json schema validation takes a while @@ -15,7 +15,7 @@ describe('testdata-tests', function() { const jsonFilename = makePathToFixture('test-fr.json'); // Compile the keyboard - const testData = loadTestdata(inputFilename, {...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false}); + const testData = await loadTestData(inputFilename, {...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false}); assert.isNotNull(testData); const jsonData = JSON.parse(readFileSync(jsonFilename, 'utf-8')); diff --git a/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts b/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts index 551ce66fa4..fbb69f921c 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts @@ -1,7 +1,6 @@ -import * as path from 'path'; import * as fs from 'fs'; import * as kmcLdml from '@keymanapp/kmc-ldml'; -import { KvkFileWriter, CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, CompilerOptions, defaultCompilerOptions, KeymanFileTypes } from '@keymanapp/common-types'; +import { CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, CompilerOptions, KeymanFileTypes } from '@keymanapp/common-types'; import { BuildActivity } from './BuildActivity.js'; import { fileURLToPath } from 'url'; import { InfrastructureMessages } from '../../messages/infrastructureMessages.js'; @@ -14,12 +13,25 @@ export class BuildLdmlKeyboard extends BuildActivity { public async build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { // TODO-LDML: consider hardware vs touch -- touch-only layout will not have a .kvk // Compile: - let [kmx,kvk,kmw] = await buildLdmlKeyboardToMemory(infile, callbacks, options); - // Output: + const outfile = options.outFile; //TODO - const fileBaseName = options.outFile ?? infile; - const outFileBase = path.basename(fileBaseName, path.extname(fileBaseName)); - const outFileDir = path.dirname(fileBaseName); + const ldmlCompilerOptions: kmcLdml.LdmlCompilerOptions = {...options, readerOptions: { + importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)) + }}; + + const compiler = new kmcLdml.LdmlKeyboardCompiler(); + if(!await compiler.init(callbacks, ldmlCompilerOptions)) { + return false; + } + + const result = await compiler.run(infile, outfile); + if(!result) { + return false; + } + + // TODO: Consider if this mkdir should be in write() + // TODO: This pattern may be needed for kmc-kmn as well? + const outFileDir = callbacks.path.dirname(outfile ?? infile); try { fs.mkdirSync(outFileDir, {recursive: true}); @@ -28,74 +40,6 @@ export class BuildLdmlKeyboard extends BuildActivity { return false; } - if(kmx && kvk) { - const outFileKmx = path.join(outFileDir, outFileBase + KeymanFileTypes.Binary.Keyboard); - // TODO: console needs to be replaced with InfrastructureMessages - console.log(`Writing compiled keyboard to ${outFileKmx}`); - fs.writeFileSync(outFileKmx, kmx); - - const outFileKvk = path.join(outFileDir, outFileBase + KeymanFileTypes.Binary.VisualKeyboard); - // TODO: console needs to be replaced with InfrastructureMessages - console.log(`Writing compiled visual keyboard to ${outFileKvk}`); - fs.writeFileSync(outFileKvk, kvk); - } else { - // TODO: console needs to be replaced with InfrastructureMessages - console.error(`An error occurred compiling ${infile}`); - return false; - } - - if(kmw) { - const outFileKmw = path.join(outFileDir, outFileBase + KeymanFileTypes.Binary.WebKeyboard); - // TODO: console needs to be replaced with InfrastructureMessages - console.log(`Writing compiled js keyboard to ${outFileKmw}`); - fs.writeFileSync(outFileKmw, kmw); - } - - return true; + return await compiler.write(result.artifacts); } } - -async function buildLdmlKeyboardToMemory(inputFilename: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise<[Uint8Array, Uint8Array, Uint8Array]> { - let compilerOptions: kmcLdml.LdmlCompilerOptions = { - ...defaultCompilerOptions, - ...options, - readerOptions: { - importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)) - } - }; - - const k = new kmcLdml.LdmlKeyboardCompiler(callbacks, compilerOptions); - let source = k.load(inputFilename); - if (!source) { - return [null, null, null]; - } - let kmx = await k.compile(source); - if (!kmx) { - return [null, null, null]; - } - - // In order for the KMX file to be loaded by non-KMXPlus components, it is helpful - // to duplicate some of the metadata - kmcLdml.KMXPlusMetadataCompiler.addKmxMetadata(kmx.kmxplus, kmx.keyboard, compilerOptions); - - // Use the builder to generate the binary output file - const builder = new kmcLdml.KMXBuilder(kmx, options.saveDebug); - const kmx_binary = builder.compile(); - - const vkcompiler = new kmcLdml.LdmlKeyboardVisualKeyboardCompiler(callbacks); - const vk = vkcompiler.compile(source); - const writer = new KvkFileWriter(); - const kvk_binary = writer.write(vk); - - // Note: we could have a step of generating source files here - // KvksFileWriter()... - // const tlcompiler = new kmc.TouchLayoutCompiler(); - // const tl = tlcompiler.compile(source); - // const tlwriter = new TouchLayoutFileWriter(); - const kmwcompiler = new kmcLdml.LdmlKeyboardKeymanWebCompiler(callbacks, compilerOptions); - const kmw_string = kmwcompiler.compile(inputFilename, source); - const encoder = new TextEncoder(); - const kmw_binary = encoder.encode(kmw_string); - - return [kmx_binary, kvk_binary, kmw_binary]; -} diff --git a/developer/src/kmc/src/commands/buildTestData/index.ts b/developer/src/kmc/src/commands/buildTestData/index.ts index 834112bc13..5d0f6a2380 100644 --- a/developer/src/kmc/src/commands/buildTestData/index.ts +++ b/developer/src/kmc/src/commands/buildTestData/index.ts @@ -5,7 +5,7 @@ import { CompilerBaseOptions, CompilerCallbacks, defaultCompilerOptions, LDMLKey import { NodeCompilerCallbacks } from '../../util/NodeCompilerCallbacks.js'; import { fileURLToPath } from 'url'; -export function buildTestData(infile: string, _options: any, commander: any) { +export async function buildTestData(infile: string, _options: any, commander: any) { const options: CompilerBaseOptions = commander.optsWithGlobals(); let compilerOptions: kmcLdml.LdmlCompilerOptions = { @@ -18,7 +18,7 @@ export function buildTestData(infile: string, _options: any, commander: any) { } }; - let testData = loadTestData(infile, compilerOptions); + let testData = await loadTestData(infile, compilerOptions); if (!testData) { return; } @@ -31,12 +31,11 @@ export function buildTestData(infile: string, _options: any, commander: any) { fs.writeFileSync(outFileJson, JSON.stringify(testData, null, ' ')); } -function loadTestData(inputFilename: string, options: kmcLdml.LdmlCompilerOptions): LDMLKeyboardTestDataXMLSourceFile { +async function loadTestData(inputFilename: string, options: kmcLdml.LdmlCompilerOptions): Promise { const callbacks: CompilerCallbacks = new NodeCompilerCallbacks(options); - const k = new kmcLdml.LdmlKeyboardCompiler(callbacks, options); - let source = k.loadTestData(inputFilename); - if (!source) { + const k = new kmcLdml.LdmlKeyboardCompiler(); + if(!await k.init(callbacks, options)) { return null; } - return source; + return await k.loadTestData(inputFilename); } From 4c53509df9420f512d49a45b2eeb09e5ebe2a840 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Fri, 8 Dec 2023 11:45:53 +0700 Subject: [PATCH 04/14] feat(developer): LexicalModelCompiler now implements KeymanCompiler Relates to #9473. Refactors the public API of LexicalModelCompiler to meet KeymanCompiler, including asyncing a bunch of functions, and moving file write responsibilities into the class itself. Removes top-level `compileModel()` and `loadFromFilename()` functions, as you should now use `LexicalModelCompiler` API. Most of the test cases needed only minor patching, but introducing the `async init()` function has caused them to all be async, despite the model compiler itself having no async requirements. I think this is fine, because it makes any future additional async initialization tasks much less painful to implement. --- .../kmc-model/src/lexical-model-compiler.ts | 92 ++++++++++++++++++- developer/src/kmc-model/src/main.ts | 80 +--------------- .../test-compile-model-with-pseudoclosure.ts | 32 ++++--- .../src/kmc-model/test/test-compile-model.ts | 13 ++- .../src/kmc-model/test/test-compile-trie.ts | 27 +++--- .../src/kmc-model/test/test-punctuation.ts | 7 +- .../src/commands/buildClasses/BuildModel.ts | 22 ++--- developer/src/kmc/src/kmlmc.ts | 18 +++- 8 files changed, 156 insertions(+), 135 deletions(-) diff --git a/developer/src/kmc-model/src/lexical-model-compiler.ts b/developer/src/kmc-model/src/lexical-model-compiler.ts index 0edd818968..8785ed32c4 100644 --- a/developer/src/kmc-model/src/lexical-model-compiler.ts +++ b/developer/src/kmc-model/src/lexical-model-compiler.ts @@ -2,20 +2,102 @@ lexical-model-compiler.ts: base file for lexical model compiler. */ -import * as ts from "typescript"; +import ts from "typescript"; import { createTrieDataStructure } from "./build-trie.js"; import { ModelDefinitions } from "./model-definitions.js"; import {decorateWithJoin} from "./join-word-breaker-decorator.js"; import {decorateWithScriptOverrides} from "./script-overrides-decorator.js"; import { LexicalModelSource, WordBreakerSpec, SimpleWordBreakerSpec } from "./lexical-model.js"; -import { ModelCompilerError, ModelCompilerMessages } from "./model-compiler-errors.js"; +import { ModelCompilerError, ModelCompilerMessageContext, ModelCompilerMessages } from "./model-compiler-errors.js"; import { callbacks, setCompilerCallbacks } from "./compiler-callbacks.js"; -import { CompilerCallbacks } from "@keymanapp/common-types"; +import { CompilerCallbacks, CompilerOptions, KeymanCompiler, KeymanCompilerArtifact, KeymanCompilerArtifacts, KeymanCompilerResult } from "@keymanapp/common-types"; -export default class LexicalModelCompiler { +/** + * An ECMAScript module as emitted by the TypeScript compiler. + */ +interface ES2015Module { + /** This is always true. */ + __esModule: boolean; + 'default'?: unknown; +}; - constructor(callbacks: CompilerCallbacks) { +export interface LexicalModelCompilerArtifacts extends KeymanCompilerArtifacts { + js: KeymanCompilerArtifact; +}; + +export interface LexicalModelCompilerResult extends KeymanCompilerResult { + artifacts: LexicalModelCompilerArtifacts; +}; + +export class LexicalModelCompiler implements KeymanCompiler { + + async init(callbacks: CompilerCallbacks, _options: CompilerOptions): Promise { setCompilerCallbacks(callbacks); + return true; + } + + async run(inputFilename: string, outputFilename?: string): Promise { + try { + let modelSource = this.loadFromFilename(inputFilename); + let containingDirectory = callbacks.path.dirname(inputFilename); + let code = this.generateLexicalModelCode('', modelSource, containingDirectory); + const result: LexicalModelCompilerResult = { + artifacts: { + js: { + data: new TextEncoder().encode(code), + filename: outputFilename ?? inputFilename.replace(/\.model\.ts$/, '.model.js') + } + } + } + return result; + } catch(e) { + callbacks.reportMessage( + e instanceof ModelCompilerError + ? e.event + : ModelCompilerMessages.Fatal_UnexpectedException({e:e}) + ); + return null; + } + } + + async write(artifacts: LexicalModelCompilerArtifacts): Promise { + callbacks.fs.writeFileSync(artifacts.js.filename, artifacts.js.data); + return true; + } + + /** + * Loads a lexical model's source module from the given filename. + * + * @param filename path to the model source file. + */ + public loadFromFilename(filename: string): LexicalModelSource { + + let sourceCode = new TextDecoder().decode(callbacks.loadFile(filename)); + // Compile the module to JavaScript code. + // NOTE: transpile module does a very simple TS to JS compilation. + // It DOES NOT check for types! + let compilationOutput = ts.transpile(sourceCode, { + // Our runtime only supports ES3 with Node/CommonJS modules on Android 5.0. + // When we drop Android 5.0 support, we can update this to a `ScriptTarget` + // matrix against target version of Keyman, here and in + // lexical-model-compiler.ts. + target: ts.ScriptTarget.ES3, + module: ts.ModuleKind.CommonJS, + }); + // Turn the module into a function in which we can inject a global. + let moduleCode = '(function(exports){' + compilationOutput + '})'; + + // Run the module; its exports will be assigned to `moduleExports`. + let moduleExports: Partial = {}; + let module = eval(moduleCode); + module(moduleExports); + + if (!moduleExports['__esModule'] || !moduleExports['default']) { + ModelCompilerMessageContext.filename = filename; + throw new ModelCompilerError(ModelCompilerMessages.Error_NoDefaultExport()); + } + + return moduleExports['default'] as LexicalModelSource; } /** diff --git a/developer/src/kmc-model/src/main.ts b/developer/src/kmc-model/src/main.ts index 667253353d..68fae091e3 100644 --- a/developer/src/kmc-model/src/main.ts +++ b/developer/src/kmc-model/src/main.ts @@ -1,79 +1 @@ -import { CompilerCallbacks } from '@keymanapp/common-types'; -import ts from 'typescript'; -import { setCompilerCallbacks } from './compiler-callbacks.js'; - -import LexicalModelCompiler from './lexical-model-compiler.js'; -import { LexicalModelSource } from './lexical-model.js'; -import { ModelCompilerError, ModelCompilerMessageContext, ModelCompilerMessages } from './model-compiler-errors.js'; - -export { default as LexicalModelCompiler } from './lexical-model-compiler.js'; - -/** - * Compiles a model.ts file, using paths relative to its location. - * - * @param filename path to model.ts source. - * @return model source code, or null on error - */ -export function compileModel(filename: string, callbacks: CompilerCallbacks): string { - setCompilerCallbacks(callbacks); - - try { - let modelSource = loadFromFilename(filename, callbacks); - let containingDirectory = callbacks.path.dirname(filename); - - return (new LexicalModelCompiler(callbacks)) - .generateLexicalModelCode('', modelSource, containingDirectory); - } catch(e) { - callbacks.reportMessage( - e instanceof ModelCompilerError - ? e.event - : ModelCompilerMessages.Fatal_UnexpectedException({e:e}) - ); - } - return null; -} - -/** - * An ECMAScript module as emitted by the TypeScript compiler. - */ -interface ES2015Module { - /** This is always true. */ - __esModule: boolean; - 'default'?: unknown; -} - -/** - * Loads a lexical model's source module from the given filename. - * - * @param filename path to the model source file. - */ -export function loadFromFilename(filename: string, callbacks: CompilerCallbacks): LexicalModelSource { - setCompilerCallbacks(callbacks); - - let sourceCode = new TextDecoder().decode(callbacks.loadFile(filename)); - // Compile the module to JavaScript code. - // NOTE: transpile module does a very simple TS to JS compilation. - // It DOES NOT check for types! - let compilationOutput = ts.transpile(sourceCode, { - // Our runtime only supports ES3 with Node/CommonJS modules on Android 5.0. - // When we drop Android 5.0 support, we can update this to a `ScriptTarget` - // matrix against target version of Keyman, here and in - // lexical-model-compiler.ts. - target: ts.ScriptTarget.ES3, - module: ts.ModuleKind.CommonJS, - }); - // Turn the module into a function in which we can inject a global. - let moduleCode = '(function(exports){' + compilationOutput + '})'; - - // Run the module; its exports will be assigned to `moduleExports`. - let moduleExports: Partial = {}; - let module = eval(moduleCode); - module(moduleExports); - - if (!moduleExports['__esModule'] || !moduleExports['default']) { - ModelCompilerMessageContext.filename = filename; - throw new ModelCompilerError(ModelCompilerMessages.Error_NoDefaultExport()); - } - - return moduleExports['default'] as LexicalModelSource; -} +export { LexicalModelCompiler } from './lexical-model-compiler.js'; diff --git a/developer/src/kmc-model/test/test-compile-model-with-pseudoclosure.ts b/developer/src/kmc-model/test/test-compile-model-with-pseudoclosure.ts index d6fcec3959..08247e17c1 100644 --- a/developer/src/kmc-model/test/test-compile-model-with-pseudoclosure.ts +++ b/developer/src/kmc-model/test/test-compile-model-with-pseudoclosure.ts @@ -1,4 +1,4 @@ -import LexicalModelCompiler from '../src/lexical-model-compiler.js'; +import { LexicalModelCompiler } from '../src/lexical-model-compiler.js'; import {assert} from 'chai'; import 'mocha'; @@ -26,8 +26,9 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () { } }; - it('variant 1: applyCasing prepends symbols, searchTermToKey removes them', function() { - let compiler = new LexicalModelCompiler(callbacks); + it('variant 1: applyCasing prepends symbols, searchTermToKey removes them', async function() { + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'], @@ -80,8 +81,9 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () { assert.isNotNull(compilation.exportedModel); }); - it('variant 2: applyCasing prepends symbols, searchTermToKey keeps them', function() { - let compiler = new LexicalModelCompiler(callbacks); + it('variant 2: applyCasing prepends symbols, searchTermToKey keeps them', async function() { + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'], @@ -124,8 +126,9 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () { assert.isNotNull(compilation.exportedModel); }); - it('variant 3: applyCasing prepends symbols, default searchTermToKey', function() { - let compiler = new LexicalModelCompiler(callbacks); + it('variant 3: applyCasing prepends symbols, default searchTermToKey', async function() { + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'], @@ -164,8 +167,9 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () { }); describe('relying on default applyCasing + searchTermToKey', function() { - it('languageUsesCasing: true', function() { - let compiler = new LexicalModelCompiler(callbacks); + it('languageUsesCasing: true', async function() { + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'], @@ -188,8 +192,9 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () { assert.isNotNull(compilation.exportedModel); }); - it('languageUsesCasing: false', function() { - let compiler = new LexicalModelCompiler(callbacks); + it('languageUsesCasing: false', async function() { + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'], @@ -213,8 +218,9 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () { assert.isNotNull(compilation.exportedModel); }); - it('languageUsesCasing: undefined', function() { - let compiler = new LexicalModelCompiler(callbacks); + it('languageUsesCasing: undefined', async function() { + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'] diff --git a/developer/src/kmc-model/test/test-compile-model.ts b/developer/src/kmc-model/test/test-compile-model.ts index 2db28b6e09..e3074d697a 100644 --- a/developer/src/kmc-model/test/test-compile-model.ts +++ b/developer/src/kmc-model/test/test-compile-model.ts @@ -1,12 +1,12 @@ import 'mocha'; import {assert} from 'chai'; -import {compileModel} from '../src/main.js'; +import { LexicalModelCompiler } from '../src/main.js'; import {makePathToFixture, compileModelSourceCode, CompilationResult} from './helpers/index.js'; import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers'; import { KeymanFileTypes } from '@keymanapp/common-types'; -describe('compileModel', function () { +describe('LexicalModelCompiler', function () { let callbacks = new TestCompilerCallbacks(); // Try to compile ALL of the correct models. @@ -22,9 +22,14 @@ describe('compileModel', function () { for (let modelID of MODELS) { let modelPath = makePathToFixture(modelID, modelID + KeymanFileTypes.Source.Model); - it(`should compile ${modelID}`, function () { - let code = compileModel(modelPath, callbacks); + it(`should compile ${modelID}`, async function () { + const compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); + const result = await compiler.run(modelPath, null); callbacks.printMessages(); + assert.isNotNull(result); + const decoder = new TextDecoder(); + const code = decoder.decode(result.artifacts.js.data); let r = compileModelSourceCode(code); let compilation = r as CompilationResult; diff --git a/developer/src/kmc-model/test/test-compile-trie.ts b/developer/src/kmc-model/test/test-compile-trie.ts index 544924da3e..5d80b72ac1 100644 --- a/developer/src/kmc-model/test/test-compile-trie.ts +++ b/developer/src/kmc-model/test/test-compile-trie.ts @@ -1,4 +1,4 @@ -import LexicalModelCompiler from '../src/lexical-model-compiler.js'; +import { LexicalModelCompiler } from '../src/lexical-model-compiler.js'; import {assert} from 'chai'; import 'mocha'; @@ -14,11 +14,12 @@ describe('LexicalModelCompiler', function () { }); describe('#generateLexicalModelCode', function () { - it('should compile a trivial word list', function () { + it('should compile a trivial word list', async function () { const MODEL_ID = 'example.qaa.trivial'; const PATH = makePathToFixture(MODEL_ID); - let compiler = new LexicalModelCompiler(callbacks); + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'] @@ -37,11 +38,12 @@ describe('LexicalModelCompiler', function () { assert.match(code, /\bwordBreaker\b["']?:\s*wordBreakers\b/); }); - it('should compile a word list exported by Microsoft Excel', function () { + it('should compile a word list exported by Microsoft Excel', async function () { const MODEL_ID = 'example.qaa.utf16le'; const PATH = makePathToFixture(MODEL_ID); - let compiler = new LexicalModelCompiler(callbacks); + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.txt'] @@ -58,11 +60,12 @@ describe('LexicalModelCompiler', function () { }); }); - it('should compile a word list with a custom word breaking function', function () { + it('should compile a word list with a custom word breaking function', async function () { const MODEL_ID = 'example.qaa.trivial'; const PATH = makePathToFixture(MODEL_ID); - let compiler = new LexicalModelCompiler(callbacks); + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'], @@ -81,11 +84,12 @@ describe('LexicalModelCompiler', function () { assert.match(code, /\bwordBreaker\b["']?:\s+function\b/); }); - it('should not generate unpaired surrogate code units', function () { + it('should not generate unpaired surrogate code units', async function () { const MODEL_ID = 'example.qaa.smp'; const PATH = makePathToFixture(MODEL_ID); - let compiler = new LexicalModelCompiler(callbacks); + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'] @@ -115,10 +119,11 @@ describe('LexicalModelCompiler', function () { assert.match(code, /\btotalWeight\b["']?:\s*27596\b/); }); - it('should include the source code of its search term to key function', function () { + it('should include the source code of its search term to key function', async function () { const MODEL_ID = 'example.qaa.trivial'; const PATH = makePathToFixture(MODEL_ID); - let compiler = new LexicalModelCompiler(callbacks); + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'] diff --git a/developer/src/kmc-model/test/test-punctuation.ts b/developer/src/kmc-model/test/test-punctuation.ts index 133eed5fed..f3ad64433e 100644 --- a/developer/src/kmc-model/test/test-punctuation.ts +++ b/developer/src/kmc-model/test/test-punctuation.ts @@ -1,4 +1,4 @@ -import LexicalModelCompiler from '../src/lexical-model-compiler.js'; +import { LexicalModelCompiler } from '../src/lexical-model-compiler.js'; import {assert} from 'chai'; import 'mocha'; @@ -10,10 +10,11 @@ describe('LexicalModelCompiler', function () { const MODEL_ID = 'example.qaa.trivial'; const PATH = makePathToFixture(MODEL_ID); - it('should compile punctuation into the generated code', function () { + it('should compile punctuation into the generated code', async function () { const callbacks = new TestCompilerCallbacks(); - let compiler = new LexicalModelCompiler(callbacks); + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'], diff --git a/developer/src/kmc/src/commands/buildClasses/BuildModel.ts b/developer/src/kmc/src/commands/buildClasses/BuildModel.ts index e69aeca780..00614c2731 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildModel.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildModel.ts @@ -1,6 +1,5 @@ -import * as fs from 'fs'; import { BuildActivity } from './BuildActivity.js'; -import { compileModel } from '@keymanapp/kmc-model'; +import { LexicalModelCompiler } from '@keymanapp/kmc-model'; import { CompilerCallbacks, CompilerOptions, KeymanFileTypes } from '@keymanapp/common-types'; export class BuildModel extends BuildActivity { @@ -9,25 +8,18 @@ export class BuildModel extends BuildActivity { public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.Model; } public get description(): string { return 'Build a lexical model'; } public async build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { - let outputFilename: string = this.getOutputFilename(infile, options); - let code = null; + const outputFilename: string = this.getOutputFilename(infile, options); - // Compile: - try { - code = compileModel(infile, callbacks); - } catch(e) { - console.error(e); + const compiler = new LexicalModelCompiler(); + if(!await compiler.init(callbacks, options)) { return false; } - if(!code) { - console.error('Compilation failed.') + const result = await compiler.run(infile, outputFilename); + if(!result) { return false; } - // Output: - fs.writeFileSync(outputFilename, code, 'utf8'); - - return true; + return await compiler.write(result.artifacts); } } \ No newline at end of file diff --git a/developer/src/kmc/src/kmlmc.ts b/developer/src/kmc/src/kmlmc.ts index 1ce171fd15..ca2fd4d6b0 100644 --- a/developer/src/kmc/src/kmlmc.ts +++ b/developer/src/kmc/src/kmlmc.ts @@ -3,9 +3,8 @@ * kmlmc - Keyman Lexical Model Compiler */ -import * as fs from 'fs'; import { Command } from 'commander'; -import { compileModel } from '@keymanapp/kmc-model'; +import { LexicalModelCompiler } from '@keymanapp/kmc-model'; import { SysExits } from './util/sysexits.js'; import KEYMAN_VERSION from "@keymanapp/keyman-version"; import { NodeCompilerCallbacks } from './util/NodeCompilerCallbacks.js'; @@ -30,10 +29,16 @@ if (!inputFilename) { const callbacks = new NodeCompilerCallbacks({logLevel: 'info'}); +const compiler = new LexicalModelCompiler(); +if(!await compiler.init(callbacks, null)) { + console.error('Initialization failed.'); + process.exit(SysExits.EX_DATAERR); +} + let code = null; // Compile: try { - code = compileModel(inputFilename, callbacks); + code = await compiler.run(inputFilename, program.opts().outFile); } catch(e) { console.error(e); process.exit(SysExits.EX_DATAERR); @@ -46,9 +51,12 @@ if(!code) { // Output: if (program.opts().outFile) { - fs.writeFileSync(program.opts().outFile, code, 'utf8'); + compiler.write(code.artifacts); } else { - console.log(code); + // TODO(lowpri): if writing to console then log messages should all be to stderr? + const decoder = new TextDecoder(); + const text = decoder.decode(code.artifacts.js.data); + console.log(text); } function exitDueToUsageError(message: string): never { From 69a2448176db50f251707370ffbeb38950b7e1a5 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Sat, 9 Dec 2023 06:48:53 +0700 Subject: [PATCH 05/14] feat(developer): PackageCompiler and WindowPackageInstallerCompiler now implement KeymanCompiler Relates to #9473. Refactors the public API of PackageCompiler and WindowsPackageInstallerCompiler to meet KeymanCompiler, including asyncing a bunch of functions, and moving file write responsibilities into the classes themselves. The classes were updated together in a single commit because WindowsPackageInstallerCompiler depends on PackageCompiler, and needed refactoring for the updated API access there anyway. Most of the test cases needed only minor patching, but introducing the `async init()` function has caused them to all be async. The test-messages module has a slight functional change with the compile process running completely rather than stopping on first message, which meant we needed to add extra dummy fixtures to avoid other errors that arose later in the compile process. The keyman.exe fixture is a text file, not a Windows executable. --- developer/src/kmc-keyboard-info/src/index.ts | 6 +- .../kmc-package/src/compiler/kmp-compiler.ts | 70 ++++++++++++++-- .../windows-package-installer-compiler.ts | 53 ++++++++++--- .../invalid/example.qaa.sencoten.model.js | 1 + .../test/fixtures/invalid/keyman.exe | 1 + .../test/fixtures/invalid/khmer_angkor.docx | 1 + .../src/kmc-package/test/test-messages.ts | 79 ++++++++----------- .../kmc-package/test/test-package-compiler.ts | 12 ++- .../src/kmc-package/test/test-versioning.ts | 4 +- ...test-windows-package-installer-compiler.ts | 12 ++- .../commands/buildClasses/BuildModelInfo.ts | 7 +- .../src/commands/buildClasses/BuildPackage.ts | 32 ++------ .../buildWindowsPackageInstaller/index.ts | 24 ++++-- developer/src/kmc/src/kmlmp.ts | 31 +++----- 14 files changed, 207 insertions(+), 126 deletions(-) create mode 100644 developer/src/kmc-package/test/fixtures/invalid/example.qaa.sencoten.model.js create mode 100644 developer/src/kmc-package/test/fixtures/invalid/keyman.exe create mode 100644 developer/src/kmc-package/test/fixtures/invalid/khmer_angkor.docx diff --git a/developer/src/kmc-keyboard-info/src/index.ts b/developer/src/kmc-keyboard-info/src/index.ts index beb4b88bb9..7d134e4acb 100644 --- a/developer/src/kmc-keyboard-info/src/index.ts +++ b/developer/src/kmc-keyboard-info/src/index.ts @@ -85,7 +85,11 @@ export class KeyboardInfoCompiler { // .kpj work is largely in kmc at present, so that would need to move to // a separate module. - const kmpCompiler = new KmpCompiler(this.callbacks); + const kmpCompiler = new KmpCompiler(); + if(!await kmpCompiler.init(this.callbacks, {})) { + // Errors will have been emitted by KmpCompiler + return null; + } const kmpJsonData = kmpCompiler.transformKpsToKmpObject(sources.kpsFilename); if(!kmpJsonData) { // Errors will have been emitted by KmpCompiler diff --git a/developer/src/kmc-package/src/compiler/kmp-compiler.ts b/developer/src/kmc-package/src/compiler/kmp-compiler.ts index 3dce88c9d5..b9993713d0 100644 --- a/developer/src/kmc-package/src/compiler/kmp-compiler.ts +++ b/developer/src/kmc-package/src/compiler/kmp-compiler.ts @@ -2,7 +2,7 @@ import * as xml2js from 'xml2js'; import JSZip from 'jszip'; import KEYMAN_VERSION from "@keymanapp/keyman-version"; -import { KmpJsonFile, KpsFile, SchemaValidators, CompilerCallbacks, KeymanFileTypes, KvkFile } from '@keymanapp/common-types'; +import { KmpJsonFile, KpsFile, SchemaValidators, CompilerCallbacks, KeymanFileTypes, KvkFile, KeymanCompiler, CompilerOptions, KeymanCompilerResult, KeymanCompilerArtifacts, KeymanCompilerArtifact } from '@keymanapp/common-types'; import { CompilerMessages } from './messages.js'; import { PackageMetadataCollector } from './package-metadata-collector.js'; import { KmpInfWriter } from './kmp-inf-writer.js'; @@ -11,6 +11,7 @@ import { MIN_LM_FILEVERSION_KMP_JSON, PackageVersionValidator } from './package- import { PackageKeyboardTargetValidator } from './package-keyboard-target-validator.js'; import { PackageMetadataUpdater } from './package-metadata-updater.js'; import { markdownToHTML } from './markdown.js'; +import { PackageValidation } from './package-validation.js'; const KMP_JSON_FILENAME = 'kmp.json'; const KMP_INF_FILENAME = 'kmp.inf'; @@ -20,9 +21,68 @@ const KMP_INF_FILENAME = 'kmp.inf'; // this filename for existing keyboard packages. const WELCOME_HTM_FILENAME = 'welcome.htm'; -export class KmpCompiler { +export interface KmpCompilerOptions extends CompilerOptions { + // Note: WindowsPackageInstallerCompilerOptions extends KmpCompilerOptions, so + // be careful when modifying this interface +}; - constructor(private callbacks: CompilerCallbacks) { +export interface KmpCompilerArtifacts extends KeymanCompilerArtifacts { + kmp: KeymanCompilerArtifact; +}; + +export interface KmpCompilerResult extends KeymanCompilerResult { + artifacts: KmpCompilerArtifacts; +}; + +export class KmpCompiler implements KeymanCompiler { + private callbacks: CompilerCallbacks; + private options: KmpCompilerOptions; + + public async init(callbacks: CompilerCallbacks, options: KmpCompilerOptions): Promise { + this.callbacks = callbacks; + this.options = options ? {...options} : {}; + return true; + } + + public async run(inputFilename: string, outputFilename?: string): Promise { + const kmpJsonData = this.transformKpsToKmpObject(inputFilename); + if(!kmpJsonData) { + return null; + } + + // + // Validate the package file + // + + const validation = new PackageValidation(this.callbacks, this.options); + if(!validation.validate(inputFilename, kmpJsonData)) { + return null; + } + + // + // Build the .kmp package file + // + + const data = await this.buildKmpFile(inputFilename, kmpJsonData); + if(!data) { + return null; + } + + const result: KmpCompilerResult = { + artifacts: { + kmp: { + data, + filename: outputFilename ?? inputFilename.replace(/\.kps$/, '.kmp') + } + } + } + + return result; + } + + public async write(artifacts: KmpCompilerArtifacts): Promise { + this.callbacks.fs.writeFileSync(artifacts.kmp.filename, artifacts.kmp.data); + return true; } public transformKpsToKmpObject(kpsFilename: string): KmpJsonFile.KmpJsonFile { @@ -346,7 +406,7 @@ export class KmpCompiler { * @param kpsFilename - Filename of the kps, not read, used only for calculating relative paths * @param kmpJsonData - The kmp.json Object */ - public buildKmpFile(kpsFilename: string, kmpJsonData: KmpJsonFile.KmpJsonFile): Promise { + public buildKmpFile(kpsFilename: string, kmpJsonData: KmpJsonFile.KmpJsonFile): Promise { const zip = JSZip(); @@ -438,7 +498,7 @@ export class KmpCompiler { } // Generate kmp file - return zip.generateAsync({type: 'binarystring', compression:'DEFLATE'}); + return zip.generateAsync({type:'uint8array', compression:'DEFLATE'}); } private buildKmpInf(data: KmpJsonFile.KmpJsonFile): Uint8Array { diff --git a/developer/src/kmc-package/src/compiler/windows-package-installer-compiler.ts b/developer/src/kmc-package/src/compiler/windows-package-installer-compiler.ts index 1a5f4ebe6b..8ac54572c1 100644 --- a/developer/src/kmc-package/src/compiler/windows-package-installer-compiler.ts +++ b/developer/src/kmc-package/src/compiler/windows-package-installer-compiler.ts @@ -11,9 +11,9 @@ */ import JSZip from 'jszip'; -import { CompilerCallbacks, KeymanFileTypes, KmpJsonFile, KpsFile } from "@keymanapp/common-types"; +import { CompilerCallbacks, KeymanCompiler, KeymanCompilerArtifact, KeymanCompilerArtifacts, KeymanCompilerResult, KeymanFileTypes, KmpJsonFile, KpsFile } from "@keymanapp/common-types"; import KEYMAN_VERSION from "@keymanapp/keyman-version"; -import { KmpCompiler } from "./kmp-compiler.js"; +import { KmpCompiler, KmpCompilerOptions } from "./kmp-compiler.js"; import { CompilerMessages } from "./messages.js"; const SETUP_INF_FILENAME = 'setup.inf'; @@ -30,15 +30,33 @@ export interface WindowsPackageInstallerSources { startWithConfiguration: boolean; }; -export class WindowsPackageInstallerCompiler { - private kmpCompiler: KmpCompiler; +export interface WindowsPackageInstallerCompilerOptions extends KmpCompilerOptions { + sources: WindowsPackageInstallerSources; +} - constructor(private callbacks: CompilerCallbacks) { - this.kmpCompiler = new KmpCompiler(this.callbacks); +export interface WindowsPackageInstallerCompilerArtifacts extends KeymanCompilerArtifacts { + exe: KeymanCompilerArtifact; +}; + +export interface WindowsPackageInstallerCompilerResult extends KeymanCompilerResult { + artifacts: WindowsPackageInstallerCompilerArtifacts; +}; + +export class WindowsPackageInstallerCompiler implements KeymanCompiler { + private kmpCompiler: KmpCompiler; + private callbacks: CompilerCallbacks; + private options: WindowsPackageInstallerCompilerOptions; + + async init(callbacks: CompilerCallbacks, options: WindowsPackageInstallerCompilerOptions): Promise { + this.callbacks = callbacks; + this.options = {...options}; + this.kmpCompiler = new KmpCompiler(); + return await this.kmpCompiler.init(callbacks, options); } - public async compile(kpsFilename: string, sources: WindowsPackageInstallerSources): Promise { - const kps = this.kmpCompiler.loadKpsFile(kpsFilename); + public async run(inputFilename: string, outputFilename?: string): Promise { + const sources = this.options.sources; + const kps = this.kmpCompiler.loadKpsFile(inputFilename); if(!kps) { // errors will already have been reported by loadKpsFile return null; @@ -64,7 +82,7 @@ export class WindowsPackageInstallerCompiler { // Nor do we use the MSIOptions field. // Build the zip - const zipBuffer = await this.buildZip(kps, kpsFilename, sources); + const zipBuffer = await this.buildZip(kps, inputFilename, sources); if(!zipBuffer) { // Error messages already reported by buildZip return null; @@ -72,7 +90,22 @@ export class WindowsPackageInstallerCompiler { // Build the sfx const sfxBuffer = this.buildSfx(zipBuffer, sources); - return sfxBuffer; + + const result: WindowsPackageInstallerCompilerResult = { + artifacts: { + exe: { + data: sfxBuffer, + filename: outputFilename ?? inputFilename.replace(/\.kps$/, '.exe') + } + } + }; + + return result; + } + + public async write(artifacts: WindowsPackageInstallerCompilerArtifacts): Promise { + this.callbacks.fs.writeFileSync(artifacts.exe.filename, artifacts.exe.data); + return true; } private async buildZip(kps: KpsFile.KpsFile, kpsFilename: string, sources: WindowsPackageInstallerSources): Promise { diff --git a/developer/src/kmc-package/test/fixtures/invalid/example.qaa.sencoten.model.js b/developer/src/kmc-package/test/fixtures/invalid/example.qaa.sencoten.model.js new file mode 100644 index 0000000000..2f309acbc0 --- /dev/null +++ b/developer/src/kmc-package/test/fixtures/invalid/example.qaa.sencoten.model.js @@ -0,0 +1 @@ +// dummy file for unit tests \ No newline at end of file diff --git a/developer/src/kmc-package/test/fixtures/invalid/keyman.exe b/developer/src/kmc-package/test/fixtures/invalid/keyman.exe new file mode 100644 index 0000000000..789aa9641a --- /dev/null +++ b/developer/src/kmc-package/test/fixtures/invalid/keyman.exe @@ -0,0 +1 @@ +This is a dummy file for testing the unit tests \ No newline at end of file diff --git a/developer/src/kmc-package/test/fixtures/invalid/khmer_angkor.docx b/developer/src/kmc-package/test/fixtures/invalid/khmer_angkor.docx new file mode 100644 index 0000000000..0f8c58ac58 --- /dev/null +++ b/developer/src/kmc-package/test/fixtures/invalid/khmer_angkor.docx @@ -0,0 +1 @@ +This is a sample text file \ No newline at end of file diff --git a/developer/src/kmc-package/test/test-messages.ts b/developer/src/kmc-package/test/test-messages.ts index 24fb658b59..d7a7f0175e 100644 --- a/developer/src/kmc-package/test/test-messages.ts +++ b/developer/src/kmc-package/test/test-messages.ts @@ -4,7 +4,6 @@ import { TestCompilerCallbacks, verifyCompilerMessagesObject } from '@keymanapp/ import { CompilerMessages } from '../src/compiler/messages.js'; import { makePathToFixture } from './helpers/index.js'; import { KmpCompiler } from '../src/compiler/kmp-compiler.js'; -import { PackageValidation } from '../src/compiler/package-validation.js'; import { CompilerErrorNamespace, CompilerOptions } from '@keymanapp/common-types'; const debug = false; @@ -20,24 +19,16 @@ describe('CompilerMessages', function () { // Message tests // - function testForMessage(context: Mocha.Context, fixture: string[], messageId?: number, options?: CompilerOptions) { + async function testForMessage(context: Mocha.Context, fixture: string[], messageId?: number, options?: CompilerOptions) { context.timeout(10000); callbacks.clear(); const kpsPath = makePathToFixture(...fixture); - const kmpCompiler = new KmpCompiler(callbacks); + const kmpCompiler = new KmpCompiler(); + assert.isTrue(await kmpCompiler.init(callbacks, options ?? {})); - let kmpJson = kmpCompiler.transformKpsToKmpObject(kpsPath); - if(kmpJson && callbacks.messages.length == 0) { - const validator = new PackageValidation(callbacks, options ?? {}); - validator.validate(kpsPath, kmpJson); // we'll ignore return value and rely on the messages - } - - if(kmpJson && callbacks.messages.length == 0) { - // We'll try building the package if we have not yet received any messages - kmpCompiler.buildKmpFile(kpsPath, kmpJson) - } + await kmpCompiler.run(kpsPath); if(debug) callbacks.printMessages(); @@ -52,106 +43,106 @@ describe('CompilerMessages', function () { // WARN_FileIsNotABinaryKvkFile it('should generate WARN_FileIsNotABinaryKvkFile if a non-binary kvk file is included', async function() { - testForMessage(this, ['xml_kvk_file', 'source', 'xml_kvk_file.kps'], CompilerMessages.WARN_FileIsNotABinaryKvkFile); + await testForMessage(this, ['xml_kvk_file', 'source', 'xml_kvk_file.kps'], CompilerMessages.WARN_FileIsNotABinaryKvkFile); }); it('should not warn if a binary kvk file is included', async function() { - testForMessage(this, ['binary_kvk_file', 'source', 'binary_kvk_file.kps']); + await testForMessage(this, ['binary_kvk_file', 'source', 'binary_kvk_file.kps']); }); // ERROR_FollowKeyboardVersionNotAllowedForModelPackages it('should generate ERROR_FollowKeyboardVersionNotAllowedForModelPackages if is set for model packages', async function() { - testForMessage(this, ['invalid', 'followkeyboardversion.qaa.sencoten.model.kps'], CompilerMessages.ERROR_FollowKeyboardVersionNotAllowedForModelPackages); + await testForMessage(this, ['invalid', 'followkeyboardversion.qaa.sencoten.model.kps'], CompilerMessages.ERROR_FollowKeyboardVersionNotAllowedForModelPackages); }); // ERROR_FollowKeyboardVersionButNoKeyboards it('should generate ERROR_FollowKeyboardVersionButNoKeyboards if is set for a package with no keyboards', async function() { - testForMessage(this, ['invalid', 'followkeyboardversion.empty.kps'], CompilerMessages.ERROR_FollowKeyboardVersionButNoKeyboards); + await testForMessage(this, ['invalid', 'followkeyboardversion.empty.kps'], CompilerMessages.ERROR_FollowKeyboardVersionButNoKeyboards); }); // ERROR_KeyboardContentFileNotFound it('should generate ERROR_KeyboardContentFileNotFound if a is listed in a package but not found in ', async function() { - testForMessage(this, ['invalid', 'keyboardcontentfilenotfound.kps'], CompilerMessages.ERROR_KeyboardContentFileNotFound); + await testForMessage(this, ['invalid', 'keyboardcontentfilenotfound.kps'], CompilerMessages.ERROR_KeyboardContentFileNotFound); }); // ERROR_KeyboardFileNotValid it('should generate ERROR_KeyboardFileNotValid if a .kmx is not valid in ', async function() { - testForMessage(this, ['invalid', 'keyboardfilenotvalid.kps'], CompilerMessages.ERROR_KeyboardFileNotValid); + await testForMessage(this, ['invalid', 'keyboardfilenotvalid.kps'], CompilerMessages.ERROR_KeyboardFileNotValid); }); // INFO_KeyboardFileHasNoKeyboardVersion it('should generate INFO_KeyboardFileHasNoKeyboardVersion if is set but keyboard has no version', async function() { - testForMessage(this, ['invalid', 'nokeyboardversion.kps'], CompilerMessages.INFO_KeyboardFileHasNoKeyboardVersion); + await testForMessage(this, ['invalid', 'nokeyboardversion.kps'], CompilerMessages.INFO_KeyboardFileHasNoKeyboardVersion); }); // ERROR_PackageCannotContainBothModelsAndKeyboards it('should generate ERROR_PackageCannotContainBothModelsAndKeyboards if package has both keyboards and models', async function() { - testForMessage(this, ['invalid', 'error_package_cannot_contain_both_models_and_keyboards.kps'], CompilerMessages.ERROR_PackageCannotContainBothModelsAndKeyboards); + await testForMessage(this, ['invalid', 'error_package_cannot_contain_both_models_and_keyboards.kps'], CompilerMessages.ERROR_PackageCannotContainBothModelsAndKeyboards); }); // HINT_PackageShouldNotRepeatLanguages (models) it('should generate HINT_PackageShouldNotRepeatLanguages if model has same language repeated', async function() { - testForMessage(this, ['invalid', 'keyman.en.hint_package_should_not_repeat_languages.model.kps'], CompilerMessages.HINT_PackageShouldNotRepeatLanguages); + await testForMessage(this, ['invalid', 'keyman.en.hint_package_should_not_repeat_languages.model.kps'], CompilerMessages.HINT_PackageShouldNotRepeatLanguages); }); // HINT_PackageShouldNotRepeatLanguages (keyboards) it('should generate HINT_PackageShouldNotRepeatLanguages if keyboard has same language repeated', async function() { - testForMessage(this, ['invalid', 'hint_package_should_not_repeat_languages.kps'], CompilerMessages.HINT_PackageShouldNotRepeatLanguages); + await testForMessage(this, ['invalid', 'hint_package_should_not_repeat_languages.kps'], CompilerMessages.HINT_PackageShouldNotRepeatLanguages); }); // WARN_PackageNameDoesNotFollowLexicalModelConventions it('should generate WARN_PackageNameDoesNotFollowLexicalModelConventions if filename has wrong conventions', async function() { - testForMessage(this, ['invalid', 'WARN_PackageNameDoesNotFollowLexicalModelConventions.kps'], CompilerMessages.WARN_PackageNameDoesNotFollowLexicalModelConventions); + await testForMessage(this, ['invalid', 'WARN_PackageNameDoesNotFollowLexicalModelConventions.kps'], CompilerMessages.WARN_PackageNameDoesNotFollowLexicalModelConventions); }); // WARN_PackageNameDoesNotFollowKeyboardConventions it('should generate WARN_PackageNameDoesNotFollowKeyboardConventions if filename has wrong conventions', async function() { - testForMessage(this, ['invalid', 'WARN_PackageNameDoesNotFollowKeyboardConventions.kps'], CompilerMessages.WARN_PackageNameDoesNotFollowKeyboardConventions); + await testForMessage(this, ['invalid', 'WARN_PackageNameDoesNotFollowKeyboardConventions.kps'], CompilerMessages.WARN_PackageNameDoesNotFollowKeyboardConventions); }); // WARN_FileInPackageDoesNotFollowFilenameConventions it('should generate WARN_FileInPackageDoesNotFollowFilenameConventions if content filename has wrong conventions', async function() { - testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions.kps'], + await testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions.kps'], CompilerMessages.WARN_FileInPackageDoesNotFollowFilenameConventions, {checkFilenameConventions: true}); - testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions_2.kps'], + await testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions_2.kps'], CompilerMessages.WARN_FileInPackageDoesNotFollowFilenameConventions, {checkFilenameConventions: true}); }); // Test the inverse -- no warning generated if checkFilenameConventions is false it('should not generate WARN_FileInPackageDoesNotFollowFilenameConventions if content filename has wrong conventions but checkFilenameConventions is false', async function() { - testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions.kps'], null, {checkFilenameConventions: false}); - testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions_2.kps'], null, {checkFilenameConventions: false}); + await testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions.kps'], null, {checkFilenameConventions: false}); + await testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions_2.kps'], null, {checkFilenameConventions: false}); }); // ERROR_PackageNameCannotBeBlank it('should generate ERROR_PackageNameCannotBeBlank if package info has empty name', async function() { - testForMessage(this, ['invalid', 'error_package_name_cannot_be_blank.kps'], CompilerMessages.ERROR_PackageNameCannotBeBlank); // blank field - testForMessage(this, ['invalid', 'error_package_name_cannot_be_blank_2.kps'], CompilerMessages.ERROR_PackageNameCannotBeBlank); // missing field + await testForMessage(this, ['invalid', 'error_package_name_cannot_be_blank.kps'], CompilerMessages.ERROR_PackageNameCannotBeBlank); // blank field + await testForMessage(this, ['invalid', 'error_package_name_cannot_be_blank_2.kps'], CompilerMessages.ERROR_PackageNameCannotBeBlank); // missing field }); // ERROR_KeyboardFileNotFound it('should generate ERROR_KeyboardFileNotFound if a is listed in a package but not found in ', async function() { - testForMessage(this, ['invalid', 'keyboardfilenotfound.kps'], CompilerMessages.ERROR_KeyboardFileNotFound); + await testForMessage(this, ['invalid', 'keyboardfilenotfound.kps'], CompilerMessages.ERROR_KeyboardFileNotFound); }); // WARN_KeyboardVersionsDoNotMatch it('should generate WARN_KeyboardVersionsDoNotMatch if two have different versions', async function() { - testForMessage(this, ['invalid', 'warn_keyboard_versions_do_not_match.kps'], CompilerMessages.WARN_KeyboardVersionsDoNotMatch); + await testForMessage(this, ['invalid', 'warn_keyboard_versions_do_not_match.kps'], CompilerMessages.WARN_KeyboardVersionsDoNotMatch); }); // ERROR_LanguageTagIsNotValid @@ -163,73 +154,73 @@ describe('CompilerMessages', function () { // HINT_LanguageTagIsNotMinimal it('should generate HINT_LanguageTagIsNotMinimal if keyboard has a non-minimal language tag', async function() { - testForMessage(this, ['invalid', 'hint_language_tag_is_not_minimal.kps'], CompilerMessages.HINT_LanguageTagIsNotMinimal); + await testForMessage(this, ['invalid', 'hint_language_tag_is_not_minimal.kps'], CompilerMessages.HINT_LanguageTagIsNotMinimal); }); // ERROR_ModelMustHaveAtLeastOneLanguage it('should generate ERROR_MustHaveAtLeastOneLanguage if model has zero language tags', async function() { - testForMessage(this, ['invalid', 'keyman.en.error_model_must_have_at_least_one_language.model.kps'], + await testForMessage(this, ['invalid', 'keyman.en.error_model_must_have_at_least_one_language.model.kps'], CompilerMessages.ERROR_ModelMustHaveAtLeastOneLanguage); }); // WARN_RedistFileShouldNotBeInPackage it('should generate WARN_RedistFileShouldNotBeInPackage if package contains a redist file', async function() { - testForMessage(this, ['invalid', 'warn_redist_file_should_not_be_in_package.kps'], + await testForMessage(this, ['invalid', 'warn_redist_file_should_not_be_in_package.kps'], CompilerMessages.WARN_RedistFileShouldNotBeInPackage); }); // WARN_DocFileDangerous it('should generate WARN_DocFileDangerous if package contains a .doc file', async function() { - testForMessage(this, ['invalid', 'warn_doc_file_dangerous.kps'], + await testForMessage(this, ['invalid', 'warn_doc_file_dangerous.kps'], CompilerMessages.WARN_DocFileDangerous); }); // ERROR_PackageMustContainAPackageOrAKeyboard it('should generate ERROR_PackageMustContainAModelOrAKeyboard if package contains no keyboard or model', async function() { - testForMessage(this, ['invalid', 'error_package_must_contain_a_model_or_a_keyboard.kps'], + await testForMessage(this, ['invalid', 'error_package_must_contain_a_model_or_a_keyboard.kps'], CompilerMessages.ERROR_PackageMustContainAModelOrAKeyboard); }); // WARN_JsKeyboardFileIsMissing it('should generate WARN_JsKeyboardFileIsMissing if package is missing corresponding .js for a touch .kmx', async function() { - testForMessage(this, ['invalid', 'warn_js_keyboard_file_is_missing.kps'], + await testForMessage(this, ['invalid', 'warn_js_keyboard_file_is_missing.kps'], CompilerMessages.WARN_JsKeyboardFileIsMissing); }); // WARN_KeyboardShouldHaveAtLeastOneLanguage it('should generate WARN_KeyboardShouldHaveAtLeastOneLanguage if keyboard has zero language tags', async function() { - testForMessage(this, ['invalid', 'warn_keyboard_should_have_at_least_one_language.kps'], + await testForMessage(this, ['invalid', 'warn_keyboard_should_have_at_least_one_language.kps'], CompilerMessages.WARN_KeyboardShouldHaveAtLeastOneLanguage); }); // HINT_JsKeyboardFileHasNoTouchTargets it('should generate HINT_JsKeyboardFileHasNoTouchTargets if keyboard has no touch targets', async function() { - testForMessage(this, ['invalid', 'hint_js_keyboard_file_has_no_touch_targets.kps'], + await testForMessage(this, ['invalid', 'hint_js_keyboard_file_has_no_touch_targets.kps'], CompilerMessages.HINT_JsKeyboardFileHasNoTouchTargets); }); it('should not generate HINT_JsKeyboardFileHasNoTouchTargets if keyboard has a touch target', async function() { - testForMessage(this, ['khmer_angkor', 'source', 'khmer_angkor.kps'], null); + await testForMessage(this, ['khmer_angkor', 'source', 'khmer_angkor.kps'], null); }); // HINT_PackageContainsSourceFile it('should generate HINT_PackageContainsSourceFile if package contains a source file', async function() { - testForMessage(this, ['invalid', 'hint_source_file_should_not_be_in_package.kps'], + await testForMessage(this, ['invalid', 'hint_source_file_should_not_be_in_package.kps'], CompilerMessages.HINT_PackageContainsSourceFile); }); // ERROR_InvalidPackageFile it('should generate ERROR_InvalidPackageFile if package source file contains invalid XML', async function() { - testForMessage(this, ['invalid', 'error_invalid_package_file.kps'], + await testForMessage(this, ['invalid', 'error_invalid_package_file.kps'], CompilerMessages.ERROR_InvalidPackageFile); }); diff --git a/developer/src/kmc-package/test/test-package-compiler.ts b/developer/src/kmc-package/test/test-package-compiler.ts index 433d730c18..3a17d321eb 100644 --- a/developer/src/kmc-package/test/test-package-compiler.ts +++ b/developer/src/kmc-package/test/test-package-compiler.ts @@ -15,14 +15,15 @@ import { CompilerMessages } from '../src/compiler/messages.js'; const debug = false; -describe('KmpCompiler', function () { +describe('KmpCompiler', async function () { const MODELS : string[] = [ 'example.qaa.sencoten', 'withfolders.qaa.sencoten', ]; const callbacks = new TestCompilerCallbacks(); - let kmpCompiler = new KmpCompiler(callbacks); + let kmpCompiler = new KmpCompiler(); + assert.isTrue(await kmpCompiler.init(callbacks, null)); for (let modelID of MODELS) { const kpsPath = modelID.includes('withfolders') ? @@ -89,7 +90,9 @@ describe('KmpCompiler', function () { const kpsPath = makePathToFixture('khmer_angkor', 'source', 'khmer_angkor.kps'); const kmpJsonRefPath = makePathToFixture('khmer_angkor', 'ref', 'kmp.json'); - const kmpCompiler = new KmpCompiler(callbacks); + const kmpCompiler = new KmpCompiler(); + assert.isTrue(await kmpCompiler.init(callbacks, null)); + const kmpJsonFixture: KmpJsonFile.KmpJsonFile = JSON.parse(fs.readFileSync(kmpJsonRefPath, 'utf-8')); // We override the fixture version so that we can compare with the compiler output @@ -193,7 +196,8 @@ describe('KmpCompiler', function () { callbacks.clear(); const kpsPath = makePathToFixture('absolute_path', 'source', 'absolute_path.kps'); - const kmpCompiler = new KmpCompiler(callbacks); + const kmpCompiler = new KmpCompiler(); + assert.isTrue(await kmpCompiler.init(callbacks, null)); let kmpJson: KmpJsonFile.KmpJsonFile = null; diff --git a/developer/src/kmc-package/test/test-versioning.ts b/developer/src/kmc-package/test/test-versioning.ts index e9771a369a..98d00b853c 100644 --- a/developer/src/kmc-package/test/test-versioning.ts +++ b/developer/src/kmc-package/test/test-versioning.ts @@ -33,7 +33,9 @@ describe('package versioning', function () { for(const [ caseTitle, filename ] of cases) { it(caseTitle, async function () { const callbacks = new TestCompilerCallbacks(); - const kmpCompiler = new KmpCompiler(callbacks); + const kmpCompiler = new KmpCompiler(); + assert.isTrue(await kmpCompiler.init(callbacks, null)); + const kpsPath = makePathToFixture('versioning', filename); const kmpJson: KmpJsonFile.KmpJsonFile = kmpCompiler.transformKpsToKmpObject(kpsPath); assert.isTrue(kmpJson !== null); diff --git a/developer/src/kmc-package/test/test-windows-package-installer-compiler.ts b/developer/src/kmc-package/test/test-windows-package-installer-compiler.ts index 8ed2fee0e7..e1fde65647 100644 --- a/developer/src/kmc-package/test/test-windows-package-installer-compiler.ts +++ b/developer/src/kmc-package/test/test-windows-package-installer-compiler.ts @@ -12,9 +12,6 @@ describe('WindowsPackageInstallerCompiler', function () { it(`should build an SFX archive`, async function () { this.timeout(10000); // this test can take a little while to run - const callbacks = new TestCompilerCallbacks(); - let compiler = new WindowsPackageInstallerCompiler(callbacks); - const kpsPath = makePathToFixture('khmer_angkor', 'source', 'khmer_angkor.kps'); const sources: WindowsPackageInstallerSources = { licenseFilename: makePathToFixture('windows-installer', 'license.txt'), @@ -25,12 +22,19 @@ describe('WindowsPackageInstallerCompiler', function () { appName: 'Testing', }; - const sfxBuffer = await compiler.compile(kpsPath, sources); + const callbacks = new TestCompilerCallbacks(); + let compiler = new WindowsPackageInstallerCompiler(); + assert.isTrue(await compiler.init(callbacks, {sources})); + + const result = await compiler.run(kpsPath, null); + assert.isNotNull(result); // This returns a buffer with a SFX loader and a zip suffix. For the sake of repository size // we actually provide a stub SFX loader and a stub MSI file, which is enough to verify that // the compiler is generating what it thinks is a valid file. + const sfxBuffer = result.artifacts.exe.data; + const zip = JSZip(); // Check that file.kmp contains just 3 files - setup.inf, keymandesktop.msi, and khmer_angkor.kmp, diff --git a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts index bddfcc38db..fe82a2185e 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts @@ -50,7 +50,12 @@ export class BuildModelInfo extends BuildActivity { return false; } - let kmpCompiler = new KmpCompiler(callbacks); + let kmpCompiler = new KmpCompiler(); + if(!await kmpCompiler.init(callbacks, options)) { + // Errors will have been emitted by KmpCompiler + return false; + } + let kmpJsonData = kmpCompiler.transformKpsToKmpObject(project.resolveInputFilePath(kps)); if(!kmpJsonData) { // Errors will have been emitted by KmpCompiler diff --git a/developer/src/kmc/src/commands/buildClasses/BuildPackage.ts b/developer/src/kmc/src/commands/buildClasses/BuildPackage.ts index 8adcd4a92c..12593b0f8e 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildPackage.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildPackage.ts @@ -1,7 +1,6 @@ -import * as fs from 'fs'; import { BuildActivity } from './BuildActivity.js'; import { CompilerCallbacks, CompilerOptions, KeymanFileTypes } from '@keymanapp/common-types'; -import { KmpCompiler, PackageValidation } from '@keymanapp/kmc-package'; +import { KmpCompiler } from '@keymanapp/kmc-package'; export class BuildPackage extends BuildActivity { public get name(): string { return 'Package'; } @@ -12,35 +11,16 @@ export class BuildPackage extends BuildActivity { const outfile = this.getOutputFilename(infile, options); - // - // Load .kps source data - // - - const kmpCompiler = new KmpCompiler(callbacks); - const kmpJsonData = kmpCompiler.transformKpsToKmpObject(infile); - if(!kmpJsonData) { + const kmpCompiler = new KmpCompiler(); + if(!await kmpCompiler.init(callbacks, options)) { return false; } - // - // Validate the package file - // - - const validation = new PackageValidation(callbacks, options); - if(!validation.validate(infile, kmpJsonData)) { + const result = await kmpCompiler.run(infile, outfile); + if(!result) { return false; } - // - // Build the .kmp package file - // - - const data = await kmpCompiler.buildKmpFile(infile, kmpJsonData); - if(!data) { - return false; - } - - fs.writeFileSync(outfile, data, 'binary'); - return true; + return await kmpCompiler.write(result.artifacts); } } diff --git a/developer/src/kmc/src/commands/buildWindowsPackageInstaller/index.ts b/developer/src/kmc/src/commands/buildWindowsPackageInstaller/index.ts index 90bd30ce7f..6f55970838 100644 --- a/developer/src/kmc/src/commands/buildWindowsPackageInstaller/index.ts +++ b/developer/src/kmc/src/commands/buildWindowsPackageInstaller/index.ts @@ -4,7 +4,7 @@ import { CompilerBaseOptions, CompilerCallbacks, defaultCompilerOptions } from ' import { NodeCompilerCallbacks } from '../../util/NodeCompilerCallbacks.js'; import { WindowsPackageInstallerCompiler, WindowsPackageInstallerSources } from '@keymanapp/kmc-package'; -interface WindowsPackageInstallerOptions extends CompilerBaseOptions { +interface WindowsPackageInstallerCommandLineOptions extends CompilerBaseOptions { msi: string; exe: string; license: string; @@ -15,7 +15,9 @@ interface WindowsPackageInstallerOptions extends CompilerBaseOptions { }; export async function buildWindowsPackageInstaller(infile: string, _options: any, commander: any) { - const options: WindowsPackageInstallerOptions = commander.optsWithGlobals(); + // TODO(lowpri): we probably should cleanup the options management here, move + // translation of command line options to kmc-* options into a separate module + const options: WindowsPackageInstallerCommandLineOptions = commander.optsWithGlobals(); const sources: WindowsPackageInstallerSources = { licenseFilename: options.license, msiFilename: options.msi, @@ -31,11 +33,8 @@ export async function buildWindowsPackageInstaller(infile: string, _options: any infile = fs.realpathSync.native(infile); const callbacks: CompilerCallbacks = new NodeCompilerCallbacks({...defaultCompilerOptions, ...options}); - const compiler = new WindowsPackageInstallerCompiler(callbacks); - - const buffer = await compiler.compile(infile, sources); - if(!buffer) { - // errors will have been reported already + const compiler = new WindowsPackageInstallerCompiler(); + if(!await compiler.init(callbacks, {...options, sources})) { process.exit(1); } @@ -43,5 +42,14 @@ export async function buildWindowsPackageInstaller(infile: string, _options: any const outFileBase = path.basename(fileBaseName, path.extname(fileBaseName)); const outFileDir = path.dirname(fileBaseName); const outFileExe = path.join(outFileDir, outFileBase + '.exe'); - fs.writeFileSync(outFileExe, buffer); + + const result = await compiler.run(infile, outFileExe); + if(!result) { + // errors will have been reported already + process.exit(1); + } + + if(!await compiler.write(result.artifacts)) { + process.exit(1); + } } diff --git a/developer/src/kmc/src/kmlmp.ts b/developer/src/kmc/src/kmlmp.ts index ae629f073e..bb0cda5e53 100644 --- a/developer/src/kmc/src/kmlmp.ts +++ b/developer/src/kmc/src/kmlmp.ts @@ -5,9 +5,8 @@ // Note: this is a deprecated package and will be removed in Keyman 18.0 -import * as fs from 'fs'; import { Command } from 'commander'; -import { PackageValidation, KmpCompiler } from '@keymanapp/kmc-package'; +import { KmpCompiler } from '@keymanapp/kmc-package'; import { SysExits } from './util/sysexits.js'; import KEYMAN_VERSION from "@keymanapp/keyman-version"; import { NodeCompilerCallbacks } from './util/NodeCompilerCallbacks.js'; @@ -34,36 +33,24 @@ if (!inputFilename) { let outputFilename: string = program.opts().outFile ? program.opts().outFile : inputFilename.replace(/\.kps$/, ".kmp"); // -// Load .kps source data +// Run the compiler // const callbacks = new NodeCompilerCallbacks({logLevel: 'info'}); -let kmpCompiler = new KmpCompiler(callbacks); -let kmpJsonData = kmpCompiler.transformKpsToKmpObject(inputFilename); -if(!kmpJsonData) { +let kmpCompiler = new KmpCompiler(); +if(!await kmpCompiler.init(callbacks, null)) { process.exit(1); } -// -// Validate the package file -// - -const validation = new PackageValidation(callbacks, {}); -if(!validation.validate(inputFilename, kmpJsonData)) { +let result = await kmpCompiler.run(inputFilename, outputFilename); +if(!result) { process.exit(1); } -// -// Build the .kmp package file -// - -let promise = kmpCompiler.buildKmpFile(inputFilename, kmpJsonData); -promise.then(data => { - fs.writeFileSync(outputFilename, data, 'binary'); -}).catch(error => { - // Consumer decides how to handle errors +if(!await kmpCompiler.write(result.artifacts)) { console.error('Failed to write kmp file'); -}); + process.exit(1); +} function exitDueToUsageError(message: string): never { console.error(`${program.name()}: ${message}`); From 13533a726a9a1b715ab4f4b97ef23fc6bb31ea65 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Sat, 9 Dec 2023 07:15:06 +0700 Subject: [PATCH 06/14] feat(developer): KeyboardInfoCompiler now implements KeymanCompiler Relates to #9473. Refactors the public API of KeyboardInfoCompiler to meet KeymanCompiler, including moving file write responsibilities into the class itself. `sources` has become a property of `options`, which is perhaps a slight violation of the intent of the `init()` function being a one-time call, as it means we'll need to instantiate an new class for each file we compile. Given the use-case for the keyboard_info compiler is almost exclusively internal, I think this is acceptable. Tell me if you think otherwise! --- developer/src/kmc-keyboard-info/src/index.ts | 53 +++++++++++++++---- .../test/test-keyboard-info-compiler.ts | 30 ++++++----- .../buildClasses/BuildKeyboardInfo.ts | 22 ++++---- 3 files changed, 71 insertions(+), 34 deletions(-) diff --git a/developer/src/kmc-keyboard-info/src/index.ts b/developer/src/kmc-keyboard-info/src/index.ts index 7d134e4acb..7ee9091610 100644 --- a/developer/src/kmc-keyboard-info/src/index.ts +++ b/developer/src/kmc-keyboard-info/src/index.ts @@ -5,7 +5,7 @@ import { minKeymanVersion } from "./min-keyman-version.js"; import { KeyboardInfoFile, KeyboardInfoFileIncludes, KeyboardInfoFileLanguageFont, KeyboardInfoFilePlatform } from "./keyboard-info-file.js"; -import { KeymanFileTypes, CompilerCallbacks, KmpJsonFile, KmxFileReader, KMX, KeymanTargets } from "@keymanapp/common-types"; +import { KeymanFileTypes, CompilerCallbacks, KmpJsonFile, KmxFileReader, KMX, KeymanTargets, KeymanCompiler, CompilerOptions, KeymanCompilerResult, KeymanCompilerArtifacts, KeymanCompilerArtifact } from "@keymanapp/common-types"; import { KeyboardInfoCompilerMessages } from "./messages.js"; import langtags from "./imports/langtags.js"; import { validateMITLicense } from "@keymanapp/developer-utils"; @@ -24,7 +24,7 @@ const HelpRoot = 'https://help.keyman.com/keyboard/'; * Build a dictionary of language tags from langtags.json */ -function init(): void { +function preinit(): void { if(langtagsByTag['en']) { // Already initialized, we can reasonably assume that 'en' will always be in // langtags.json. @@ -62,9 +62,30 @@ export interface KeyboardInfoSources { forPublishing: boolean; }; -export class KeyboardInfoCompiler { - constructor(private callbacks: CompilerCallbacks) { - init(); +export interface KeyboardInfoCompilerOptions extends CompilerOptions { + sources: KeyboardInfoSources; +}; + +export interface KeyboardInfoCompilerArtifacts extends KeymanCompilerArtifacts { + keyboard_info: KeymanCompilerArtifact; +}; + +export interface KeyboardInfoCompilerResult extends KeymanCompilerResult { + artifacts: KeyboardInfoCompilerArtifacts; +}; + +export class KeyboardInfoCompiler implements KeymanCompiler { + private callbacks: CompilerCallbacks; + private options: KeyboardInfoCompilerOptions; + + constructor() { + preinit(); + } + + public async init(callbacks: CompilerCallbacks, options: KeyboardInfoCompilerOptions): Promise { + this.callbacks = callbacks; + this.options = {...options}; + return true; } /** @@ -77,9 +98,8 @@ export class KeyboardInfoCompiler { * * @param sources Details on files from which to extract metadata */ - public async writeKeyboardInfoFile( - sources: KeyboardInfoSources - ): Promise { + public async run(inputFilename: string, outputFilename?: string): Promise { + const sources = this.options.sources; // TODO(lowpri): work from .kpj and nothing else as input. Blocked because // .kpj work is largely in kmc at present, so that would need to move to @@ -311,7 +331,22 @@ export class KeyboardInfoCompiler { }, null, 2)); } - return new TextEncoder().encode(jsonOutput); + const data = new TextEncoder().encode(jsonOutput); + const result: KeyboardInfoCompilerResult = { + artifacts: { + keyboard_info: { + data, + filename: outputFilename ?? inputFilename.replace(/\.kpj$/, '.keyboard_info') + } + } + }; + + return result; + } + + public async write(artifacts: KeyboardInfoCompilerArtifacts): Promise { + this.callbacks.fs.writeFileSync(artifacts.keyboard_info.filename, artifacts.keyboard_info.data); + return true; } private mapKeymanTargetToPlatform(target: KeymanTargets.KeymanTarget): KeyboardInfoFilePlatform[] { diff --git a/developer/src/kmc-keyboard-info/test/test-keyboard-info-compiler.ts b/developer/src/kmc-keyboard-info/test/test-keyboard-info-compiler.ts index 40ff764285..eab1202c8c 100644 --- a/developer/src/kmc-keyboard-info/test/test-keyboard-info-compiler.ts +++ b/developer/src/kmc-keyboard-info/test/test-keyboard-info-compiler.ts @@ -3,7 +3,7 @@ import { assert } from 'chai'; import 'mocha'; import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers'; import { makePathToFixture } from './helpers/index.js'; -import { KeyboardInfoCompiler } from '../src/index.js'; +import { KeyboardInfoCompiler, KeyboardInfoCompilerResult } from '../src/index.js'; const callbacks = new TestCompilerCallbacks(); @@ -13,31 +13,35 @@ beforeEach(function() { describe('keyboard-info-compiler', function () { it('compile a .keyboard_info file correctly', async function() { + const kpjFilename = makePathToFixture('khmer_angkor', 'khmer_angkor.kpj'); const jsFilename = makePathToFixture('khmer_angkor', 'build', 'khmer_angkor.js'); const kpsFilename = makePathToFixture('khmer_angkor', 'source', 'khmer_angkor.kps'); const kmpFilename = makePathToFixture('khmer_angkor', 'build', 'khmer_angkor.kmp'); const buildKeyboardInfoFilename = makePathToFixture('khmer_angkor', 'build', 'khmer_angkor.keyboard_info'); - const compiler = new KeyboardInfoCompiler(callbacks); - let data = null; + const sources = { + kmpFilename, + sourcePath: 'release/k/khmer_angkor', + kpsFilename, + jsFilename: jsFilename, + forPublishing: true, + }; + + const compiler = new KeyboardInfoCompiler(); + assert.isTrue(await compiler.init(callbacks, {sources})); + let result: KeyboardInfoCompilerResult = null; try { - data = await compiler.writeKeyboardInfoFile({ - kmpFilename, - sourcePath: 'release/k/khmer_angkor', - kpsFilename, - jsFilename: jsFilename, - forPublishing: true, - }); + result = await compiler.run(kpjFilename, null); } catch(e) { callbacks.printMessages(); throw e; } - if(data == null) { + if(result == null) { callbacks.printMessages(); } - assert.isNotNull(data); + assert.isNotNull(result); - const actual = JSON.parse(new TextDecoder().decode(data)); + const actual = JSON.parse(new TextDecoder().decode(result.artifacts.keyboard_info.data)); const expected = JSON.parse(fs.readFileSync(buildKeyboardInfoFilename, 'utf-8')); // `lastModifiedDate` is dependent on time of run (not worth mocking) diff --git a/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts b/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts index 936c358564..6ed023a139 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts @@ -35,29 +35,27 @@ export class BuildKeyboardInfo extends BuildActivity { const keyboard = project.files.find(file => file.fileType == KeymanFileTypes.Source.KeymanKeyboard); const jsFilename = keyboard ? project.resolveOutputFilePath(keyboard, KeymanFileTypes.Source.KeymanKeyboard, KeymanFileTypes.Binary.WebKeyboard) : null; const lastCommitDate = getLastGitCommitDate(project.projectPath); - - const compiler = new KeyboardInfoCompiler(callbacks); - const data = await compiler.writeKeyboardInfoFile({ + const sources = { kmpFilename: project.resolveOutputFilePath(kps, KeymanFileTypes.Source.Package, KeymanFileTypes.Binary.Package), kpsFilename: project.resolveInputFilePath(kps), jsFilename: jsFilename && fs.existsSync(jsFilename) ? jsFilename : undefined, sourcePath: calculateSourcePath(infile), lastCommitDate, forPublishing: !!options.forPublishing, - }); + }; + + const compiler = new KeyboardInfoCompiler(); + if(!await compiler.init(callbacks, {sources})) { + return false; + } + const outputFilename = project.getOutputFilePath(KeymanFileTypes.Binary.KeyboardInfo); + const data = await compiler.run(infile, outputFilename); if(data == null) { // Error messages have already been emitted by KeyboardInfoCompiler return false; } - const outputFilename = project.getOutputFilePath(KeymanFileTypes.Binary.KeyboardInfo); - - fs.writeFileSync( - outputFilename, - data - ); - - return true; + return await compiler.write(data.artifacts); } } From 7b6b4f0add1d687a2e8dc8e00ee52fb43d416c0e Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Sat, 9 Dec 2023 07:28:26 +0700 Subject: [PATCH 07/14] feat(developer): ModelInfoCompiler now implements KeymanCompiler Relates to #9473. Refactors the public API of ModelInfoCompiler to meet KeymanCompiler, including moving file write responsibilities into the class itself. `sources` has become a property of `options`, which is perhaps a slight violation of the intent of the `init()` function being a one-time call, as it means we'll need to instantiate an new class for each file we compile. Given the use-case for the model_info compiler is almost exclusively internal, I think this is acceptable. Tell me if you think otherwise! --- .../kmc-model-info/src/model-info-compiler.ts | 50 ++++++++++++++++--- .../test/test-model-info-compiler.ts | 19 ++++--- .../commands/buildClasses/BuildModelInfo.ts | 22 ++++---- 3 files changed, 66 insertions(+), 25 deletions(-) diff --git a/developer/src/kmc-model-info/src/model-info-compiler.ts b/developer/src/kmc-model-info/src/model-info-compiler.ts index 8a7101e86f..934ffe3200 100644 --- a/developer/src/kmc-model-info/src/model-info-compiler.ts +++ b/developer/src/kmc-model-info/src/model-info-compiler.ts @@ -5,7 +5,7 @@ import { minKeymanVersion } from "./min-keyman-version.js"; import { ModelInfoFile } from "./model-info-file.js"; -import { CompilerCallbacks, KmpJsonFile } from "@keymanapp/common-types"; +import { CompilerCallbacks, CompilerOptions, KeymanCompiler, KeymanCompilerArtifact, KeymanCompilerArtifacts, KeymanCompilerResult, KmpJsonFile } from "@keymanapp/common-types"; import { ModelInfoCompilerMessages } from "./messages.js"; import { validateMITLicense } from "@keymanapp/developer-utils"; @@ -39,8 +39,30 @@ export class ModelInfoSources { }; /* c8 ignore stop */ -export class ModelInfoCompiler { - constructor(private callbacks: CompilerCallbacks) { +export interface ModelInfoCompilerOptions extends CompilerOptions { + sources: ModelInfoSources; +}; + +export interface ModelInfoCompilerArtifacts extends KeymanCompilerArtifacts { + model_info: KeymanCompilerArtifact; +}; + +export interface ModelInfoCompilerResult extends KeymanCompilerResult { + artifacts: ModelInfoCompilerArtifacts; +}; + + +export class ModelInfoCompiler implements KeymanCompiler { + private callbacks: CompilerCallbacks; + private options: ModelInfoCompilerOptions; + + constructor() { + } + + public async init(callbacks: CompilerCallbacks, options: ModelInfoCompilerOptions): Promise { + this.callbacks = callbacks; + this.options = {...options}; + return true; } /** @@ -51,9 +73,8 @@ export class ModelInfoCompiler { * * @param sources Details on files from which to extract additional metadata */ - writeModelMetadataFile( - sources: ModelInfoSources - ): Uint8Array { + public async run(inputFilename: string, outputFilename?: string): Promise { + const sources = this.options.sources; /* * Model info looks like this: @@ -171,7 +192,22 @@ export class ModelInfoCompiler { } const jsonOutput = JSON.stringify(model_info, null, 2); - return new TextEncoder().encode(jsonOutput); + const data = new TextEncoder().encode(jsonOutput); + const result: ModelInfoCompilerResult = { + artifacts: { + model_info: { + data, + filename: outputFilename ?? inputFilename.replace(/\.kpj$/, '.model_info') + } + } + }; + + return result; + } + + public async write(artifacts: ModelInfoCompilerArtifacts): Promise { + this.callbacks.fs.writeFileSync(artifacts.model_info.filename, artifacts.model_info.data); + return true; } private isLicenseMIT(filename: string) { diff --git a/developer/src/kmc-model-info/test/test-model-info-compiler.ts b/developer/src/kmc-model-info/test/test-model-info-compiler.ts index d74a2f5c60..729eca3ac9 100644 --- a/developer/src/kmc-model-info/test/test-model-info-compiler.ts +++ b/developer/src/kmc-model-info/test/test-model-info-compiler.ts @@ -13,16 +13,18 @@ beforeEach(function() { }); describe('model-info-compiler', function () { - it('compile a .model_info file correctly', function() { + it('compile a .model_info file correctly', async function() { + const kpjFilename = makePathToFixture('sil.cmo.bw', 'sil.cmo.bw.model.kpj'); const kpsFilename = makePathToFixture('sil.cmo.bw', 'source', 'sil.cmo.bw.model.kps'); const kmpFileName = makePathToFixture('sil.cmo.bw', 'build', 'sil.cmo.bw.model.kmp'); const buildModelInfoFilename = makePathToFixture('sil.cmo.bw', 'build', 'sil.cmo.bw.model_info'); - const kmpCompiler = new KmpCompiler(callbacks); + const kmpCompiler = new KmpCompiler(); + assert.isTrue(await kmpCompiler.init(callbacks, {})); const kmpJsonData = kmpCompiler.transformKpsToKmpObject(kpsFilename); const modelFileName = makePathToFixture('sil.cmo.bw', 'build', 'sil.cmo.bw.model.js'); - const data = (new ModelInfoCompiler(callbacks)).writeModelMetadataFile({ + const sources = { kmpFileName, kmpJsonData, model_id: 'sil.cmo.bw', @@ -30,13 +32,16 @@ describe('model-info-compiler', function () { sourcePath: 'release/sil/sil.cmo.bw', kpsFilename, forPublishing: true, - }); - if(data == null) { + }; + const compiler = new ModelInfoCompiler(); + assert.isTrue(await compiler.init(callbacks, {sources})); + const result = await compiler.run(kpjFilename, null); + if(result == null) { callbacks.printMessages(); } - assert.isNotNull(data); + assert.isNotNull(result); - const actual = JSON.parse(new TextDecoder().decode(data)); + const actual = JSON.parse(new TextDecoder().decode(result.artifacts.model_info.data)); let expected = JSON.parse(fs.readFileSync(buildModelInfoFilename, 'utf-8')); // `lastModifiedDate` is dependent on time of run (not worth mocking) diff --git a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts index fe82a2185e..322fa5707b 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts @@ -1,4 +1,3 @@ -import * as fs from 'fs'; import { BuildActivity } from './BuildActivity.js'; import { CompilerCallbacks, KeymanFileTypes } from '@keymanapp/common-types'; import { ModelInfoCompiler } from '@keymanapp/kmc-model-info'; @@ -63,8 +62,7 @@ export class BuildModelInfo extends BuildActivity { } const lastCommitDate = getLastGitCommitDate(project.projectPath); - const compiler = new ModelInfoCompiler(callbacks); - const data = compiler.writeModelMetadataFile({ + const sources = { model_id: callbacks.path.basename(project.projectPath, KeymanFileTypes.Source.Project), kmpJsonData, sourcePath: calculateSourcePath(infile), @@ -73,18 +71,20 @@ export class BuildModelInfo extends BuildActivity { kpsFilename: project.resolveInputFilePath(kps), lastCommitDate, forPublishing: !!options.forPublishing, - }); + }; + const outputFilename = project.getOutputFilePath(KeymanFileTypes.Binary.ModelInfo); - if(data == null) { + const compiler = new ModelInfoCompiler(); + if(!await compiler.init(callbacks, {sources})) { + return false; + } + const result = await compiler.run(infile, outputFilename); + + if(result == null) { // Error messages have already been emitted by writeModelMetadataFile return false; } - fs.writeFileSync( - project.getOutputFilePath(KeymanFileTypes.Binary.ModelInfo), - data - ); - - return true; + return await compiler.write(result.artifacts); } } \ No newline at end of file From f9c3be09a6e7a010cd7d6bfd2451957a5345a5af Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 11 Dec 2023 09:47:23 +0700 Subject: [PATCH 08/14] refactor(developer): move outFile out of CompilerBaseOptions `outFile` is not a compiler option, and so it should not be a part of the `CompilerBaseOptions` interface. This formally separates the command-line options from the compiler options, which eliminates a number of restrictive structures. --- .../web/types/src/util/compiler-interfaces.ts | 1 - developer/src/kmc/src/commands/build.ts | 15 ++++++++---- .../commands/buildClasses/BuildActivity.ts | 7 +++--- .../buildClasses/BuildKeyboardInfo.ts | 4 +++- .../commands/buildClasses/BuildKmnKeyboard.ts | 3 +-- .../buildClasses/BuildLdmlKeyboard.ts | 24 +++++++++---------- .../src/commands/buildClasses/BuildModel.ts | 6 ++--- .../commands/buildClasses/BuildModelInfo.ts | 4 +++- .../src/commands/buildClasses/BuildPackage.ts | 4 ++-- .../src/commands/buildClasses/BuildProject.ts | 18 +++++++------- .../kmc/src/commands/buildTestData/index.ts | 5 ++-- .../buildWindowsPackageInstaller/index.ts | 9 ++++--- .../src/messages/infrastructureMessages.ts | 4 ++++ developer/src/kmc/src/util/baseOptions.ts | 19 +++++++++++++-- developer/src/kmc/test/test-build.ts | 2 +- .../kmc/test/test-infrastructureMessages.ts | 6 ++--- developer/src/kmc/test/test-project-build.ts | 2 +- 17 files changed, 80 insertions(+), 53 deletions(-) diff --git a/common/web/types/src/util/compiler-interfaces.ts b/common/web/types/src/util/compiler-interfaces.ts index fc6a33fa1a..6409c934f5 100644 --- a/common/web/types/src/util/compiler-interfaces.ts +++ b/common/web/types/src/util/compiler-interfaces.ts @@ -403,7 +403,6 @@ export interface CompilerBaseOptions { * Format of output for log to console */ logFormat?: CompilerLogFormat; - outFile?: string; //TODO:REMOVE /** * Colorize log output, default is detected from console */ diff --git a/developer/src/kmc/src/commands/build.ts b/developer/src/kmc/src/commands/build.ts index 0d091680b7..21aa6fdc3b 100644 --- a/developer/src/kmc/src/commands/build.ts +++ b/developer/src/kmc/src/commands/build.ts @@ -19,7 +19,6 @@ function commandOptionsToCompilerOptions(options: any): ExtendedCompilerOptions // CompilerOptions properties... return { // CompilerBaseOptions - outFile: options.outFile, logLevel: options.logLevel, logFormat: options.logFormat, color: options.color, @@ -66,7 +65,8 @@ File lists can be referenced with @filelist.txt. If no input file is supplied, kmc will build the current folder.`) .action(async (filenames: string[], _options: any, commander: any) => { - const options = commandOptionsToCompilerOptions(commander.optsWithGlobals()); + const commanderOptions/*:{TODO?} CommandLineCompilerOptions*/ = commander.optsWithGlobals(); + const options = commandOptionsToCompilerOptions(commanderOptions); const callbacks = new NodeCompilerCallbacks(options); if(!filenames.length) { @@ -75,12 +75,17 @@ If no input file is supplied, kmc will build the current folder.`) filenames.push('.'); } + if(filenames.length > 1 && commanderOptions.outFile) { + // -o can only be specified with a single input file + callbacks.reportMessage(InfrastructureMessages.Error_OutFileCanOnlyBeSpecifiedWithSingleInfile()); + } + if(!expandFileLists(filenames, callbacks)) { process.exit(1); } for(let filename of filenames) { - if(!await build(filename, callbacks, options)) { + if(!await build(filename, commanderOptions.outFile, callbacks, options)) { // Once a file fails to build, we bail on subsequent builds process.exit(1); } @@ -105,7 +110,7 @@ If no input file is supplied, kmc will build the current folder.`) .action(buildWindowsPackageInstaller); } -async function build(filename: string, parentCallbacks: NodeCompilerCallbacks, options: CompilerOptions): Promise { +async function build(filename: string, outfile: string, parentCallbacks: NodeCompilerCallbacks, options: CompilerOptions): Promise { try { // TEST: allow command-line simulation of infrastructure fatal errors, and // also for unit tests @@ -151,7 +156,7 @@ async function build(filename: string, parentCallbacks: NodeCompilerCallbacks, o const callbacks = new CompilerFileCallbacks(buildFilename, options, parentCallbacks); callbacks.reportMessage(InfrastructureMessages.Info_BuildingFile({filename:buildFilename, relativeFilename})); - let result = await builder.build(filename, callbacks, options); + let result = await builder.build(filename, outfile, callbacks, options); result = result && !callbacks.hasFailureMessage(); if(result) { callbacks.reportMessage(builder instanceof BuildProject diff --git a/developer/src/kmc/src/commands/buildClasses/BuildActivity.ts b/developer/src/kmc/src/commands/buildClasses/BuildActivity.ts index dae545366f..064eb8ffb2 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildActivity.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildActivity.ts @@ -6,10 +6,9 @@ export abstract class BuildActivity { public abstract get sourceExtension(): KeymanFileTypes.Source; public abstract get compiledExtension(): KeymanFileTypes.Binary; public abstract get description(): string; - public abstract build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise; - protected getOutputFilename(infile: string, options: CompilerOptions): string { - return options.outFile ? - options.outFile : + public abstract build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise; + protected getOutputFilename(infile: string, outfile?: string): string { + return outfile ?? infile.replace(new RegExp(escapeRegExp(this.sourceExtension), "g"), this.compiledExtension); } }; \ No newline at end of file diff --git a/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts b/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts index 6ed023a139..1ede2decda 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts @@ -13,7 +13,7 @@ export class BuildKeyboardInfo extends BuildActivity { public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.Project; } public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.KeyboardInfo; } public get description(): string { return 'Build a keyboard metadata file'; } - public async build(infile: string, callbacks: CompilerCallbacks, options: ExtendedCompilerOptions): Promise { + public async build(infile: string, _outfile: string, callbacks: CompilerCallbacks, options: ExtendedCompilerOptions): Promise { if(!KeymanFileTypes.filenameIs(infile, KeymanFileTypes.Source.Project)) { // Even if the project file does not exist, we use its name as our reference // in order to avoid ambiguity @@ -48,6 +48,8 @@ export class BuildKeyboardInfo extends BuildActivity { if(!await compiler.init(callbacks, {sources})) { return false; } + + // Note: should we always ignore the passed-in output filename for .keyboard_info? const outputFilename = project.getOutputFilePath(KeymanFileTypes.Binary.KeyboardInfo); const data = await compiler.run(infile, outputFilename); diff --git a/developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts b/developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts index 519d251b61..6965bd3b9c 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts @@ -11,9 +11,8 @@ export class BuildKmnKeyboard extends BuildActivity { public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.KeymanKeyboard; } public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.Keyboard; } public get description(): string { return 'Build a Keyman keyboard'; } - public async build(infile: string, /*TODO: outfile?: string,*/ callbacks: CompilerCallbacks, options: CompilerOptions): Promise { + public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { // We need to resolve paths to absolute paths before calling kmc-kmn - let outfile = options.outFile;//TODO: remove here if(outfile) { outfile = getPosixAbsolutePath(outfile); const folderName = path.dirname(outfile); diff --git a/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts b/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts index fbb69f921c..f93aabcf47 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts @@ -10,10 +10,19 @@ export class BuildLdmlKeyboard extends BuildActivity { public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.LdmlKeyboard; } public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.Keyboard; } public get description(): string { return 'Build a LDML keyboard'; } - public async build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { + public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { // TODO-LDML: consider hardware vs touch -- touch-only layout will not have a .kvk // Compile: - const outfile = options.outFile; //TODO + // TODO: Consider if this mkdir should be in write() + // TODO: This pattern may be needed for kmc-kmn as well? + const outFileDir = callbacks.path.dirname(outfile ?? infile); + + try { + fs.mkdirSync(outFileDir, {recursive: true}); + } catch(e) { + callbacks.reportMessage(InfrastructureMessages.Error_CannotCreateFolder({folderName:outFileDir, e})); + return false; + } const ldmlCompilerOptions: kmcLdml.LdmlCompilerOptions = {...options, readerOptions: { importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)) @@ -29,17 +38,6 @@ export class BuildLdmlKeyboard extends BuildActivity { return false; } - // TODO: Consider if this mkdir should be in write() - // TODO: This pattern may be needed for kmc-kmn as well? - const outFileDir = callbacks.path.dirname(outfile ?? infile); - - try { - fs.mkdirSync(outFileDir, {recursive: true}); - } catch(e) { - callbacks.reportMessage(InfrastructureMessages.Error_CannotCreateFolder({folderName:outFileDir, e})); - return false; - } - return await compiler.write(result.artifacts); } } diff --git a/developer/src/kmc/src/commands/buildClasses/BuildModel.ts b/developer/src/kmc/src/commands/buildClasses/BuildModel.ts index 00614c2731..41d6fcf792 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildModel.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildModel.ts @@ -7,15 +7,15 @@ export class BuildModel extends BuildActivity { public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.Model; } public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.Model; } public get description(): string { return 'Build a lexical model'; } - public async build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { - const outputFilename: string = this.getOutputFilename(infile, options); + public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { + outfile = this.getOutputFilename(infile, outfile); const compiler = new LexicalModelCompiler(); if(!await compiler.init(callbacks, options)) { return false; } - const result = await compiler.run(infile, outputFilename); + const result = await compiler.run(infile, outfile); if(!result) { return false; } diff --git a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts index 322fa5707b..4000af6894 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts @@ -24,7 +24,7 @@ export class BuildModelInfo extends BuildActivity { * @param options * @returns */ - public async build(infile: string, callbacks: CompilerCallbacks, options: ExtendedCompilerOptions): Promise { + public async build(infile: string, _outfile: string, callbacks: CompilerCallbacks, options: ExtendedCompilerOptions): Promise { if(!KeymanFileTypes.filenameIs(infile, KeymanFileTypes.Source.Project)) { // Even if the project file does not exist, we use its name as our reference // in order to avoid ambiguity @@ -72,6 +72,8 @@ export class BuildModelInfo extends BuildActivity { lastCommitDate, forPublishing: !!options.forPublishing, }; + + // Note: should we always ignore the passed-in output filename for .model_info? const outputFilename = project.getOutputFilePath(KeymanFileTypes.Binary.ModelInfo); const compiler = new ModelInfoCompiler(); diff --git a/developer/src/kmc/src/commands/buildClasses/BuildPackage.ts b/developer/src/kmc/src/commands/buildClasses/BuildPackage.ts index 12593b0f8e..05172d4419 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildPackage.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildPackage.ts @@ -7,9 +7,9 @@ export class BuildPackage extends BuildActivity { public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.Package; } public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.Package; } public get description(): string { return 'Build a Keyman package'; } - public async build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { + public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { - const outfile = this.getOutputFilename(infile, options); + outfile = this.getOutputFilename(infile, outfile); const kmpCompiler = new KmpCompiler(); if(!await kmpCompiler.init(callbacks, options)) { diff --git a/developer/src/kmc/src/commands/buildClasses/BuildProject.ts b/developer/src/kmc/src/commands/buildClasses/BuildProject.ts index 5cbd3da80e..043edbf11c 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildProject.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildProject.ts @@ -12,7 +12,12 @@ export class BuildProject extends BuildActivity { public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.Project; } public get compiledExtension(): KeymanFileTypes.Binary { return null; } public get description(): string { return 'Build a keyboard or lexical model project'; } - public async build(infile: string, callbacks: CompilerCallbacks, options: ExtendedCompilerOptions): Promise { + public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: ExtendedCompilerOptions): Promise { + if(outfile) { + callbacks.reportMessage(InfrastructureMessages.Error_OutFileNotValidForProjects()); + return false; + } + let builder = new ProjectBuilder(infile, callbacks, options); return builder.run(); } @@ -31,11 +36,6 @@ class ProjectBuilder { } async run(): Promise { - if(this.options.outFile) { - this.callbacks.reportMessage(InfrastructureMessages.Error_OutFileNotValidForProjects()); - return false; - } - this.project = loadProject(this.infile, this.callbacks); if(!this.project) { return false; @@ -97,7 +97,7 @@ class ProjectBuilder { async buildTarget(file: KeymanDeveloperProjectFile, activity: BuildActivity): Promise { const options = {...this.options}; - options.outFile = this.project.resolveOutputFilePath(file, activity.sourceExtension, activity.compiledExtension); + const outfile = this.project.resolveOutputFilePath(file, activity.sourceExtension, activity.compiledExtension); options.checkFilenameConventions = this.project.options.checkFilenameConventions ?? this.options.checkFilenameConventions; const infile = this.project.resolveInputFilePath(file); @@ -105,9 +105,9 @@ class ProjectBuilder { const callbacks = new CompilerFileCallbacks(buildFilename, options, this.callbacks); callbacks.reportMessage(InfrastructureMessages.Info_BuildingFile({filename: infile, relativeFilename:buildFilename})); - fs.mkdirSync(path.dirname(options.outFile), {recursive:true}); + fs.mkdirSync(path.dirname(outfile), {recursive:true}); - let result = await activity.build(infile, callbacks, options); + let result = await activity.build(infile, outfile, callbacks, options); // check if we had a message that causes the build to be a failure // note: command line option here, if set, overrides project setting diff --git a/developer/src/kmc/src/commands/buildTestData/index.ts b/developer/src/kmc/src/commands/buildTestData/index.ts index 5d0f6a2380..4253f55cf6 100644 --- a/developer/src/kmc/src/commands/buildTestData/index.ts +++ b/developer/src/kmc/src/commands/buildTestData/index.ts @@ -1,12 +1,13 @@ import * as fs from 'fs'; import * as path from 'path'; import * as kmcLdml from '@keymanapp/kmc-ldml'; -import { CompilerBaseOptions, CompilerCallbacks, defaultCompilerOptions, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboardXMLSourceFileReader } from '@keymanapp/common-types'; +import { CompilerCallbacks, defaultCompilerOptions, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboardXMLSourceFileReader } from '@keymanapp/common-types'; import { NodeCompilerCallbacks } from '../../util/NodeCompilerCallbacks.js'; import { fileURLToPath } from 'url'; +import { CommandLineBaseOptions } from 'src/util/baseOptions.js'; export async function buildTestData(infile: string, _options: any, commander: any) { - const options: CompilerBaseOptions = commander.optsWithGlobals(); + const options: CommandLineBaseOptions = commander.optsWithGlobals(); let compilerOptions: kmcLdml.LdmlCompilerOptions = { ...defaultCompilerOptions, diff --git a/developer/src/kmc/src/commands/buildWindowsPackageInstaller/index.ts b/developer/src/kmc/src/commands/buildWindowsPackageInstaller/index.ts index 6f55970838..5d64908856 100644 --- a/developer/src/kmc/src/commands/buildWindowsPackageInstaller/index.ts +++ b/developer/src/kmc/src/commands/buildWindowsPackageInstaller/index.ts @@ -1,10 +1,11 @@ import * as fs from 'fs'; import * as path from 'path'; -import { CompilerBaseOptions, CompilerCallbacks, defaultCompilerOptions } from '@keymanapp/common-types'; +import { CompilerCallbacks, defaultCompilerOptions } from '@keymanapp/common-types'; import { NodeCompilerCallbacks } from '../../util/NodeCompilerCallbacks.js'; import { WindowsPackageInstallerCompiler, WindowsPackageInstallerSources } from '@keymanapp/kmc-package'; +import { CommandLineBaseOptions } from 'src/util/baseOptions.js'; -interface WindowsPackageInstallerCommandLineOptions extends CompilerBaseOptions { +interface WindowsPackageInstallerCommandLineOptions extends CommandLineBaseOptions { msi: string; exe: string; license: string; @@ -28,6 +29,8 @@ export async function buildWindowsPackageInstaller(infile: string, _options: any titleImageFilename: options.titleImage } + const outfile: string = options.outFile; + // Normalize case for the filename and expand the path; this avoids false // positive case mismatches on input filenames and glommed paths infile = fs.realpathSync.native(infile); @@ -38,7 +41,7 @@ export async function buildWindowsPackageInstaller(infile: string, _options: any process.exit(1); } - const fileBaseName = options.outFile ?? infile; + const fileBaseName = outfile ?? infile; const outFileBase = path.basename(fileBaseName, path.extname(fileBaseName)); const outFileDir = path.dirname(fileBaseName); const outFileExe = path.join(outFileDir, outFileBase + '.exe'); diff --git a/developer/src/kmc/src/messages/infrastructureMessages.ts b/developer/src/kmc/src/messages/infrastructureMessages.ts index 50334a6fd6..561b81ed4c 100644 --- a/developer/src/kmc/src/messages/infrastructureMessages.ts +++ b/developer/src/kmc/src/messages/infrastructureMessages.ts @@ -87,5 +87,9 @@ export class InfrastructureMessages { static Error_UnsupportedProjectVersion = (o:{version:string}) => m(this.ERROR_UnsupportedProjectVersion, `Project version ${o.version} is not supported by this version of Keyman Developer.`); static ERROR_UnsupportedProjectVersion = SevError | 0x0013; + + static Error_OutFileCanOnlyBeSpecifiedWithSingleInfile = () => m(this.ERROR_OutFileCanOnlyBeSpecifiedWithSingleInfile, + `Parameter -out-file can only be used with a single input file.`); + static ERROR_OutFileCanOnlyBeSpecifiedWithSingleInfile = SevError | 0x0014; } diff --git a/developer/src/kmc/src/util/baseOptions.ts b/developer/src/kmc/src/util/baseOptions.ts index 9416376951..dd91f0bae6 100644 --- a/developer/src/kmc/src/util/baseOptions.ts +++ b/developer/src/kmc/src/util/baseOptions.ts @@ -1,8 +1,23 @@ -import { ALL_COMPILER_LOG_FORMATS, ALL_COMPILER_LOG_LEVELS } from "@keymanapp/common-types"; +import { ALL_COMPILER_LOG_FORMATS, ALL_COMPILER_LOG_LEVELS, CompilerLogFormat, CompilerLogLevel } from "@keymanapp/common-types"; import { Command, Option } from "commander"; -// These options map to CompilerBaseOptions +/** + * Abstract interface for compiler options + */ +export interface CommandLineBaseOptions { + // These options map to CompilerBaseOptions + logLevel?: CompilerLogLevel; + logFormat?: CompilerLogFormat; + color?: boolean; + + // This option is not in CompilerBaseOptions + outFile?:string; +} + +/** + * These options map to CompilerBaseOptions + */ export class BaseOptions { public static addLogLevel(program: Command) { return program.addOption(new Option('-l, --log-level ', 'Log level').choices(ALL_COMPILER_LOG_LEVELS).default('info')); diff --git a/developer/src/kmc/test/test-build.ts b/developer/src/kmc/test/test-build.ts index 26ef09e746..d859e3f26d 100644 --- a/developer/src/kmc/test/test-build.ts +++ b/developer/src/kmc/test/test-build.ts @@ -34,7 +34,7 @@ describe('compilerWarningsAsErrors', function () { const builder = new BuildProject(); const path = makePathToFixture('compiler-warnings-as-errors', `compiler_warnings_as_errors_${truth.kpj === true ? 'true' : (truth.kpj === false ? 'false' : 'undefined')}.kpj`); - const result = await builder.build(path, callbacks, { + const result = await builder.build(path, null, callbacks, { compilerWarningsAsErrors: truth.cli, }); if(truth.result != result) { diff --git a/developer/src/kmc/test/test-infrastructureMessages.ts b/developer/src/kmc/test/test-infrastructureMessages.ts index 6d543a5fd7..34898a5ad9 100644 --- a/developer/src/kmc/test/test-infrastructureMessages.ts +++ b/developer/src/kmc/test/test-infrastructureMessages.ts @@ -24,7 +24,7 @@ describe('InfrastructureMessages', function () { const expectedMessages = [InfrastructureMessages.FATAL_UnexpectedException]; process.env.SENTRY_CLIENT_TEST_BUILD_EXCEPTION = '1'; - await unitTestEndpoints.build(null, ncb, {}); + await unitTestEndpoints.build(null, null, ncb, {}); delete process.env.SENTRY_CLIENT_TEST_BUILD_EXCEPTION; assertMessagesEqual(ncb.messages, expectedMessages); @@ -79,7 +79,7 @@ describe('InfrastructureMessages', function () { InfrastructureMessages.INFO_WarningsHaveFailedBuild, InfrastructureMessages.INFO_FileNotBuiltSuccessfully ]; - await unitTestEndpoints.build(filename, ncb, {compilerWarningsAsErrors: true}); + await unitTestEndpoints.build(filename, null, ncb, {compilerWarningsAsErrors: true}); assertMessagesEqual(ncb.messages, expectedMessages); }); @@ -93,7 +93,7 @@ describe('InfrastructureMessages', function () { InfrastructureMessages.ERROR_UnsupportedProjectVersion, InfrastructureMessages.INFO_ProjectNotBuiltSuccessfully ]; - await unitTestEndpoints.build(filename, ncb, {compilerWarningsAsErrors: true}); + await unitTestEndpoints.build(filename, null, ncb, {compilerWarningsAsErrors: true}); assertMessagesEqual(ncb.messages, expectedMessages); }); }); diff --git a/developer/src/kmc/test/test-project-build.ts b/developer/src/kmc/test/test-project-build.ts index 8856316880..ebcff59dc0 100644 --- a/developer/src/kmc/test/test-project-build.ts +++ b/developer/src/kmc/test/test-project-build.ts @@ -11,7 +11,7 @@ describe('BuildProject', function () { it('should build a keyboard project', async function() { const builder = new BuildProject(); const path = makePathToFixture('relative_paths', 'k_000___null_keyboard.kpj'); - let result = await builder.build(path, callbacks, { + let result = await builder.build(path, null, callbacks, { shouldAddCompilerVersion: false, compilerWarningsAsErrors: true, saveDebug: false, From f4918d36e29dc2fc250af49cae96e41238459608 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 11 Dec 2023 10:10:31 +0700 Subject: [PATCH 09/14] feat(developer): refactor common compiler steps into BuildActivity.runCompiler Relates to #9473. Now that each compiler has a common interface, we can consolidate the common code into BuildActivity. This makes it very easy to see where the compiler calls have special cases, such as the model_info and keyboard_info compilers (these are primarily for internal use, so the special case work should not be a problem). --- .../commands/buildClasses/BuildActivity.ts | 37 ++++++++++++++++--- .../buildClasses/BuildKeyboardInfo.ts | 16 +------- .../commands/buildClasses/BuildKmnKeyboard.ts | 22 +---------- .../buildClasses/BuildLdmlKeyboard.ts | 26 +------------ .../src/commands/buildClasses/BuildModel.ts | 13 +------ .../commands/buildClasses/BuildModelInfo.ts | 12 +----- .../src/commands/buildClasses/BuildPackage.ts | 16 +------- developer/src/kmc/src/util/escapeRegExp.ts | 7 ---- 8 files changed, 41 insertions(+), 108 deletions(-) delete mode 100644 developer/src/kmc/src/util/escapeRegExp.ts diff --git a/developer/src/kmc/src/commands/buildClasses/BuildActivity.ts b/developer/src/kmc/src/commands/buildClasses/BuildActivity.ts index 064eb8ffb2..086333149a 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildActivity.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildActivity.ts @@ -1,5 +1,6 @@ -import { CompilerCallbacks, CompilerOptions, KeymanFileTypes } from "@keymanapp/common-types"; -import { escapeRegExp } from "../../util/escapeRegExp.js"; +import * as fs from 'fs'; +import { CompilerCallbacks, CompilerOptions, KeymanCompiler, KeymanFileTypes } from "@keymanapp/common-types"; +import { InfrastructureMessages } from '../../messages/infrastructureMessages.js'; export abstract class BuildActivity { public abstract get name(): string; @@ -7,8 +8,34 @@ export abstract class BuildActivity { public abstract get compiledExtension(): KeymanFileTypes.Binary; public abstract get description(): string; public abstract build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise; - protected getOutputFilename(infile: string, outfile?: string): string { - return outfile ?? - infile.replace(new RegExp(escapeRegExp(this.sourceExtension), "g"), this.compiledExtension); + + protected async runCompiler(compiler: KeymanCompiler, infile: string, outfile: string, callbacks: CompilerCallbacks, options: T): Promise { + if(!await compiler.init(callbacks, options)) { + return false; + } + + const result = await compiler.run(infile, outfile); + if(!result) { + return false; + } + + if(!this.createOutputFolder(outfile ?? infile, callbacks)) { + return false; + } + + return await compiler.write(result.artifacts); + } + + private createOutputFolder(targetFilename: string, callbacks: CompilerCallbacks): boolean { + const targetFolder = callbacks.path.dirname(targetFilename); + + try { + fs.mkdirSync(targetFolder, {recursive: true}); + } catch(e) { + callbacks.reportMessage(InfrastructureMessages.Error_CannotCreateFolder({folderName:targetFolder, e})); + return false; + } + + return true; } }; \ No newline at end of file diff --git a/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts b/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts index 1ede2decda..605870741e 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts @@ -43,21 +43,9 @@ export class BuildKeyboardInfo extends BuildActivity { lastCommitDate, forPublishing: !!options.forPublishing, }; - - const compiler = new KeyboardInfoCompiler(); - if(!await compiler.init(callbacks, {sources})) { - return false; - } - // Note: should we always ignore the passed-in output filename for .keyboard_info? const outputFilename = project.getOutputFilePath(KeymanFileTypes.Binary.KeyboardInfo); - const data = await compiler.run(infile, outputFilename); - - if(data == null) { - // Error messages have already been emitted by KeyboardInfoCompiler - return false; - } - - return await compiler.write(data.artifacts); + const compiler = new KeyboardInfoCompiler(); + return await super.runCompiler(compiler, infile, outputFilename, callbacks, {...options, sources}); } } diff --git a/developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts b/developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts index 6965bd3b9c..e540aba451 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts @@ -3,8 +3,6 @@ import { platform } from 'os'; import { KmnCompiler } from '@keymanapp/kmc-kmn'; import { CompilerOptions, CompilerCallbacks, KeymanFileTypes } from '@keymanapp/common-types'; import { BuildActivity } from './BuildActivity.js'; -import * as fs from 'fs'; -import { InfrastructureMessages } from '../../messages/infrastructureMessages.js'; export class BuildKmnKeyboard extends BuildActivity { public get name(): string { return 'Keyman keyboard'; } @@ -13,29 +11,13 @@ export class BuildKmnKeyboard extends BuildActivity { public get description(): string { return 'Build a Keyman keyboard'; } public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { // We need to resolve paths to absolute paths before calling kmc-kmn + infile = getPosixAbsolutePath(infile); if(outfile) { outfile = getPosixAbsolutePath(outfile); - const folderName = path.dirname(outfile); - try { - fs.mkdirSync(folderName, {recursive: true}); - } catch(e) { - callbacks.reportMessage(InfrastructureMessages.Error_CannotCreateFolder({folderName, e})); - return false; - } } - infile = getPosixAbsolutePath(infile); const compiler = new KmnCompiler(); - if(!await compiler.init(callbacks, options)) { - return false; - } - - const result = await compiler.run(infile, outfile); - if(!result) { - return false; - } - - return await compiler.write(result.artifacts); + return await super.runCompiler(compiler, infile, outfile, callbacks, options); } } diff --git a/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts b/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts index f93aabcf47..0e0cfd9766 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts @@ -1,9 +1,7 @@ -import * as fs from 'fs'; import * as kmcLdml from '@keymanapp/kmc-ldml'; import { CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, CompilerOptions, KeymanFileTypes } from '@keymanapp/common-types'; import { BuildActivity } from './BuildActivity.js'; import { fileURLToPath } from 'url'; -import { InfrastructureMessages } from '../../messages/infrastructureMessages.js'; export class BuildLdmlKeyboard extends BuildActivity { public get name(): string { return 'LDML keyboard'; } @@ -12,32 +10,10 @@ export class BuildLdmlKeyboard extends BuildActivity { public get description(): string { return 'Build a LDML keyboard'; } public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { // TODO-LDML: consider hardware vs touch -- touch-only layout will not have a .kvk - // Compile: - // TODO: Consider if this mkdir should be in write() - // TODO: This pattern may be needed for kmc-kmn as well? - const outFileDir = callbacks.path.dirname(outfile ?? infile); - - try { - fs.mkdirSync(outFileDir, {recursive: true}); - } catch(e) { - callbacks.reportMessage(InfrastructureMessages.Error_CannotCreateFolder({folderName:outFileDir, e})); - return false; - } - const ldmlCompilerOptions: kmcLdml.LdmlCompilerOptions = {...options, readerOptions: { importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)) }}; - const compiler = new kmcLdml.LdmlKeyboardCompiler(); - if(!await compiler.init(callbacks, ldmlCompilerOptions)) { - return false; - } - - const result = await compiler.run(infile, outfile); - if(!result) { - return false; - } - - return await compiler.write(result.artifacts); + return await super.runCompiler(compiler, infile, outfile, callbacks, ldmlCompilerOptions); } } diff --git a/developer/src/kmc/src/commands/buildClasses/BuildModel.ts b/developer/src/kmc/src/commands/buildClasses/BuildModel.ts index 41d6fcf792..b2afc59294 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildModel.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildModel.ts @@ -8,18 +8,7 @@ export class BuildModel extends BuildActivity { public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.Model; } public get description(): string { return 'Build a lexical model'; } public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { - outfile = this.getOutputFilename(infile, outfile); - const compiler = new LexicalModelCompiler(); - if(!await compiler.init(callbacks, options)) { - return false; - } - - const result = await compiler.run(infile, outfile); - if(!result) { - return false; - } - - return await compiler.write(result.artifacts); + return await super.runCompiler(compiler, infile, outfile, callbacks, options); } } \ No newline at end of file diff --git a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts index 4000af6894..912bca986a 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts @@ -77,16 +77,6 @@ export class BuildModelInfo extends BuildActivity { const outputFilename = project.getOutputFilePath(KeymanFileTypes.Binary.ModelInfo); const compiler = new ModelInfoCompiler(); - if(!await compiler.init(callbacks, {sources})) { - return false; - } - const result = await compiler.run(infile, outputFilename); - - if(result == null) { - // Error messages have already been emitted by writeModelMetadataFile - return false; - } - - return await compiler.write(result.artifacts); + return await super.runCompiler(compiler, infile, outputFilename, callbacks, {...options, sources}); } } \ No newline at end of file diff --git a/developer/src/kmc/src/commands/buildClasses/BuildPackage.ts b/developer/src/kmc/src/commands/buildClasses/BuildPackage.ts index 05172d4419..1bb26b3b7f 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildPackage.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildPackage.ts @@ -8,19 +8,7 @@ export class BuildPackage extends BuildActivity { public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.Package; } public get description(): string { return 'Build a Keyman package'; } public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { - - outfile = this.getOutputFilename(infile, outfile); - - const kmpCompiler = new KmpCompiler(); - if(!await kmpCompiler.init(callbacks, options)) { - return false; - } - - const result = await kmpCompiler.run(infile, outfile); - if(!result) { - return false; - } - - return await kmpCompiler.write(result.artifacts); + const compiler = new KmpCompiler(); + return await super.runCompiler(compiler, infile, outfile, callbacks, options); } } diff --git a/developer/src/kmc/src/util/escapeRegExp.ts b/developer/src/kmc/src/util/escapeRegExp.ts deleted file mode 100644 index 8972b6bfed..0000000000 --- a/developer/src/kmc/src/util/escapeRegExp.ts +++ /dev/null @@ -1,7 +0,0 @@ - -const escapedRegexp = /[.*+?^${}()|[\]\\]/g; - -// See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#escaping -export function escapeRegExp(s: string) { - return s.replace(escapedRegexp, "\\$&"); // $& means the whole matched string -} From 52038e22aab57e5ded9e822a6d273721412c5b37 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 11 Dec 2023 11:43:24 +0700 Subject: [PATCH 10/14] chore(developer): remove unintended usage of `callbacks.path`/`callbacks.fs` in kmc --- common/web/types/src/kpj/keyman-developer-project.ts | 12 ++++++++++-- developer/src/kmc/src/commands/analyze.ts | 2 +- .../kmc/src/commands/buildClasses/BuildActivity.ts | 3 ++- .../kmc/src/commands/buildClasses/BuildModelInfo.ts | 3 ++- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/common/web/types/src/kpj/keyman-developer-project.ts b/common/web/types/src/kpj/keyman-developer-project.ts index a53cbca20e..fdc8466497 100644 --- a/common/web/types/src/kpj/keyman-developer-project.ts +++ b/common/web/types/src/kpj/keyman-developer-project.ts @@ -37,8 +37,16 @@ export class KeymanDeveloperProject { for(let filename of files) { let fullPath = this.callbacks.path.join(sourcePath, filename); if(KeymanFileTypes.filenameIs(filename, KeymanFileTypes.Source.LdmlKeyboard)) { - if(!this.callbacks.fs.readFileSync(fullPath, 'utf-8').match(/ldmlKeyboard3\.dtd/)) { - // Skip this .xml because we assume it isn't really a keyboard .xml + try { + const data = this.callbacks.loadFile(fullPath); + const text = new TextDecoder().decode(data); + if(!text?.match(/ldmlKeyboard3\.dtd/)) { + // Skip this .xml because we assume it isn't really a keyboard .xml + continue; + } + } catch(e) { + // We'll just silently skip this file because we were not able to load it, + // so let's hope it wasn't a real LDML keyboard XML :-) continue; } } diff --git a/developer/src/kmc/src/commands/analyze.ts b/developer/src/kmc/src/commands/analyze.ts index eb2feb528a..15c8ac3f39 100644 --- a/developer/src/kmc/src/commands/analyze.ts +++ b/developer/src/kmc/src/commands/analyze.ts @@ -105,7 +105,7 @@ async function analyzeOskCharUse(callbacks: CompilerCallbacks, filenames: string async function analyzeOskRewritePua(callbacks: CompilerCallbacks, filenames: string[], options: AnalysisActivityOptions) { const analyzer = new AnalyzeOskRewritePua(callbacks); - const mapping: any = JSON.parse(callbacks.fs.readFileSync(options.mappingFile, 'UTF-8')); + const mapping: any = JSON.parse(fs.readFileSync(options.mappingFile, 'utf-8')); return await runOnFiles(callbacks, filenames, async (filename: string): Promise => { if(!await analyzer.analyze(filename, mapping)) { diff --git a/developer/src/kmc/src/commands/buildClasses/BuildActivity.ts b/developer/src/kmc/src/commands/buildClasses/BuildActivity.ts index 086333149a..864611c26d 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildActivity.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildActivity.ts @@ -1,4 +1,5 @@ import * as fs from 'fs'; +import * as path from 'path'; import { CompilerCallbacks, CompilerOptions, KeymanCompiler, KeymanFileTypes } from "@keymanapp/common-types"; import { InfrastructureMessages } from '../../messages/infrastructureMessages.js'; @@ -27,7 +28,7 @@ export abstract class BuildActivity { } private createOutputFolder(targetFilename: string, callbacks: CompilerCallbacks): boolean { - const targetFolder = callbacks.path.dirname(targetFilename); + const targetFolder = path.dirname(targetFilename); try { fs.mkdirSync(targetFolder, {recursive: true}); diff --git a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts index 912bca986a..470c90639d 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts @@ -1,3 +1,4 @@ +import * as path from 'path'; import { BuildActivity } from './BuildActivity.js'; import { CompilerCallbacks, KeymanFileTypes } from '@keymanapp/common-types'; import { ModelInfoCompiler } from '@keymanapp/kmc-model-info'; @@ -63,7 +64,7 @@ export class BuildModelInfo extends BuildActivity { const lastCommitDate = getLastGitCommitDate(project.projectPath); const sources = { - model_id: callbacks.path.basename(project.projectPath, KeymanFileTypes.Source.Project), + model_id: path.basename(project.projectPath, KeymanFileTypes.Source.Project), kmpJsonData, sourcePath: calculateSourcePath(infile), modelFileName: project.resolveOutputFilePath(model, KeymanFileTypes.Source.Model, KeymanFileTypes.Binary.Model), From d99c4cd669e497140a41e8f6a6eacb61dd05b276 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 11 Dec 2023 12:21:20 +0700 Subject: [PATCH 11/14] chore(web): update LexicalModelCompiler call --- common/web/input-processor/tests/cases/languageProcessor.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/common/web/input-processor/tests/cases/languageProcessor.js b/common/web/input-processor/tests/cases/languageProcessor.js index cd7d87dda9..1cabec0ee8 100644 --- a/common/web/input-processor/tests/cases/languageProcessor.js +++ b/common/web/input-processor/tests/cases/languageProcessor.js @@ -56,8 +56,9 @@ describe('LanguageProcessor', function() { }); }); - describe('.predict', function() { - let compiler = new LexicalModelCompiler(callbacks); + describe('.predict', async function() { + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, {})); const MODEL_ID = 'example.qaa.trivial'; // ES-module mode leaves out `__dirname`, so we rebuild it using other components. From e2ce64d9be0cf01cb25da50c23bff8ddb8ccd1d6 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 13 Dec 2023 05:38:51 +0700 Subject: [PATCH 12/14] fix(web): mocha describe does not do async mocha describe() does not accept an async function. Any async prep should be done in a before() function, which does support async. One helpful ref: https://github.com/mochajs/mocha/issues/2975#issuecomment-1004176440 --- .../tests/cases/inputProcessorTests.js | 2 +- .../tests/cases/languageProcessor.js | 67 +++++++++++-------- developer/src/kmc-kmn/test/test-features.ts | 2 +- .../kmc-package/test/test-package-compiler.ts | 11 +-- 4 files changed, 49 insertions(+), 33 deletions(-) diff --git a/common/web/input-processor/tests/cases/inputProcessorTests.js b/common/web/input-processor/tests/cases/inputProcessorTests.js index 4085efee4f..939ea36004 100644 --- a/common/web/input-processor/tests/cases/inputProcessorTests.js +++ b/common/web/input-processor/tests/cases/inputProcessorTests.js @@ -184,7 +184,7 @@ describe('InputProcessor', function() { // rest of the recorder stuff since it uses only KeyboardProcessor, not InputProcessor. let testDefinitions = new KeyboardTest(JSON.parse(testJSONtext)); - before(async function () { + this.before(async function () { // Load the keyboard. let keyboardLoader = new NodeKeyboardLoader(new KeyboardInterface({}, MinimalKeymanGlobal)); const keyboard = await keyboardLoader.loadKeyboardFromPath(require.resolve('@keymanapp/common-test-resources/keyboards/test_8568_deadkeys.js')); diff --git a/common/web/input-processor/tests/cases/languageProcessor.js b/common/web/input-processor/tests/cases/languageProcessor.js index 1cabec0ee8..034afcf6fb 100644 --- a/common/web/input-processor/tests/cases/languageProcessor.js +++ b/common/web/input-processor/tests/cases/languageProcessor.js @@ -56,9 +56,16 @@ describe('LanguageProcessor', function() { }); }); - describe('.predict', async function() { - let compiler = new LexicalModelCompiler(); - assert.isTrue(await compiler.init(callbacks, {})); + describe('.predict', function() { + let compiler = null; + + this.beforeAll(async function() { + compiler = new LexicalModelCompiler(); + console.dir(compiler) + console.dir(compiler.init) + assert.isTrue(await compiler.init(callbacks, {})); + }); + const MODEL_ID = 'example.qaa.trivial'; // ES-module mode leaves out `__dirname`, so we rebuild it using other components. @@ -68,20 +75,23 @@ describe('LanguageProcessor', function() { const PATH = path.join(__dirname, '../../../../../developer/src/kmc-model/test/fixtures', MODEL_ID); describe('using angle brackets for quotes', function() { - let modelCode = compiler.generateLexicalModelCode(MODEL_ID, { - format: 'trie-1.0', - sources: ['wordlist.tsv'], - punctuation: { - quotesForKeepSuggestion: { open: `«`, close: `»`}, - insertAfterWord: " " , // OGHAM SPACE MARK - } - }, PATH); + let modelCode = null, modelSpec = null; + this.beforeAll(function() { + modelCode = compiler.generateLexicalModelCode(MODEL_ID, { + format: 'trie-1.0', + sources: ['wordlist.tsv'], + punctuation: { + quotesForKeepSuggestion: { open: `«`, close: `»`}, + insertAfterWord: " " , // OGHAM SPACE MARK + } + }, PATH); - let modelSpec = { - id: MODEL_ID, - languages: ['en'], - code: modelCode - }; + modelSpec = { + id: MODEL_ID, + languages: ['en'], + code: modelCode + }; + }); it("successfully loads the model", function(done) { let languageProcessor = new LanguageProcessor(worker); @@ -116,18 +126,21 @@ describe('LanguageProcessor', function() { }); describe('properly cases generated suggestions', function() { - let modelCode = compiler.generateLexicalModelCode(MODEL_ID, { - format: 'trie-1.0', - sources: ['wordlist.tsv'], - languageUsesCasing: true, - //applyCasing // we rely on the compiler's default implementation here. - }, PATH); + let modelCode = null, modelSpec = null; + this.beforeAll(function () { + modelCode = compiler.generateLexicalModelCode(MODEL_ID, { + format: 'trie-1.0', + sources: ['wordlist.tsv'], + languageUsesCasing: true, + //applyCasing // we rely on the compiler's default implementation here. + }, PATH); - let modelSpec = { - id: MODEL_ID, - languages: ['en'], - code: modelCode - }; + modelSpec = { + id: MODEL_ID, + languages: ['en'], + code: modelCode + }; + }); describe("does not alter casing when input is lowercased", function() { it("when input is fully lowercased", function(done) { diff --git a/developer/src/kmc-kmn/test/test-features.ts b/developer/src/kmc-kmn/test/test-features.ts index 7f4be10951..a030ba984c 100644 --- a/developer/src/kmc-kmn/test/test-features.ts +++ b/developer/src/kmc-kmn/test/test-features.ts @@ -5,7 +5,7 @@ import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers'; import { makePathToFixture } from './helpers/index.js'; import { KMX, KmxFileReader } from '@keymanapp/common-types'; -describe('Keyboard compiler features', async function() { +describe('Keyboard compiler features', function() { let compiler: KmnCompiler = null; let callbacks: TestCompilerCallbacks = null; diff --git a/developer/src/kmc-package/test/test-package-compiler.ts b/developer/src/kmc-package/test/test-package-compiler.ts index 3a17d321eb..1398befca2 100644 --- a/developer/src/kmc-package/test/test-package-compiler.ts +++ b/developer/src/kmc-package/test/test-package-compiler.ts @@ -15,15 +15,18 @@ import { CompilerMessages } from '../src/compiler/messages.js'; const debug = false; -describe('KmpCompiler', async function () { +describe('KmpCompiler', function () { const MODELS : string[] = [ 'example.qaa.sencoten', 'withfolders.qaa.sencoten', ]; - const callbacks = new TestCompilerCallbacks(); - let kmpCompiler = new KmpCompiler(); - assert.isTrue(await kmpCompiler.init(callbacks, null)); + let kmpCompiler: KmpCompiler = null; + + this.beforeAll(async function() { + kmpCompiler = new KmpCompiler(); + assert.isTrue(await kmpCompiler.init(callbacks, null)); + }); for (let modelID of MODELS) { const kpsPath = modelID.includes('withfolders') ? From 4f32875a57d90be81c0dcd3dd89156d421260f4b Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 13 Dec 2023 08:06:37 +0700 Subject: [PATCH 13/14] chore(web): cleanup tests --- common/web/input-processor/tests/cases/inputProcessorTests.js | 2 +- common/web/input-processor/tests/cases/languageProcessor.js | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/common/web/input-processor/tests/cases/inputProcessorTests.js b/common/web/input-processor/tests/cases/inputProcessorTests.js index 939ea36004..4085efee4f 100644 --- a/common/web/input-processor/tests/cases/inputProcessorTests.js +++ b/common/web/input-processor/tests/cases/inputProcessorTests.js @@ -184,7 +184,7 @@ describe('InputProcessor', function() { // rest of the recorder stuff since it uses only KeyboardProcessor, not InputProcessor. let testDefinitions = new KeyboardTest(JSON.parse(testJSONtext)); - this.before(async function () { + before(async function () { // Load the keyboard. let keyboardLoader = new NodeKeyboardLoader(new KeyboardInterface({}, MinimalKeymanGlobal)); const keyboard = await keyboardLoader.loadKeyboardFromPath(require.resolve('@keymanapp/common-test-resources/keyboards/test_8568_deadkeys.js')); diff --git a/common/web/input-processor/tests/cases/languageProcessor.js b/common/web/input-processor/tests/cases/languageProcessor.js index 034afcf6fb..15b16da86a 100644 --- a/common/web/input-processor/tests/cases/languageProcessor.js +++ b/common/web/input-processor/tests/cases/languageProcessor.js @@ -61,8 +61,6 @@ describe('LanguageProcessor', function() { this.beforeAll(async function() { compiler = new LexicalModelCompiler(); - console.dir(compiler) - console.dir(compiler.init) assert.isTrue(await compiler.init(callbacks, {})); }); From 2a737258393a06deb01fdcf41262251ea9aaaad3 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 13 Dec 2023 08:59:57 +0700 Subject: [PATCH 14/14] chore(developer): fix duplicated message number --- developer/src/kmc/src/messages/infrastructureMessages.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer/src/kmc/src/messages/infrastructureMessages.ts b/developer/src/kmc/src/messages/infrastructureMessages.ts index dd49272325..0100bdccf7 100644 --- a/developer/src/kmc/src/messages/infrastructureMessages.ts +++ b/developer/src/kmc/src/messages/infrastructureMessages.ts @@ -94,6 +94,6 @@ export class InfrastructureMessages { static Error_OutFileCanOnlyBeSpecifiedWithSingleInfile = () => m(this.ERROR_OutFileCanOnlyBeSpecifiedWithSingleInfile, `Parameter --out-file can only be used with a single input file.`); - static ERROR_OutFileCanOnlyBeSpecifiedWithSingleInfile = SevError | 0x0014; + static ERROR_OutFileCanOnlyBeSpecifiedWithSingleInfile = SevError | 0x0015; }